Copy_of_5_Functions_continued.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Princeton-CDH/python4poets/blob/main/Copy_of_5_Functions_continued.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 5. Processing Natural Language"
      ],
      "metadata": {
        "id": "fYHXJGtxYR5F"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4pXHtz5hZlE9"
      },
      "source": [
        "## Oulipo constraints\n",
        "\n",
        "#### Lipogram\n",
        "Writing that excludes one or more letters. The previous sentence is a lipogram in B, F, J, K, Q, V, Y, and Z (it does not contain any of those letters)."
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -qU git+https://github.com/Princeton-CDH/python4poets\n",
        "from python4poets import *"
      ],
      "metadata": {
        "id": "xsC7EU2u9YnN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e0b1639e-1219-4887-e674-07cb702072ff"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "austen"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "id": "cbaq3z8Ln-Md",
        "outputId": "a4295f64-c8f0-448b-b941-9a9ea341dccf"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'It is a truth universally acknowledged, that a single man in possession of a good fortune, must be in want of a wife.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenize(austen)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_oiMH_uwn_IR",
        "outputId": "ba54ee6e-30d3-4944-b634-27a96b042532"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['It',\n",
              " 'is',\n",
              " 'a',\n",
              " 'truth',\n",
              " 'universally',\n",
              " 'acknowledged',\n",
              " ',',\n",
              " 'that',\n",
              " 'a',\n",
              " 'single',\n",
              " 'man',\n",
              " 'in',\n",
              " 'possession',\n",
              " 'of',\n",
              " 'a',\n",
              " 'good',\n",
              " 'fortune',\n",
              " ',',\n",
              " 'must',\n",
              " 'be',\n",
              " 'in',\n",
              " 'want',\n",
              " 'of',\n",
              " 'a',\n",
              " 'wife',\n",
              " '.']"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def lipogram(sentence, bad_letters=''):\n",
        "    # filter the words so that only those words without these letters remain\n",
        "    # words = [\n",
        "    #     word\n",
        "    #     for word in tokenize(sentence)\n",
        "    #     if not set(word) & set(bad_letters)\n",
        "    # ]\n",
        "\n",
        "    words_all = tokenize(sentence)\n",
        "\n",
        "\n",
        "    # return those words put back together into a sentence string\n",
        "    return untokenize(words)\n"
      ],
      "metadata": {
        "id": "TDCCgz1R8xNl"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentence = \"Writing that excludes one or more letters.\""
      ],
      "metadata": {
        "id": "U-EHokkG9sav"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "words_all = tokenize(sentence)\n",
        "words_all"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CrbCYeRbpZhu",
        "outputId": "4121f399-3d5e-4458-b462-6acd2c8a872d"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Writing', 'that', 'excludes', 'one', 'or', 'more', 'letters', '.']"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def lipogram2(sentence, letter):\n",
        "    # string input -> tokenized -> list\n",
        "    words_all = tokenize(sentence)\n",
        "\n",
        "    # for each word in this list\n",
        "    for word in words_all:\n",
        "\n",
        "        # if the bad letter is not in the word\n",
        "        if letter not in word:\n",
        "\n",
        "            # then print that word\n",
        "            print(word)\n"
      ],
      "metadata": {
        "id": "1__RCisepbt-"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lipogram2(\"Hello world\", 'o')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Oc_gb68iqSta",
        "outputId": "98b8a088-3f99-4710-8848-8d0f734beb3a"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Writing\n",
            "that\n",
            "excludes\n",
            "letters\n",
            ".\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "lipogram(sentence, 'l')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 288
        },
        "id": "t_-DHsSG9q-D",
        "outputId": "ddaddce8-21e8-4df9-cd5c-1182f68ace8d"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-3-e5eb39dc1135>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlipogram\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'l'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-1-ba09bbfa54ee>\u001b[0m in \u001b[0;36mlipogram\u001b[0;34m(sentence, bad_letters)\u001b[0m\n\u001b[1;32m      3\u001b[0m     words = [\n\u001b[1;32m      4\u001b[0m         \u001b[0mword\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mword\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m&\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbad_letters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     ]\n",
            "\u001b[0;31mNameError\u001b[0m: name 'tokenize' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "lipogram(sentence, 'e')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "zh4JbA8N-LbX",
        "outputId": "00067e40-c2d0-4878-c287-30ed1f7f346f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Writing that or.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "lipogram(sentence, 'b')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "cWMZaGXG95YD",
        "outputId": "a491d770-e5a4-48b3-b740-eaf9afbacd07"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Writing that excludes one or more letters.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "lipogram(sentence, 'bfjkqvyz') == sentence"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j8Iz7Ynd97gE",
        "outputId": "cc0693e3-1553-4df9-ce87-b2bc47239500"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "lipogram(austen, 'bfjkqvyz')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "v6pwG0v--xYN",
        "outputId": "0409ddea-4355-4980-e8b2-3be2777c55d2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'It is a truth, that a single man in possession a good, must in want a.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "#### Prisoner's constraint, also called Macao constraint\n",
        "A type of lipogram that omits letters with ascenders and descenders (b, d, f, g, h, j, k, l, p, q, t, and y).\n",
        "\n",
        "∇ *Try creating this function yourself.*"
      ],
      "metadata": {
        "id": "N8LhfegH9Z4N"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def prisoners_constraint(sentence):\n",
        "    return ### something ###"
      ],
      "metadata": {
        "id": "4O8ziEzZ8WdQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "prisoners_constraint(austen)"
      ],
      "metadata": {
        "id": "W3SZDQB9-alr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### S+7, sometimes called N+7\n",
        "Replace every noun in a text with the seventh noun after it in a dictionary. For example, \"Call me Ishmael. Some years ago...\" becomes \"Call me islander. Some yeggs ago...\". Results will vary depending upon the dictionary used. This technique can also be performed on other lexical classes, such as verbs.\n",
        "\n",
        "Draft:\n",
        "\n",
        "```python\n",
        "def oulipo_n7(text):\n",
        "    # get a list of nouns\n",
        "    \n",
        "    # in each list of those nouns, find its corresponding noun 7 nouns up\n",
        "\n",
        "    # return a version of the input text with those correspondences as swapped\n",
        "\n",
        "```"
      ],
      "metadata": {
        "id": "P7uV_pMd7rve"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2dR5XXdBncuH",
        "outputId": "108d00ae-9769-490b-baf4-02989c0b76bb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "40791 ['a-bomb', 'a-hole', 'a-listers', 'a-team', 'aahs', 'aardvark', 'abacus', 'abalone', 'abandonment', 'abasement']\n"
          ]
        }
      ],
      "source": [
        "# get a list of nouns\n",
        "list_of_nouns = get_list_of_nouns()\n",
        "\n",
        "# how many?\n",
        "num_nouns = len(list_of_nouns)\n",
        "\n",
        "print(num_nouns, list_of_nouns[:10])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v2XCg_TAncuH",
        "outputId": "dafd1391-892d-4060-f045-e41d54c7f69a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[('mescaline', 'mesocosms'),\n",
              " ('morphs', 'mortality'),\n",
              " ('regeneration', 'regimens'),\n",
              " ('thumpers', 'thundercloud'),\n",
              " ('acacia', 'academics'),\n",
              " ('regulator', 'rehearing'),\n",
              " ('luggage', 'lumbago'),\n",
              " ('misbehavior', 'miscarriages'),\n",
              " ('developers', 'deviants'),\n",
              " ('mater', 'materialization')]"
            ]
          },
          "execution_count": 173,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# create a \"dictionary\" mapping each noun n to its n+7\n",
        "orig_to_n7 = {\n",
        "    # a mapping of orig noun (located at n) to new noun (located at n7)\n",
        "    list_of_nouns[n] : list_of_nouns[n+7]\n",
        "    for n in range(num_nouns)\n",
        "    if n < num_nouns - 7\n",
        "}\n",
        "\n",
        "# show 10 random examples\n",
        "random.sample(orig_to_n7.items(), 10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mJBYGOpincuH"
      },
      "outputs": [],
      "source": [
        "# use this to swap\n",
        "def oulipo_n7(text):\n",
        "    return swap_words(text, orig_to_n7)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "p_aajM2JPJka",
        "outputId": "643d2654-0302-4479-e967-49db3e3fac99"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'It is a tryout universally acknowledged, that a single mana in possum of a good forty-niners, must be in want of a wigglers.'"
            ]
          },
          "execution_count": 146,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "oulipo_n7(austen)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xo0iUZ36ncuH",
        "outputId": "28e4b2b0-1743-4de4-cb73-b4bb166d29c1"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'Harry had a thin faceful, knobbly knife, black haircut and bright-green eyesores. He wore round glassware held together with a lotus of Sellotape because of all the timespan Dudley had punched him on the nosepiece. The only think-tanks Harry liked about his own appellation was a very thin scaremongering on his forelegs which was shaped like a bombard of lightyears.'"
            ]
          },
          "execution_count": 149,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "oulipo_n7(potter)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iOx7AYK1aI3s"
      },
      "source": [
        "### Other language functions\n",
        "\n",
        "#### Pig-latin?\n",
        "\n",
        "* pig latin -> igpay atinlay\n",
        "* simple -> implesay\n",
        "* smile -> ilesmay"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9G-0yRC9aLDV"
      },
      "outputs": [],
      "source": [
        "def to_pig_latin(english_word):\n",
        "  # find first consonants\n",
        "  \n",
        "  # put consonants at end of word\n",
        "\n",
        "  # add final vowel\n",
        "\n",
        "  # return finished word\n",
        "  pass"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "FINISH THIS FUNCTION: In every comment string, there is a missing line of code. Write that line of code."
      ],
      "metadata": {
        "id": "EhAueg0PFZw2"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 58,
      "metadata": {
        "id": "Kru-luXderzC"
      },
      "outputs": [],
      "source": [
        "def find_first_consonants(word, vowels = {'a','e','i','o','u'}):\n",
        "    \"\"\"\n",
        "    A function which finds the first consonants of any word.\n",
        "    Give it the input `word`, return the first few consonant letters (if any).\n",
        "\n",
        "    For example:\n",
        "    * \"pig\" -> \"p\"\n",
        "    * \"smile\" -> \"sm\"\n",
        "    \"\"\"\n",
        "\n",
        "    punctuation = {',', '.', ';', '!', '?', '...'}\n",
        "\n",
        "    # start an empty \"string\" of consonant letters\n",
        "    consonants = ''\n",
        "\n",
        "    # for each letter in the word we were given...\n",
        "    for letter in word:\n",
        "        # print(f'new letter: {letter}')\n",
        "\n",
        "        # if this letter is a vowel...\n",
        "        if letter in vowels:\n",
        "            # print(f'{letter} is a vowel')\n",
        "            # stop the loop! we've gone too far (remember you can use \"break\" to emergency halt a loop)\n",
        "            break\n",
        "        # otherwise, it's a consonant\n",
        "        elif letter in punctuation:\n",
        "            # skip it?\n",
        "            pass\n",
        "        else:\n",
        "            # print(f'{letter} is a consonant')\n",
        "            # add this consonant to string above\n",
        "            consonants = consonants + letter           # consonants += letter\n",
        "\n",
        "        # print(f'so far: {consonants}')\n",
        "        # print()\n",
        "\n",
        "    # return back the string of consonants\n",
        "    return consonants\n",
        "    "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 59,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "iu56TLVVg46P",
        "outputId": "5ef1b024-1e7f-4fd2-8ce0-3cc4d0fdb81a"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"sm'\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 59
        }
      ],
      "source": [
        "# Is this \"p\"?\n",
        "find_first_consonants('sm\\'eagol')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 60,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "L5_9uMjJhPA5",
        "outputId": "941dbed3-962c-44ab-9ae4-1629b3001444"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'sm'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 60
        }
      ],
      "source": [
        "# Is this \"sm\"?\n",
        "find_first_consonants('smile')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 61,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "dtQqKyoLhWBY",
        "outputId": "f061fad0-6f6d-468a-bd14-17c49e0df2b8"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "''"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 61
        }
      ],
      "source": [
        "# Is this... empty?\n",
        "find_first_consonants('energy')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 62,
      "metadata": {
        "id": "t7g6XRpeer6x"
      },
      "outputs": [],
      "source": [
        "# Let's make a bunch of assertions that OUGHT to be true!\n",
        "# if any of them break, we'll know to fix the function\n",
        "\n",
        "assert find_first_consonants('pig') == 'p'\n",
        "assert find_first_consonants('smile') == 'sm'\n",
        "assert find_first_consonants('joke') == 'j'\n",
        "assert find_first_consonants('energy') == ''\n",
        "assert find_first_consonants('yellow') == 'y'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 62,
      "metadata": {
        "id": "9K7Id047hQ52"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 62,
      "metadata": {
        "id": "W6xtQjm6fMB9"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": 71,
      "metadata": {
        "id": "LjIYClrLfQOi"
      },
      "outputs": [],
      "source": [
        "def to_pig_latin(word):\n",
        "  # find first consonants\n",
        "  first_consonants = find_first_consonants(word)\n",
        "\n",
        "  # move consonants to end of word\n",
        "  num_first_consonants = len(first_consonants)\n",
        "\n",
        "  word_without_first_consonants = word[ num_first_consonants : ]\n",
        "\n",
        "  print(word)\n",
        "  print(first_consonants)\n",
        "  print(num_first_consonants)\n",
        "  print(word_without_first_consonants)\n",
        "  \n",
        "  word_with_first_consonants_moved_to_end = word_without_first_consonants + first_consonants\n",
        "  print(word_with_first_consonants_moved_to_end)\n",
        "\n",
        "#   # add final vowel\n",
        "  word_with_first_consonants_moved_to_end_plus_final_vowel = word_with_first_consonants_moved_to_end + 'ay'\n",
        "  print(word_with_first_consonants_moved_to_end_plus_final_vowel)\n",
        "\n",
        "  return word_with_first_consonants_moved_to_end_plus_final_vowel\n",
        "#   # return finished word\n",
        "#   return ?"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "to_pig_latin('latin')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "id": "kJmpeAhbusHO",
        "outputId": "366e6fe4-da41-4d85-8b75-7793b3ba81fa"
      },
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "latin\n",
            "l\n",
            "1\n",
            "atin\n",
            "atinl\n",
            "atinlay\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'atinlay'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "oREsU460urtu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 74,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "id": "z_ptvzL2gBDc",
        "outputId": "de19a4a9-4620-49a0-88db-8901e7101de9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "smile\n",
            "sm\n",
            "2\n",
            "ile\n",
            "ilesm\n",
            "ilesmay\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'ilesmay'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 74
        }
      ],
      "source": [
        "to_pig_latin('smile')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "1Oe_bt9TgCZA",
        "outputId": "d736bb4b-feb2-4cbe-b4b8-878991753920"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'atinlay'"
            ]
          },
          "execution_count": 39,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "to_pig_latin('latin')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "-QJ3HEORgGjc",
        "outputId": "2bc3c210-c33e-45ec-f2e2-b52c91083269"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'ilesmay'"
            ]
          },
          "execution_count": 41,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "to_pig_latin('smile')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 75,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "id": "gtccL9yxgH3C",
        "outputId": "d91c8135-dfe0-4cb1-e726-6beec770986d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "simple\n",
            "s\n",
            "1\n",
            "imple\n",
            "imples\n",
            "implesay\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'implesay'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 75
        }
      ],
      "source": [
        "to_pig_latin('simple')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## The Three Steps Of 90% NLP\n",
        "\n",
        "Biased to English and languages with whitespace-based tokenization.\n",
        "\n",
        "1. Text filename (string) -> Text (string)\n",
        "\n",
        "```python\n",
        "text_string = read_file(filename_or_url)\n",
        "```\n",
        "\n",
        "2. Text (string) -> Words (list)\n",
        "\n",
        "```python\n",
        "words_list = tokenize(text_string)\n",
        "```\n",
        "\n",
        "3. Words (list) -> Counts (dict)\n",
        "\n",
        "```python\n",
        "word_counts = Counter(words_list)\n",
        "```"
      ],
      "metadata": {
        "id": "1JYKuHiJGL8Z"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Getting texts"
      ],
      "metadata": {
        "id": "nLLKF8HiIdiz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for x in tokenize(austen):\n",
        "    print(x)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PK50HQFroQDl",
        "outputId": "8c3cb225-9807-4469-9e15-ec4b7a652b6e"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['It',\n",
              " 'is',\n",
              " 'a',\n",
              " 'truth',\n",
              " 'universally',\n",
              " 'acknowledged',\n",
              " ',',\n",
              " 'that',\n",
              " 'a',\n",
              " 'single',\n",
              " 'man',\n",
              " 'in',\n",
              " 'possession',\n",
              " 'of',\n",
              " 'a',\n",
              " 'good',\n",
              " 'fortune',\n",
              " ',',\n",
              " 'must',\n",
              " 'be',\n",
              " 'in',\n",
              " 'want',\n",
              " 'of',\n",
              " 'a',\n",
              " 'wife',\n",
              " '.']"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# We can \"clone\" a repository of texts\n",
        "!git clone https://github.com/amardeepmsingh/African-American-Literature-Text-Corpus-1853-1923"
      ],
      "metadata": {
        "id": "TRKn5mfAGKxY",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7426fe45-4ada-4bcc-a747-328ade474b0f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'African-American-Literature-Text-Corpus-1853-1923'...\n",
            "remote: Enumerating objects: 111, done.\u001b[K\n",
            "remote: Total 111 (delta 0), reused 0 (delta 0), pack-reused 111\u001b[K\n",
            "Receiving objects: 100% (111/111), 9.19 MiB | 8.18 MiB/s, done.\n",
            "Resolving deltas: 100% (1/1), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @TODO: Copy path of a text\n",
        "filepath = '/content/African-American-Literature-Text-Corpus-1853-1923/paul-laurence-dunbar-complete-poems-of-paul-laurence-dunbar.txt'"
      ],
      "metadata": {
        "id": "ZtPc959tJJ8V"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "∇ *What is a file path? This is knowledge somehow lost to new generations.*"
      ],
      "metadata": {
        "id": "LiB7LAJoT1z1"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 1 of 90% of all NLP: string of filename -> string of entire text"
      ],
      "metadata": {
        "id": "auIG27QJUc2D"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# let `fulltext` name the contents of the file\n",
        "fulltext = open(filepath).read()\n",
        "\n",
        "# @TODO: show the first 280 characters?\n",
        "print( ? )"
      ],
      "metadata": {
        "id": "1hKPE3HmI4Cw",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 131
        },
        "outputId": "679b21b9-caaf-44eb-92e0-abd574c81fa4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "SyntaxError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-15-fbf89604fc44>\"\u001b[0;36m, line \u001b[0;32m7\u001b[0m\n\u001b[0;31m    print( ? )\u001b[0m\n\u001b[0m           ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "∇ *What is a plain text file (.txt)?*"
      ],
      "metadata": {
        "id": "phFgIusxX070"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Or we can use a URL\n",
        "url = 'https://www.gutenberg.org/cache/epub/12242/pg12242.txt'\n",
        "fulltext_fromonline = get_txt_from_url(url)\n",
        "\n",
        "# @TODO: Show the last 280 characters?\n",
        "print( ? )"
      ],
      "metadata": {
        "id": "xskXPWluUaye",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 131
        },
        "outputId": "90bf845c-50ff-4fba-95c4-a7d12d5ab4ee"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "SyntaxError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;36m  File \u001b[0;32m\"<ipython-input-17-97a6811f4b5f>\"\u001b[0;36m, line \u001b[0;32m6\u001b[0m\n\u001b[0;31m    print( ? )\u001b[0m\n\u001b[0m           ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "∇ *Do you know what Project Gutenberg is? Internet Archive? How to find texts?*"
      ],
      "metadata": {
        "id": "ki3DTLDaXq74"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 2 of 90% of all NLP: string of entire text -> list of words (\"tokenization\")"
      ],
      "metadata": {
        "id": "mNy2hdCSYdlc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def tokenize(text):\n",
        "    import nltk\n",
        "    return nltk.word_tokenize(text)"
      ],
      "metadata": {
        "id": "8uG3n9szZSFa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# austen as a string\n",
        "austen"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "id": "gj1EsK9qYr6n",
        "outputId": "d6d92b18-8ebc-4063-ceb9-e7b58beb3bca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'It is a truth universally acknowledged, that a single man in possession of a good fortune, must be in want of a wife.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def naive_tokenizer(x):\n",
        "    return x.split()"
      ],
      "metadata": {
        "id": "SnzGReqOYkJS"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\"Hello           world\".split()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BO1epDkpo0ng",
        "outputId": "4ebd5545-9eeb-437f-fabe-d037fd9740f9"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Hello', 'world']"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# What does tokenize do?\n",
        "tokenize??"
      ],
      "metadata": {
        "id": "lnpMzjrOYlvC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# what does nltk's tokenize do?\n",
        "nltk.word_tokenize??"
      ],
      "metadata": {
        "id": "MLuPsayHY-vd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def simple_tokenize(text):\n",
        "    text_no_punc = remove_punctuation(text)\n",
        "    return text_no_punc.split()\n",
        "\n",
        "def remove_punctuation(string):\n",
        "    return ''.join(\n",
        "        letter\n",
        "        for letter in string\n",
        "        if letter.isalpha()\n",
        "        or letter == ' '\n",
        "    )\n"
      ],
      "metadata": {
        "id": "nlWNCMYlY_eY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @TODO: What if we wanted to add lowercasing to our tokenizer?"
      ],
      "metadata": {
        "id": "gwoByECQaSbR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "simple_tokenize(austen)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "78WGg8vcaMHz",
        "outputId": "ff32d968-9246-4670-c998-716d255c12fe"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['It',\n",
              " 'is',\n",
              " 'a',\n",
              " 'truth',\n",
              " 'universally',\n",
              " 'acknowledged',\n",
              " 'that',\n",
              " 'a',\n",
              " 'single',\n",
              " 'man',\n",
              " 'in',\n",
              " 'possession',\n",
              " 'of',\n",
              " 'a',\n",
              " 'good',\n",
              " 'fortune',\n",
              " 'must',\n",
              " 'be',\n",
              " 'in',\n",
              " 'want',\n",
              " 'of',\n",
              " 'a',\n",
              " 'wife']"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "nltk.word_tokenize(austen)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hf5HQ1nmaaMe",
        "outputId": "482cf1d4-3e41-478e-a353-72bf9d36a2e8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['It',\n",
              " 'is',\n",
              " 'a',\n",
              " 'truth',\n",
              " 'universally',\n",
              " 'acknowledged',\n",
              " ',',\n",
              " 'that',\n",
              " 'a',\n",
              " 'single',\n",
              " 'man',\n",
              " 'in',\n",
              " 'possession',\n",
              " 'of',\n",
              " 'a',\n",
              " 'good',\n",
              " 'fortune',\n",
              " ',',\n",
              " 'must',\n",
              " 'be',\n",
              " 'in',\n",
              " 'want',\n",
              " 'of',\n",
              " 'a',\n",
              " 'wife',\n",
              " '.']"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# How compare?\n",
        "set(nltk.word_tokenize(austen)) - set(simple_tokenize(austen))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tdFOmX2jaeiC",
        "outputId": "5f76e492-487d-4458-efe0-3090ac470ae0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{',', '.'}"
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @TODO: Do same above over again for 'potter' string."
      ],
      "metadata": {
        "id": "BrGRbJf-al-r"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Step 3 of 90% of all NLP: list of words -> dictionary of word counts"
      ],
      "metadata": {
        "id": "9tpAqZM1YRM1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# step 2 over again\n",
        "words = tokenize(potter)"
      ],
      "metadata": {
        "id": "nQs4u4rrIWAr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# step 3\n",
        "from collections import Counter   # built into python but stored outside global namespace\n",
        "counts = Counter(words)"
      ],
      "metadata": {
        "id": "3wnj0K23azL4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "counts"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UaPjJUN0a7x8",
        "outputId": "b949f91f-9736-4bfe-8f7f-6cee6bda9a47"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Counter({'Harry': 2,\n",
              "         'had': 2,\n",
              "         'a': 4,\n",
              "         'thin': 2,\n",
              "         'face': 1,\n",
              "         ',': 2,\n",
              "         'knobbly': 1,\n",
              "         'knees': 1,\n",
              "         'black': 1,\n",
              "         'hair': 1,\n",
              "         'and': 1,\n",
              "         'bright-green': 1,\n",
              "         'eyes': 1,\n",
              "         '.': 3,\n",
              "         'He': 1,\n",
              "         'wore': 1,\n",
              "         'round': 1,\n",
              "         'glasses': 1,\n",
              "         'held': 1,\n",
              "         'together': 1,\n",
              "         'with': 1,\n",
              "         'lot': 1,\n",
              "         'of': 3,\n",
              "         'Sellotape': 1,\n",
              "         'because': 1,\n",
              "         'all': 1,\n",
              "         'the': 2,\n",
              "         'times': 1,\n",
              "         'Dudley': 1,\n",
              "         'punched': 1,\n",
              "         'him': 1,\n",
              "         'on': 2,\n",
              "         'nose': 1,\n",
              "         'The': 1,\n",
              "         'only': 1,\n",
              "         'thing': 1,\n",
              "         'liked': 1,\n",
              "         'about': 1,\n",
              "         'his': 2,\n",
              "         'own': 1,\n",
              "         'appearance': 1,\n",
              "         'was': 2,\n",
              "         'very': 1,\n",
              "         'scar': 1,\n",
              "         'forehead': 1,\n",
              "         'which': 1,\n",
              "         'shaped': 1,\n",
              "         'like': 1,\n",
              "         'bolt': 1,\n",
              "         'lightning': 1})"
            ]
          },
          "metadata": {},
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Most common?\n",
        "counts.most_common(10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "owTSam_ta-WK",
        "outputId": "4d43a03a-bbbb-45fa-cabb-b9c944688937"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('a', 4),\n",
              " ('.', 3),\n",
              " ('of', 3),\n",
              " ('Harry', 2),\n",
              " ('had', 2),\n",
              " ('thin', 2),\n",
              " (',', 2),\n",
              " ('the', 2),\n",
              " ('on', 2),\n",
              " ('his', 2)]"
            ]
          },
          "metadata": {},
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Remove stopwords?\n",
        "from nltk.corpus import stopwords\n",
        "boring_words = set(stopwords.words('english'))\n",
        "print(boring_words)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JRC5KnF2bIF4",
        "outputId": "ce3f3652-2794-4729-ccc1-1ffc56c739a3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'his', 'doing', 'how', 'while', 'too', 'above', 'isn', 'll', \"you're\", 'themselves', 've', 'only', 'not', 'itself', 'now', 'being', 'is', 'there', 'yourselves', 'if', 'the', 'herself', 'as', 'what', 'so', \"shouldn't\", \"wasn't\", 'by', 'these', 'about', 'couldn', 'weren', \"wouldn't\", 'hadn', 'or', 'y', 'theirs', 'each', 'until', 'were', 'some', \"doesn't\", 'hasn', 'didn', 'an', 'shouldn', 'off', 'because', 'd', 'won', 'such', 'yours', 's', 'no', 'was', 'than', 're', 'their', 'aren', \"mightn't\", 'has', 'just', 'ma', 'once', 'during', 'they', 'have', 'who', 'does', 'm', 'wasn', 'she', 'ours', 'he', 'same', 'her', 'down', 'i', \"should've\", 'and', 'you', 'do', 'him', 'hers', 'your', 'from', 'its', 'had', 'ourselves', \"isn't\", 'own', \"needn't\", 'am', 'which', 'himself', \"it's\", 'out', 'needn', 'other', 'it', 'here', 't', \"that'll\", 'those', \"you've\", 'haven', \"you'll\", 'be', 'through', \"hasn't\", 'we', \"don't\", 'mustn', 'that', 'wouldn', 'having', 'nor', 'doesn', \"couldn't\", 'all', 'then', \"weren't\", 'into', 'a', 'against', 'after', 'don', 'for', 'any', \"hadn't\", \"you'd\", 'are', 'mightn', 'should', 'up', 'over', 'most', 'at', 'with', 'more', 'yourself', 'me', 'between', 'did', 'further', \"didn't\", 'both', \"haven't\", 'been', 'to', 'few', \"she's\", 'on', 'again', 'my', 'will', 'before', 'o', 'myself', 'them', 'can', 'below', 'our', 'very', \"shan't\", 'where', 'when', 'why', 'but', 'of', 'whom', 'under', \"won't\", 'this', 'in', \"mustn't\", \"aren't\", 'ain', 'shan'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# @TODO: How many boring words?\n",
        "\n"
      ],
      "metadata": {
        "id": "WecZSoWIbnoq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# New counter\n",
        "def count_without_stopwords(words):\n",
        "    counts = Counter()\n",
        "    for word in words:\n",
        "        if word not in boring_words:\n",
        "            counts[word] += 1\n",
        "    return counts"
      ],
      "metadata": {
        "id": "_uY3XfVjbpFQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "counts2 = count_without_stopwords(words)\n",
        "counts2"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rsoA1Lk6b_5q",
        "outputId": "defeea2b-0ad4-4445-eb98-91c0a005fc6a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Counter({'Harry': 2,\n",
              "         'thin': 2,\n",
              "         'face': 1,\n",
              "         ',': 2,\n",
              "         'knobbly': 1,\n",
              "         'knees': 1,\n",
              "         'black': 1,\n",
              "         'hair': 1,\n",
              "         'bright-green': 1,\n",
              "         'eyes': 1,\n",
              "         '.': 3,\n",
              "         'He': 1,\n",
              "         'wore': 1,\n",
              "         'round': 1,\n",
              "         'glasses': 1,\n",
              "         'held': 1,\n",
              "         'together': 1,\n",
              "         'lot': 1,\n",
              "         'Sellotape': 1,\n",
              "         'times': 1,\n",
              "         'Dudley': 1,\n",
              "         'punched': 1,\n",
              "         'nose': 1,\n",
              "         'The': 1,\n",
              "         'thing': 1,\n",
              "         'liked': 1,\n",
              "         'appearance': 1,\n",
              "         'scar': 1,\n",
              "         'forehead': 1,\n",
              "         'shaped': 1,\n",
              "         'like': 1,\n",
              "         'bolt': 1,\n",
              "         'lightning': 1})"
            ]
          },
          "metadata": {},
          "execution_count": 79
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "counts2.most_common(10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fr-F0a5bcHEA",
        "outputId": "f39b9362-b508-4bda-f6e7-e9f622d9dd3c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('.', 3),\n",
              " ('Harry', 2),\n",
              " ('thin', 2),\n",
              " (',', 2),\n",
              " ('face', 1),\n",
              " ('knobbly', 1),\n",
              " ('knees', 1),\n",
              " ('black', 1),\n",
              " ('hair', 1),\n",
              " ('bright-green', 1)]"
            ]
          },
          "metadata": {},
          "execution_count": 80
        }
      ]
    }
  ]
}