preprocess_full.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DaeIXsUWphsd"
      },
      "source": [
        "IMPORT THƯ VIỆN"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ptB_NwXtphsh",
        "outputId": "b2bed12c-9ad8-409a-e2ef-d7b523762080"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import re\n",
        "import nltk\n",
        "import spacy\n",
        "import string\n",
        "from bs4 import BeautifulSoup\n",
        "pd.options.mode.chained_assignment = None\n",
        "from re import sub\n",
        "import csv\n",
        "import html as ihtml"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "7CvFdwRJphsi"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 84772 entries, 0 to 84771\n",
            "Data columns (total 5 columns):\n",
            " #   Column     Non-Null Count  Dtype \n",
            "---  ------     --------------  ----- \n",
            " 0   Name       84749 non-null  object\n",
            " 1   Comment    84762 non-null  object\n",
            " 2   Time       84771 non-null  object\n",
            " 3   Store      84758 non-null  object\n",
            " 4   Translate  84772 non-null  object\n",
            "dtypes: object(5)\n",
            "memory usage: 3.2+ MB\n"
          ]
        }
      ],
      "source": [
        "df = pd.read_csv(\"rawDataset - rawDataset.csv\")\n",
        "df.info()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "v9NOE_awLDNB"
      },
      "outputs": [],
      "source": [
        "df = df.drop('Comment', axis=1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "iXrXxg2eLU0Q"
      },
      "outputs": [],
      "source": [
        "df.columns = df.columns.str.replace('Translate', 'Comment')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "EOkjiJ4VMXJ3"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name</th>\n",
              "      <th>Time</th>\n",
              "      <th>Store</th>\n",
              "      <th>Comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Lisa Shen</td>\n",
              "      <td>08-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>every introvert&amp;#39;s dream</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>WTFIWFYDB</td>\n",
              "      <td>09-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>Not really, there are a lot of people inside r...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>lol exactly</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>@WTFIWFYDB wtf are you talking about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Soundwave</td>\n",
              "      <td>11-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>This won’t change anything for introverts; the...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84767</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>Okay thank you</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84768</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>The actual reason is that I want to be able to...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84769</th>\n",
              "      <td>Hookedmedia.1</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>Why</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84770</th>\n",
              "      <td>Kelvin_kufley</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>Why cause you can’t steal?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84771</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>Yes that’s literally exactly the reason</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>84772 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                Name        Time       Store  \\\n",
              "0          Lisa Shen  08-03-2020   Amazon Go   \n",
              "1          WTFIWFYDB  09-03-2020   Amazon Go   \n",
              "2        John Miller  10-03-2020   Amazon Go   \n",
              "3        John Miller  10-03-2020   Amazon Go   \n",
              "4          Soundwave  11-03-2020   Amazon Go   \n",
              "...              ...         ...         ...   \n",
              "84767  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "84768  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "84769  Hookedmedia.1  02/11/2022  Smart Cart   \n",
              "84770  Kelvin_kufley  02/11/2022  Smart Cart   \n",
              "84771  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "\n",
              "                                                 Comment  \n",
              "0                            every introvert&#39;s dream  \n",
              "1      Not really, there are a lot of people inside r...  \n",
              "2                                            lol exactly  \n",
              "3                   @WTFIWFYDB wtf are you talking about  \n",
              "4      This won’t change anything for introverts; the...  \n",
              "...                                                  ...  \n",
              "84767                                     Okay thank you  \n",
              "84768  The actual reason is that I want to be able to...  \n",
              "84769                                                Why  \n",
              "84770                         Why cause you can’t steal?  \n",
              "84771            Yes that’s literally exactly the reason  \n",
              "\n",
              "[84772 rows x 4 columns]"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e-8x14vGphsl"
      },
      "source": [
        "1. XÓA URL"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "qQ1pFEhv0-Ao"
      },
      "outputs": [],
      "source": [
        "df[\"Comment\"] = df[\"Comment\"].astype(str)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "l21nLUCBphsm"
      },
      "outputs": [],
      "source": [
        "def clean_text(text):\n",
        "    text = BeautifulSoup(ihtml.unescape(text)).text\n",
        "    text = re.sub(r\"http[s]?://\\S+\", \"\", text)\n",
        "    text = re.sub(r\"\\s+\", \" \", text)    \n",
        "    return text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "8G-ZFCop0_7s"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\HienPhuong\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\bs4\\__init__.py:435: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
            "  warnings.warn(\n",
            "C:\\Users\\HienPhuong\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python310\\site-packages\\bs4\\__init__.py:404: MarkupResemblesLocatorWarning: The input looks more like a URL than markup. You may want to use an HTTP client like requests to get the document behind the URL, and feed that document to Beautiful Soup.\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "for i in range(len(df)):\n",
        "  df.Comment[i]=clean_text(df.Comment[i])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "uTPjUeQHphsm"
      },
      "outputs": [],
      "source": [
        "def remove_urls(text):\n",
        "    url_pattern = re.compile(r'https?://\\S+|www\\.\\S+')\n",
        "    return url_pattern.sub(r'', text)\n",
        "for i in range(len(df)):\n",
        "  df.Comment[i]=remove_urls(df.Comment[i])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "CErLX2n8phsm"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0                                  every introvert's dream\n",
              "1        Not really, there are a lot of people inside r...\n",
              "2                                              lol exactly\n",
              "3                     @WTFIWFYDB wtf are you talking about\n",
              "4        This won’t change anything for introverts; the...\n",
              "                               ...                        \n",
              "84767                                       Okay thank you\n",
              "84768    The actual reason is that I want to be able to...\n",
              "84769                                                  Why\n",
              "84770                           Why cause you can’t steal?\n",
              "84771              Yes that’s literally exactly the reason\n",
              "Name: Comment, Length: 84772, dtype: object"
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df.Comment"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "remove html tag"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name</th>\n",
              "      <th>Time</th>\n",
              "      <th>Store</th>\n",
              "      <th>Comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Lisa Shen</td>\n",
              "      <td>08-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>every introvert's dream</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>WTFIWFYDB</td>\n",
              "      <td>09-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>Not really, there are a lot of people inside r...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>lol exactly</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>@WTFIWFYDB wtf are you talking about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Soundwave</td>\n",
              "      <td>11-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>This won’t change anything for introverts; the...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          Name        Time      Store  \\\n",
              "0    Lisa Shen  08-03-2020  Amazon Go   \n",
              "1    WTFIWFYDB  09-03-2020  Amazon Go   \n",
              "2  John Miller  10-03-2020  Amazon Go   \n",
              "3  John Miller  10-03-2020  Amazon Go   \n",
              "4    Soundwave  11-03-2020  Amazon Go   \n",
              "\n",
              "                                             Comment  \n",
              "0                            every introvert's dream  \n",
              "1  Not really, there are a lot of people inside r...  \n",
              "2                                        lol exactly  \n",
              "3               @WTFIWFYDB wtf are you talking about  \n",
              "4  This won’t change anything for introverts; the...  "
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "def remove_html(text):\n",
        "    html_pattern = re.compile('<.*?>')\n",
        "    return html_pattern.sub(r'', text)\n",
        "\n",
        "df[\"Comment\"] = df[\"Comment\"].apply(lambda text: remove_html(text))\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nzN28EY3phso"
      },
      "source": [
        "Expand Constraction "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "4cMgSKlJphso"
      },
      "outputs": [],
      "source": [
        "# Dictionary of English Contractions\n",
        "contractions_dict = { \"ain't\": \"are not\",\"'s\":\" is\",\"aren't\": \"are not\",\n",
        "                     \"can't\": \"cannot\",\"can't've\": \"cannot have\",\n",
        "                     \"'cause\": \"because\",\"could've\": \"could have\",\"couldn't\": \"could not\",\n",
        "                     \"couldn't've\": \"could not have\", \"didn't\": \"did not\",\"doesn't\": \"does not\",\n",
        "                     \"don't\": \"do not\",\"hadn't\": \"had not\",\"hadn't've\": \"had not have\",\n",
        "                     \"hasn't\": \"has not\",\"haven't\": \"have not\",\"he'd\": \"he would\",\n",
        "                     \"he'd've\": \"he would have\",\"he'll\": \"he will\", \"he'll've\": \"he will have\",\n",
        "                     \"how'd\": \"how did\",\"how'd'y\": \"how do you\",\"how'll\": \"how will\",\n",
        "                     \"i'd\": \"i would\", \"i'd've\": \"i would have\",\"i'll\": \"i will\", \n",
        "                     \"i'll've\": \"i will have\",\"i'm\": \"i am\",\"i've\": \"i have\", \"isn't\": \"is not\",\n",
        "                     \"it'd\": \"it would\",\"it'd've\": \"it would have\",\"it'll\": \"it will\",\n",
        "                     \"it'll've\": \"it will have\", \"let's\": \"let us\",\"ma'am\": \"madam\",\n",
        "                     \"mayn't\": \"may not\",\"might've\": \"might have\",\"mightn't\": \"might not\", \n",
        "                     \"mightn't've\": \"might not have\",\"must've\": \"must have\",\"mustn't\": \"must not\",\n",
        "                     \"mustn't've\": \"must not have\", \"needn't\": \"need not\",\n",
        "                     \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\",\"oughtn't\": \"ought not\",\n",
        "                     \"oughtn't've\": \"ought not have\",\"shan't\": \"shall not\",\"sha'n't\": \"shall not\",\n",
        "                     \"shan't've\": \"shall not have\",\"she'd\": \"she would\",\"she'd've\": \"she would have\",\n",
        "                     \"she'll\": \"she will\", \"she'll've\": \"she will have\",\"should've\": \"should have\",\n",
        "                     \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\",\"so've\": \"so have\",\n",
        "                     \"that'd\": \"that would\",\"that'd've\": \"that would have\", \"there'd\": \"there would\",\n",
        "                     \"there'd've\": \"there would have\", \"they'd\": \"they would\",\n",
        "                     \"they'd've\": \"they would have\",\"they'll\": \"they will\",\n",
        "                     \"they'll've\": \"they will have\", \"they're\": \"they are\",\"they've\": \"they have\",\n",
        "                     \"to've\": \"to have\",\"wasn't\": \"was not\",\"we'd\": \"we would\",\n",
        "                     \"we'd've\": \"we would have\",\"we'll\": \"we will\",\"we'll've\": \"we will have\",\n",
        "                     \"we're\": \"we are\",\"we've\": \"we have\", \"weren't\": \"were not\",\"what'll\": \"what will\",\n",
        "                     \"what'll've\": \"what will have\",\"what're\": \"what are\", \"what've\": \"what have\",\n",
        "                     \"when've\": \"when have\",\"where'd\": \"where did\", \"where've\": \"where have\",\n",
        "                     \"who'll\": \"who will\",\"who'll've\": \"who will have\",\"who've\": \"who have\",\n",
        "                     \"why've\": \"why have\",\"will've\": \"will have\",\"won't\": \"will not\",\n",
        "                     \"won't've\": \"will not have\", \"would've\": \"would have\",\"wouldn't\": \"would not\",\n",
        "                     \"wouldn't've\": \"would not have\",\"y'all\": \"you all\", \"y'all'd\": \"you all would\",\n",
        "                     \"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\n",
        "                     \"y'all've\": \"you all have\", \"you'd\": \"you would\",\"you'd've\": \"you would have\",\n",
        "                     \"you'll\": \"you will\",\"you'll've\": \"you will have\", \"you're\": \"you are\",\n",
        "                     \"you've\": \"you have\"}\n",
        "\n",
        "# Regular expression for finding contractions\n",
        "contractions_re=re.compile('(%s)' % '|'.join(contractions_dict.keys()))\n",
        "\n",
        "# Function for expanding contractions\n",
        "def expand_contractions(text,contractions_dict=contractions_dict):\n",
        "  def replace(match):\n",
        "    return contractions_dict[match.group(0)]\n",
        "  return contractions_re.sub(replace, text)\n",
        "\n",
        "# Expanding Contractions in the reviews\n",
        "df['Comment']=df['Comment'].apply(lambda x:expand_contractions(x))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "_czsOTG4v77-",
        "outputId": "7a6b4d16-9450-4af4-ca80-8ae4439fca47"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name</th>\n",
              "      <th>Time</th>\n",
              "      <th>Store</th>\n",
              "      <th>Comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Lisa Shen</td>\n",
              "      <td>08-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>every introvert is dream</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>WTFIWFYDB</td>\n",
              "      <td>09-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>Not really, there are a lot of people inside r...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>lol exactly</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>@WTFIWFYDB wtf are you talking about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Soundwave</td>\n",
              "      <td>11-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>This won’t change anything for introverts; the...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84767</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>Okay thank you</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84768</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>The actual reason is that I want to be able to...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84769</th>\n",
              "      <td>Hookedmedia.1</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>Why</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84770</th>\n",
              "      <td>Kelvin_kufley</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>Why cause you can’t steal?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84771</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>Yes that’s literally exactly the reason</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>84772 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                Name        Time       Store  \\\n",
              "0          Lisa Shen  08-03-2020   Amazon Go   \n",
              "1          WTFIWFYDB  09-03-2020   Amazon Go   \n",
              "2        John Miller  10-03-2020   Amazon Go   \n",
              "3        John Miller  10-03-2020   Amazon Go   \n",
              "4          Soundwave  11-03-2020   Amazon Go   \n",
              "...              ...         ...         ...   \n",
              "84767  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "84768  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "84769  Hookedmedia.1  02/11/2022  Smart Cart   \n",
              "84770  Kelvin_kufley  02/11/2022  Smart Cart   \n",
              "84771  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "\n",
              "                                                 Comment  \n",
              "0                               every introvert is dream  \n",
              "1      Not really, there are a lot of people inside r...  \n",
              "2                                            lol exactly  \n",
              "3                   @WTFIWFYDB wtf are you talking about  \n",
              "4      This won’t change anything for introverts; the...  \n",
              "...                                                  ...  \n",
              "84767                                     Okay thank you  \n",
              "84768  The actual reason is that I want to be able to...  \n",
              "84769                                                Why  \n",
              "84770                         Why cause you can’t steal?  \n",
              "84771            Yes that’s literally exactly the reason  \n",
              "\n",
              "[84772 rows x 4 columns]"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ss_zqIgd1ZhA"
      },
      "source": [
        "in lớn để chat word "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "8WqEI54f1XBo"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0                                 EVERY INTROVERT IS DREAM\n",
              "1        NOT REALLY, THERE ARE A LOT OF PEOPLE INSIDE R...\n",
              "2                                              LOL EXACTLY\n",
              "3                     @WTFIWFYDB WTF ARE YOU TALKING ABOUT\n",
              "4        THIS WON’T CHANGE ANYTHING FOR INTROVERTS; THE...\n",
              "                               ...                        \n",
              "84767                                       OKAY THANK YOU\n",
              "84768    THE ACTUAL REASON IS THAT I WANT TO BE ABLE TO...\n",
              "84769                                                  WHY\n",
              "84770                           WHY CAUSE YOU CAN’T STEAL?\n",
              "84771              YES THAT’S LITERALLY EXACTLY THE REASON\n",
              "Name: Comment, Length: 84772, dtype: object"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df[\"Comment\"] = df[\"Comment\"].str.upper()\n",
        "df.Comment"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sRzUKtFFphsm"
      },
      "source": [
        "Chuẩn hóa 1 số từ viết tắt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "qyhSH-1Bphsn"
      },
      "outputs": [],
      "source": [
        "chat_words_str = \"\"\"\n",
        "AFAIK=As Far As I Know\n",
        "AFK=Away From Keyboard\n",
        "ASAP=As Soon As Possible\n",
        "ATK=At The Keyboard\n",
        "ATM=At The Moment\n",
        "A3=Anytime, Anywhere, Anyplace\n",
        "BAK=Back At Keyboard\n",
        "BBL=Be Back Later\n",
        "BBS=Be Back Soon\n",
        "BFN=Bye For Now\n",
        "B4N=Bye For Now\n",
        "BRB=Be Right Back\n",
        "BRT=Be Right There\n",
        "BTW=By The Way\n",
        "B4=Before\n",
        "B4N=Bye For Now\n",
        "CU=See You\n",
        "CUL8R=See You Later\n",
        "CYA=See You\n",
        "FAQ=Frequently Asked Questions\n",
        "FC=Fingers Crossed\n",
        "FWIW=For What It's Worth\n",
        "FYI=For Your Information\n",
        "GAL=Get A Life\n",
        "GG=Good Game\n",
        "GN=Good Night\n",
        "GMTA=Great Minds Think Alike\n",
        "GR8=Great!\n",
        "G9=Genius\n",
        "IC=I See\n",
        "ICQ=I Seek you (also a chat program)\n",
        "ILU=ILU: I Love You\n",
        "IMHO=In My Honest/Humble Opinion\n",
        "IMO=In My Opinion\n",
        "IOW=In Other Words\n",
        "IRL=In Real Life\n",
        "KISS=Keep It Simple, Stupid\n",
        "LDR=Long Distance Relationship\n",
        "LMAO=Laugh My A.. Off\n",
        "LOL=Laughing Out Loud\n",
        "LTNS=Long Time No See\n",
        "L8R=Later\n",
        "MTE=My Thoughts Exactly\n",
        "M8=Mate\n",
        "NRN=No Reply Necessary\n",
        "OIC=Oh I See\n",
        "PITA=Pain In The A..\n",
        "PRT=Party\n",
        "PRW=Parents Are Watching\n",
        "ROFL=Rolling On The Floor Laughing\n",
        "ROFLOL=Rolling On The Floor Laughing Out Loud\n",
        "ROTFLMAO=Rolling On The Floor Laughing My A.. Off\n",
        "SK8=Skate\n",
        "STATS=Your sex and age\n",
        "ASL=Age, Sex, Location\n",
        "THX=Thank You\n",
        "TTFN=Ta-Ta For Now!\n",
        "TTYL=Talk To You Later\n",
        "U=You\n",
        "U2=You Too\n",
        "U4E=Yours For Ever\n",
        "WB=Welcome Back\n",
        "WTF=What The F...\n",
        "WTG=Way To Go!\n",
        "WUF=Where Are You From?\n",
        "W8=Wait...\n",
        "7K=Sick:-D Laugher\n",
        "&=and\n",
        "\"\"\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "wqTjgERWphsn"
      },
      "outputs": [],
      "source": [
        "chat_words_map_dict = {}\n",
        "chat_words_list = []\n",
        "for line in chat_words_str.split(\"\\n\"):\n",
        "    if line != \"\":\n",
        "        cw = line.split(\"=\")[0]\n",
        "        cw_expanded = line.split(\"=\")[1]\n",
        "        chat_words_list.append(cw)\n",
        "        chat_words_map_dict[cw] = cw_expanded\n",
        "chat_words_list = set(chat_words_list)\n",
        "\n",
        "def chat_words_conversion(text):\n",
        "    new_text = []\n",
        "    for w in text.split():\n",
        "        if w.upper() in chat_words_list:\n",
        "            new_text.append(chat_words_map_dict[w.upper()])\n",
        "        else:\n",
        "            new_text.append(w)\n",
        "    return \" \".join(new_text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "UpxDirYA1Rmc"
      },
      "outputs": [],
      "source": [
        "for i in range(len(df)):\n",
        "  df.Comment[i]=chat_words_conversion(df.Comment[i])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IlXJ47Xnphsn"
      },
      "source": [
        "2. Đưa dữ liệu về chữ thường"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mlsQNd_cphso",
        "outputId": "2409a97b-c272-4aea-bc57-f9443901214e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0                                 every introvert is dream\n",
              "1        not really, there are a lot of people inside r...\n",
              "2                                laughing out loud exactly\n",
              "3           @wtfiwfydb what the f... are you talking about\n",
              "4        this won’t change anything for introverts; the...\n",
              "                               ...                        \n",
              "84767                                       okay thank you\n",
              "84768    the actual reason is that i want to be able to...\n",
              "84769                                                  why\n",
              "84770                           why cause you can’t steal?\n",
              "84771              yes that’s literally exactly the reason\n",
              "Name: Comment, Length: 84772, dtype: object"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df[\"Comment\"] = df[\"Comment\"].str.lower()\n",
        "df.Comment"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {},
      "outputs": [],
      "source": [
        "from itertools import groupby"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {},
      "outputs": [],
      "source": [
        "def remove_repeatcharacter(text):\n",
        "    return ''.join(''.join(s)[:2] for _, s in groupby(text))\n",
        "df[\"Comment\"] = df[\"Comment\"].apply(lambda text: remove_repeatcharacter(text))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\HienPhuong\\AppData\\Local\\Temp\\ipykernel_16180\\1403891560.py:2: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  df[\"Comment\"] = df[\"Comment\"].str.replace(\"@[A-Za-z0-9]+\",\"\")\n"
          ]
        }
      ],
      "source": [
        "# remove tag @username\n",
        "df[\"Comment\"] = df[\"Comment\"].str.replace(\"@[A-Za-z0-9]+\",\"\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name</th>\n",
              "      <th>Time</th>\n",
              "      <th>Store</th>\n",
              "      <th>Comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Lisa Shen</td>\n",
              "      <td>08-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>every introvert is dream</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>WTFIWFYDB</td>\n",
              "      <td>09-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>not really, there are a lot of people inside r...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>laughing out loud exactly</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>what the f.. are you talking about</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Soundwave</td>\n",
              "      <td>11-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>this won’t change anything for introverts; the...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84767</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>okay thank you</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84768</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>the actual reason is that i want to be able to...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84769</th>\n",
              "      <td>Hookedmedia.1</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>why</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84770</th>\n",
              "      <td>Kelvin_kufley</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>why cause you can’t steal?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84771</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>yes that’s literally exactly the reason</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>84772 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                Name        Time       Store  \\\n",
              "0          Lisa Shen  08-03-2020   Amazon Go   \n",
              "1          WTFIWFYDB  09-03-2020   Amazon Go   \n",
              "2        John Miller  10-03-2020   Amazon Go   \n",
              "3        John Miller  10-03-2020   Amazon Go   \n",
              "4          Soundwave  11-03-2020   Amazon Go   \n",
              "...              ...         ...         ...   \n",
              "84767  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "84768  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "84769  Hookedmedia.1  02/11/2022  Smart Cart   \n",
              "84770  Kelvin_kufley  02/11/2022  Smart Cart   \n",
              "84771  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "\n",
              "                                                 Comment  \n",
              "0                               every introvert is dream  \n",
              "1      not really, there are a lot of people inside r...  \n",
              "2                              laughing out loud exactly  \n",
              "3                     what the f.. are you talking about  \n",
              "4      this won’t change anything for introverts; the...  \n",
              "...                                                  ...  \n",
              "84767                                     okay thank you  \n",
              "84768  the actual reason is that i want to be able to...  \n",
              "84769                                                why  \n",
              "84770                         why cause you can’t steal?  \n",
              "84771            yes that’s literally exactly the reason  \n",
              "\n",
              "[84772 rows x 4 columns]"
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oHa7MJezphsp"
      },
      "source": [
        "Chuyển emoji, emoticon -> text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pqK8bTUIphsp",
        "outputId": "d0083a7b-4fb0-4e44-e098-f689c7faf48c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: emot in c:\\users\\hienphuong\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.10_qbz5n2kfra8p0\\localcache\\local-packages\\python310\\site-packages (3.1)\n"
          ]
        }
      ],
      "source": [
        "!pip install emot"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JA67APkNphsp",
        "outputId": "8530676b-0000-40a4-cc21-b39112862f68"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'Happy face or smiley 😊'"
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import re\n",
        "from emot.emo_unicode import EMOTICONS_EMO\n",
        "def convert_emoticons(text):\n",
        "    for emot in EMOTICONS_EMO:\n",
        "        text = text.replace(emot, \" \".join(EMOTICONS_EMO[emot].replace(\",\",\"\").replace(\":\",\"\").split()))\n",
        "    return text\n",
        "\n",
        "convert_emoticons(\":) 😊\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "knWoNpnCphsp",
        "outputId": "7b9892ca-21f0-471c-89a6-28b406aaa06c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: emoji in c:\\users\\hienphuong\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.10_qbz5n2kfra8p0\\localcache\\local-packages\\python310\\site-packages (2.2.0)\n",
            "Note: you may need to restart the kernel to use updated packages.\n"
          ]
        }
      ],
      "source": [
        "pip install emoji"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YHiGSEYQphsq",
        "outputId": "33894273-3732-46f3-889d-4d0529b82f04"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: demoji in c:\\users\\hienphuong\\appdata\\local\\packages\\pythonsoftwarefoundation.python.3.10_qbz5n2kfra8p0\\localcache\\local-packages\\python310\\site-packages (1.1.0)\n",
            "Note: you may need to restart the kernel to use updated packages.\n"
          ]
        }
      ],
      "source": [
        "pip install demoji"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "QYQFQLF-phsq"
      },
      "outputs": [],
      "source": [
        "import emoji\n",
        "def extract_emojis(text):\n",
        "     new_text = []\n",
        "     new_text.append(emoji.demojize(text, delimiters=(\"\", \"\")))\n",
        "     return \" \".join(new_text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T5Nn2yLawxPR",
        "outputId": "ceeba5b7-fb2d-4706-b0eb-d8ddc0cd0ea0"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'smiling_face_with_smiling_eyes'"
            ]
          },
          "execution_count": 28,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "extract_emojis(\"😊\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "id": "1Q4UpOuuwxoA"
      },
      "outputs": [],
      "source": [
        "for i in range(len(df)):\n",
        "  df.Comment[i]=extract_emojis(df.Comment[i])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "id": "YavGg7jzzZW_"
      },
      "outputs": [],
      "source": [
        "df['Comment'] = df['Comment'].str.replace(\"_\", \" \")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "id": "n1HudLyCzuXp"
      },
      "outputs": [],
      "source": [
        "df['Comment'] = df['Comment'].str.replace(\"-\", \" \")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "id": "rdt2K66jz1En"
      },
      "outputs": [],
      "source": [
        "df['Comment'] = df['Comment'].str.replace(\"  \", \" \")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IoO-1Fx5z_16",
        "outputId": "68cf0ad0-d6b1-45ac-e430-f095aaaf1d92"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\HienPhuong\\AppData\\Local\\Temp\\ipykernel_16180\\1485136920.py:1: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  df.Comment= df.Comment.str.replace(r'\\b\\w\\b', '').str.replace(r'\\s+', ' ')\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "0                                 every introvert is dream\n",
              "1        not really, there are lot of people inside reg...\n",
              "2                                laughing out loud exactly\n",
              "3                        what the .. are you talking about\n",
              "4        this won’ change anything for introverts; ther...\n",
              "                               ...                        \n",
              "84767                                       okay thank you\n",
              "84768    the actual reason is that want to be able to s...\n",
              "84769                                                  why\n",
              "84770                            why cause you can’ steal?\n",
              "84771               yes that’ literally exactly the reason\n",
              "Name: Comment, Length: 84772, dtype: object"
            ]
          },
          "execution_count": 33,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df.Comment= df.Comment.str.replace(r'\\b\\w\\b', '').str.replace(r'\\s+', ' ')\n",
        "df.Comment"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1cbp7m5W0ITj",
        "outputId": "b71d2436-f41e-4b61-ae1d-f443eba333b7"
      },
      "outputs": [],
      "source": [
        "# lemmatize words with spacy\n",
        "import spacy\n",
        "nlp = spacy.load('en_core_web_sm')\n",
        "def lemmatize_words(text):\n",
        "    return \" \".join([token.lemma_ for token in nlp(text)])\n",
        "text = \"last day i did homework, walked . now i am going to work. today he goes to school, police walks to school\"\n",
        "lemmatize_words(text)\n",
        "# apply lemmatize_words function to comment column\n",
        "df[\"Comment\"] = df[\"Comment\"].apply(lambda text: lemmatize_words(text))"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "tách câu"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name</th>\n",
              "      <th>Time</th>\n",
              "      <th>Store</th>\n",
              "      <th>Comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Lisa Shen</td>\n",
              "      <td>08-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>every introvert be dream</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>WTFIWFYDB</td>\n",
              "      <td>09-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>not really , there be lot of people inside reg...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>WTFIWFYDB</td>\n",
              "      <td>09-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>laugh out loud exactly</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>what the</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>252733</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>okay thank you</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>252734</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>the actual reason be that want to be able to s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>252735</th>\n",
              "      <td>Hookedmedia.1</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>why</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>252736</th>\n",
              "      <td>Kelvin_kufley</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>why cause you can ' steal ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>252737</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>yes that ' literally exactly the reason</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>252738 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                 Name        Time       Store  \\\n",
              "0           Lisa Shen  08-03-2020   Amazon Go   \n",
              "1           WTFIWFYDB  09-03-2020   Amazon Go   \n",
              "2           WTFIWFYDB  09-03-2020   Amazon Go   \n",
              "3         John Miller  10-03-2020   Amazon Go   \n",
              "4         John Miller  10-03-2020   Amazon Go   \n",
              "...               ...         ...         ...   \n",
              "252733  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "252734  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "252735  Hookedmedia.1  02/11/2022  Smart Cart   \n",
              "252736  Kelvin_kufley  02/11/2022  Smart Cart   \n",
              "252737  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "\n",
              "                                                  Comment  \n",
              "0                                every introvert be dream  \n",
              "1       not really , there be lot of people inside reg...  \n",
              "2                                                          \n",
              "3                                  laugh out loud exactly  \n",
              "4                                               what the   \n",
              "...                                                   ...  \n",
              "252733                                     okay thank you  \n",
              "252734  the actual reason be that want to be able to s...  \n",
              "252735                                                why  \n",
              "252736                        why cause you can ' steal ?  \n",
              "252737            yes that ' literally exactly the reason  \n",
              "\n",
              "[252738 rows x 4 columns]"
            ]
          },
          "execution_count": 36,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df = (\n",
        " df.assign(Comment=df['Comment'].str.split('.'))\n",
        "   .explode('Comment')\n",
        "   .reset_index(drop=True)\n",
        ")\n",
        "df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name</th>\n",
              "      <th>Time</th>\n",
              "      <th>Store</th>\n",
              "      <th>Comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Lisa Shen</td>\n",
              "      <td>08-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>every introvert be dream</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>WTFIWFYDB</td>\n",
              "      <td>09-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>not really , there be lot of people inside reg...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>WTFIWFYDB</td>\n",
              "      <td>09-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>laugh out loud exactly</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>what the</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>253129</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>okay thank you</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>253130</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>the actual reason be that want to be able to s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>253131</th>\n",
              "      <td>Hookedmedia.1</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>why</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>253132</th>\n",
              "      <td>Kelvin_kufley</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>why cause you can ' steal ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>253133</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>yes that ' literally exactly the reason</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>253134 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                 Name        Time       Store  \\\n",
              "0           Lisa Shen  08-03-2020   Amazon Go   \n",
              "1           WTFIWFYDB  09-03-2020   Amazon Go   \n",
              "2           WTFIWFYDB  09-03-2020   Amazon Go   \n",
              "3         John Miller  10-03-2020   Amazon Go   \n",
              "4         John Miller  10-03-2020   Amazon Go   \n",
              "...               ...         ...         ...   \n",
              "253129  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "253130  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "253131  Hookedmedia.1  02/11/2022  Smart Cart   \n",
              "253132  Kelvin_kufley  02/11/2022  Smart Cart   \n",
              "253133  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "\n",
              "                                                  Comment  \n",
              "0                                every introvert be dream  \n",
              "1       not really , there be lot of people inside reg...  \n",
              "2                                                          \n",
              "3                                  laugh out loud exactly  \n",
              "4                                               what the   \n",
              "...                                                   ...  \n",
              "253129                                     okay thank you  \n",
              "253130  the actual reason be that want to be able to s...  \n",
              "253131                                                why  \n",
              "253132                        why cause you can ' steal ?  \n",
              "253133            yes that ' literally exactly the reason  \n",
              "\n",
              "[253134 rows x 4 columns]"
            ]
          },
          "execution_count": 37,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df = (\n",
        " df.assign(Comment=df['Comment'].str.split('except'))\n",
        "   .explode('Comment')\n",
        "   .reset_index(drop=True)\n",
        ")\n",
        "df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name</th>\n",
              "      <th>Time</th>\n",
              "      <th>Store</th>\n",
              "      <th>Comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Lisa Shen</td>\n",
              "      <td>08-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>every introvert be dream</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>WTFIWFYDB</td>\n",
              "      <td>09-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>not really , there be lot of people inside reg...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>WTFIWFYDB</td>\n",
              "      <td>09-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>laugh out loud exactly</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>what the</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>268231</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>okay thank you</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>268232</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>the actual reason be that want to be able to s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>268233</th>\n",
              "      <td>Hookedmedia.1</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>why</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>268234</th>\n",
              "      <td>Kelvin_kufley</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>why cause you can ' steal ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>268235</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>yes that ' literally exactly the reason</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>268236 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                 Name        Time       Store  \\\n",
              "0           Lisa Shen  08-03-2020   Amazon Go   \n",
              "1           WTFIWFYDB  09-03-2020   Amazon Go   \n",
              "2           WTFIWFYDB  09-03-2020   Amazon Go   \n",
              "3         John Miller  10-03-2020   Amazon Go   \n",
              "4         John Miller  10-03-2020   Amazon Go   \n",
              "...               ...         ...         ...   \n",
              "268231  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "268232  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "268233  Hookedmedia.1  02/11/2022  Smart Cart   \n",
              "268234  Kelvin_kufley  02/11/2022  Smart Cart   \n",
              "268235  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "\n",
              "                                                  Comment  \n",
              "0                                every introvert be dream  \n",
              "1       not really , there be lot of people inside reg...  \n",
              "2                                                          \n",
              "3                                  laugh out loud exactly  \n",
              "4                                               what the   \n",
              "...                                                   ...  \n",
              "268231                                     okay thank you  \n",
              "268232  the actual reason be that want to be able to s...  \n",
              "268233                                                why  \n",
              "268234                        why cause you can ' steal ?  \n",
              "268235            yes that ' literally exactly the reason  \n",
              "\n",
              "[268236 rows x 4 columns]"
            ]
          },
          "execution_count": 38,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df = (\n",
        " df.assign(Comment=df['Comment'].str.split('but'))\n",
        "   .explode('Comment')\n",
        "   .reset_index(drop=True)\n",
        ")\n",
        "df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lJZIOQkGphsr"
      },
      "source": [
        "BỎ SỐ, BỎ DẤU CÂU"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rfoJAksZphsr",
        "outputId": "e8400740-aca6-471b-c522-528826b1cb33"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\HienPhuong\\AppData\\Local\\Temp\\ipykernel_16180\\761449715.py:1: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  df['Comment'] = df['Comment'].str.replace('\\d+', '')\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "0                                  every introvert be dream\n",
              "1         not really , there be lot of people inside reg...\n",
              "2                                                          \n",
              "3                                    laugh out loud exactly\n",
              "4                                                 what the \n",
              "                                ...                        \n",
              "268231                                       okay thank you\n",
              "268232    the actual reason be that want to be able to s...\n",
              "268233                                                  why\n",
              "268234                          why cause you can ' steal ?\n",
              "268235              yes that ' literally exactly the reason\n",
              "Name: Comment, Length: 268236, dtype: object"
            ]
          },
          "execution_count": 39,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df['Comment'] = df['Comment'].str.replace('\\d+', '')\n",
        "df.Comment"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "id": "8rHzEaILphsr"
      },
      "outputs": [],
      "source": [
        "PUNCT_TO_REMOVE = string.punctuation\n",
        "def remove_punctuation(text):\n",
        "    \"\"\"custom function to remove the punctuation\"\"\"\n",
        "    return text.translate(str.maketrans('', '', PUNCT_TO_REMOVE))\n",
        "\n",
        "df[\"Comment\"] = df[\"Comment\"].apply(lambda text: remove_punctuation(text))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KSkAlIMC0oU8",
        "outputId": "28861f0c-36da-4a74-f537-16fce40720fb"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0                                  every introvert be dream\n",
              "1         not really  there be lot of people inside rega...\n",
              "2                                                          \n",
              "3                                    laugh out loud exactly\n",
              "4                                                 what the \n",
              "                                ...                        \n",
              "268231                                       okay thank you\n",
              "268232    the actual reason be that want to be able to s...\n",
              "268233                                                  why\n",
              "268234                            why cause you can  steal \n",
              "268235               yes that  literally exactly the reason\n",
              "Name: Comment, Length: 268236, dtype: object"
            ]
          },
          "execution_count": 41,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df.Comment"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mjiYVQGlphsr"
      },
      "source": [
        "XÓA STOPWORDS"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\", \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn', \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn', \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", 'won', \"won't\", 'wouldn', \"wouldn't\"]\n"
          ]
        }
      ],
      "source": [
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "print(stopwords.words('english'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\", \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'only', 'own', 'same', 'so', 'than', 's', 't', 'can', 'will', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', \"aren't\", \"couldn't\", \"didn't\", \"doesn't\", \"hadn't\", \"hasn't\", \"haven't\", \"isn't\", 'ma', \"mightn't\", \"mustn't\", \"needn't\", \"shan't\", \"shouldn't\", \"wasn't\", \"weren't\", \"won't\", \"wouldn't\"]\n"
          ]
        }
      ],
      "source": [
        "# stop_words = stopwords.words('english').remove(\"not\", \"no\", \"never\", \"nor\", \"but\",\"too\", \"very\", \"can\", \"but\", \"very\", \"just\", \"don\", \"doesn\", \"didn\", \"wasn\", \"weren\", \"isn\", \"aren\", \"haven\", \"hasn\", \"hadn\", \"won\", \"wouldn\", \"shouldn\", \"couldn\", \"mustn\", \"mightn\", \"needn\", \"shan\",  \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", 'won', \"won't\", 'wouldn', \"wouldn't\")\n",
        "# stop_words\n",
        "all_stopwords = stopwords.words('english')\n",
        "all_stopwords.remove('not')\n",
        "all_stopwords.remove('nor')\n",
        "all_stopwords.remove('no')\n",
        "all_stopwords.remove('but')\n",
        "all_stopwords.remove('too')\n",
        "all_stopwords.remove('very')\n",
        "all_stopwords.remove('just')\n",
        "all_stopwords.remove('don')\n",
        "all_stopwords.remove('doesn')\n",
        "all_stopwords.remove('didn')\n",
        "all_stopwords.remove('wasn')\n",
        "all_stopwords.remove('weren')\n",
        "all_stopwords.remove('isn')\n",
        "all_stopwords.remove('aren')\n",
        "all_stopwords.remove('haven')\n",
        "all_stopwords.remove('hasn')\n",
        "all_stopwords.remove('hadn')\n",
        "all_stopwords.remove('won')\n",
        "all_stopwords.remove('wouldn')\n",
        "all_stopwords.remove('shouldn')\n",
        "all_stopwords.remove('couldn')\n",
        "all_stopwords.remove('mustn')\n",
        "all_stopwords.remove('mightn')\n",
        "all_stopwords.remove('needn')\n",
        "all_stopwords.remove('shan')\n",
        "print(all_stopwords)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'dont like bad not please'"
            ]
          },
          "execution_count": 44,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# remove stopwords in all_stopwords\n",
        "def remove_stopwords(text):\n",
        "    return \" \".join([word for word in str(text).split() if word not in all_stopwords])\n",
        "text= \"i dont like he because he is bad not do it please don't\"\n",
        "remove_stopwords(text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {},
      "outputs": [],
      "source": [
        "df[\"Comment\"] = df[\"Comment\"].apply(lambda text: remove_stopwords(text))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YOpXhuH1phss"
      },
      "source": [
        "Remove ký tự lạ, ko xóa white space"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FyHGExVgphst",
        "outputId": "562e03bf-2ecd-427f-e319-40763d68cba2"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\HienPhuong\\AppData\\Local\\Temp\\ipykernel_16180\\3429141368.py:1: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  df['Comment']=df['Comment'].str.replace('[^\\w\\s]', '')\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "0                                     every introvert dream\n",
              "1         not really lot people inside regardless online...\n",
              "2                                                          \n",
              "3                                        laugh loud exactly\n",
              "4                                                          \n",
              "                                ...                        \n",
              "268231                                           okay thank\n",
              "268232    actual reason want able steal peace kelvin kufley\n",
              "268233                                                     \n",
              "268234                                          cause steal\n",
              "268235                         yes literally exactly reason\n",
              "Name: Comment, Length: 268236, dtype: object"
            ]
          },
          "execution_count": 46,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df['Comment']=df['Comment'].str.replace('[^\\w\\s]', '')\n",
        "#show comment\n",
        "df.Comment"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 47,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0                                     every introvert dream\n",
              "1         not really lot people inside regardless online...\n",
              "2                                                          \n",
              "3                                        laugh loud exactly\n",
              "4                                                          \n",
              "                                ...                        \n",
              "268231                                           okay thank\n",
              "268232    actual reason want able steal peace kelvin kufley\n",
              "268233                                                     \n",
              "268234                                          cause steal\n",
              "268235                         yes literally exactly reason\n",
              "Name: Comment, Length: 268236, dtype: object"
            ]
          },
          "execution_count": 47,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#remove character 1 word nonsens \n",
        "df['Comment'] = df['Comment'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>1]))\n",
        "#show change after remove character 1 word nonsens\n",
        "df.Comment"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(268236, 4)"
            ]
          },
          "execution_count": 48,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# count row in df \n",
        "df.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Name       31\n",
              "Time        1\n",
              "Store      22\n",
              "Comment     0\n",
              "dtype: int64"
            ]
          },
          "execution_count": 49,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#check and remove null value\n",
        "df.isnull().sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Name       0\n",
              "Time       0\n",
              "Store      0\n",
              "Comment    0\n",
              "dtype: int64"
            ]
          },
          "execution_count": 50,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#remove null value\n",
        "df.dropna(inplace=True)\n",
        "df.isnull().sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Name           0\n",
              "Time           0\n",
              "Store          0\n",
              "Comment    44920\n",
              "dtype: int64"
            ]
          },
          "execution_count": 52,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#set NaN value\n",
        "df['Comment'] = df['Comment'].replace('', np.nan)\n",
        "df.isnull().sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 53,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name</th>\n",
              "      <th>Time</th>\n",
              "      <th>Store</th>\n",
              "      <th>Comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Lisa Shen</td>\n",
              "      <td>08-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>every introvert dream</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>WTFIWFYDB</td>\n",
              "      <td>09-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>not really lot people inside regardless online...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>WTFIWFYDB</td>\n",
              "      <td>09-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>laugh loud exactly</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>268231</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>okay thank</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>268232</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>actual reason want able steal peace kelvin kufley</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>268233</th>\n",
              "      <td>Hookedmedia.1</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>268234</th>\n",
              "      <td>Kelvin_kufley</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>cause steal</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>268235</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>yes literally exactly reason</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>268187 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                 Name        Time       Store  \\\n",
              "0           Lisa Shen  08-03-2020   Amazon Go   \n",
              "1           WTFIWFYDB  09-03-2020   Amazon Go   \n",
              "2           WTFIWFYDB  09-03-2020   Amazon Go   \n",
              "3         John Miller  10-03-2020   Amazon Go   \n",
              "4         John Miller  10-03-2020   Amazon Go   \n",
              "...               ...         ...         ...   \n",
              "268231  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "268232  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "268233  Hookedmedia.1  02/11/2022  Smart Cart   \n",
              "268234  Kelvin_kufley  02/11/2022  Smart Cart   \n",
              "268235  Hannah Thomas  02/11/2022  Smart Cart   \n",
              "\n",
              "                                                  Comment  \n",
              "0                                   every introvert dream  \n",
              "1       not really lot people inside regardless online...  \n",
              "2                                                     NaN  \n",
              "3                                      laugh loud exactly  \n",
              "4                                                     NaN  \n",
              "...                                                   ...  \n",
              "268231                                         okay thank  \n",
              "268232  actual reason want able steal peace kelvin kufley  \n",
              "268233                                                NaN  \n",
              "268234                                        cause steal  \n",
              "268235                       yes literally exactly reason  \n",
              "\n",
              "[268187 rows x 4 columns]"
            ]
          },
          "execution_count": 53,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 54,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Name       0\n",
              "Time       0\n",
              "Store      0\n",
              "Comment    0\n",
              "dtype: int64"
            ]
          },
          "execution_count": 54,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#remove NaN value \n",
        "df.dropna(inplace=True)\n",
        "df.isnull().sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 72,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Name</th>\n",
              "      <th>Time</th>\n",
              "      <th>Store</th>\n",
              "      <th>Comment</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Lisa Shen</td>\n",
              "      <td>08-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>every introvert dream</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>WTFIWFYDB</td>\n",
              "      <td>09-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>not really lot people inside regardless online...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>laugh loud exactly</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>John Miller</td>\n",
              "      <td>10-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>talk</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Soundwave</td>\n",
              "      <td>11-03-2020</td>\n",
              "      <td>Amazon Go</td>\n",
              "      <td>win change anything introvert already self che...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>268230</th>\n",
              "      <td>Lindseypatterson123</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>ok smart cart cancel just girl</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>268231</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>okay thank</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>268232</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>actual reason want able steal peace kelvin kufley</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>268234</th>\n",
              "      <td>Kelvin_kufley</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>cause steal</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>268235</th>\n",
              "      <td>Hannah Thomas</td>\n",
              "      <td>02/11/2022</td>\n",
              "      <td>Smart Cart</td>\n",
              "      <td>yes literally exactly reason</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>223267 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                       Name        Time       Store  \\\n",
              "0                 Lisa Shen  08-03-2020   Amazon Go   \n",
              "1                 WTFIWFYDB  09-03-2020   Amazon Go   \n",
              "3               John Miller  10-03-2020   Amazon Go   \n",
              "6               John Miller  10-03-2020   Amazon Go   \n",
              "7                 Soundwave  11-03-2020   Amazon Go   \n",
              "...                     ...         ...         ...   \n",
              "268230  Lindseypatterson123  02/11/2022  Smart Cart   \n",
              "268231        Hannah Thomas  02/11/2022  Smart Cart   \n",
              "268232        Hannah Thomas  02/11/2022  Smart Cart   \n",
              "268234        Kelvin_kufley  02/11/2022  Smart Cart   \n",
              "268235        Hannah Thomas  02/11/2022  Smart Cart   \n",
              "\n",
              "                                                  Comment  \n",
              "0                                   every introvert dream  \n",
              "1       not really lot people inside regardless online...  \n",
              "3                                      laugh loud exactly  \n",
              "6                                                    talk  \n",
              "7       win change anything introvert already self che...  \n",
              "...                                                   ...  \n",
              "268230                     ok smart cart cancel just girl  \n",
              "268231                                         okay thank  \n",
              "268232  actual reason want able steal peace kelvin kufley  \n",
              "268234                                        cause steal  \n",
              "268235                       yes literally exactly reason  \n",
              "\n",
              "[223267 rows x 4 columns]"
            ]
          },
          "execution_count": 72,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 73,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(223267, 4)"
            ]
          },
          "execution_count": 73,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#show row in df \n",
        "df.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 75,
      "metadata": {},
      "outputs": [],
      "source": [
        "df = df.reset_index()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 87,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'true'"
            ]
          },
          "execution_count": 87,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#to lower case \n",
        "df['Comment'] = df['Comment'].str.lower()\n",
        "df.Comment[19]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 85,
      "metadata": {},
      "outputs": [],
      "source": [
        "#save dataframe to csv\n",
        "df.to_csv('data_clean.csv', index=False)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.9"
    },
    "vscode": {
      "interpreter": {
        "hash": "7008a76ded2a80656b717ec2236e0e47fc7bffffd728332632b96ba5b4dde424"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}