tweets-cleaning-with-python.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "898fb323",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2022-10-12T17:54:35.201657Z",
     "iopub.status.busy": "2022-10-12T17:54:35.201104Z",
     "iopub.status.idle": "2022-10-12T17:54:35.220324Z",
     "shell.execute_reply": "2022-10-12T17:54:35.218800Z"
    },
    "papermill": {
     "duration": 0.033391,
     "end_time": "2022-10-12T17:54:35.223572",
     "exception": false,
     "start_time": "2022-10-12T17:54:35.190181",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/sentiment-dataset-with-1-million-tweets/dataset.csv\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1203bf86",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:54:35.241944Z",
     "iopub.status.busy": "2022-10-12T17:54:35.241520Z",
     "iopub.status.idle": "2022-10-12T17:54:40.923667Z",
     "shell.execute_reply": "2022-10-12T17:54:40.922497Z"
    },
    "papermill": {
     "duration": 5.694467,
     "end_time": "2022-10-12T17:54:40.926299",
     "exception": false,
     "start_time": "2022-10-12T17:54:35.231832",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"/kaggle/input/sentiment-dataset-with-1-million-tweets/dataset.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9aeeb29d",
   "metadata": {
    "papermill": {
     "duration": 0.008599,
     "end_time": "2022-10-12T17:54:40.943086",
     "exception": false,
     "start_time": "2022-10-12T17:54:40.934487",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Tweets Cleaning with Python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c18b6654",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:54:40.963101Z",
     "iopub.status.busy": "2022-10-12T17:54:40.962378Z",
     "iopub.status.idle": "2022-10-12T17:54:40.989603Z",
     "shell.execute_reply": "2022-10-12T17:54:40.988611Z"
    },
    "papermill": {
     "duration": 0.039833,
     "end_time": "2022-10-12T17:54:40.992048",
     "exception": false,
     "start_time": "2022-10-12T17:54:40.952215",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Language</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>@Charlie_Corley @Kristine1G @amyklobuchar @Sty...</td>\n",
       "      <td>en</td>\n",
       "      <td>litigious</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>#BadBunny: Como dos gotas de agua: Joven se di...</td>\n",
       "      <td>es</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://t.co/YJNiO0p1JV Flagstar Bank disclose...</td>\n",
       "      <td>en</td>\n",
       "      <td>litigious</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Rwanda is set to host the headquarters of Unit...</td>\n",
       "      <td>en</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>OOPS. I typed her name incorrectly (today’s br...</td>\n",
       "      <td>en</td>\n",
       "      <td>litigious</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text Language      Label\n",
       "0  @Charlie_Corley @Kristine1G @amyklobuchar @Sty...       en  litigious\n",
       "1  #BadBunny: Como dos gotas de agua: Joven se di...       es   negative\n",
       "2  https://t.co/YJNiO0p1JV Flagstar Bank disclose...       en  litigious\n",
       "3  Rwanda is set to host the headquarters of Unit...       en   positive\n",
       "4  OOPS. I typed her name incorrectly (today’s br...       en  litigious"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2a514e2",
   "metadata": {
    "papermill": {
     "duration": 0.007488,
     "end_time": "2022-10-12T17:54:41.007756",
     "exception": false,
     "start_time": "2022-10-12T17:54:41.000268",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Lowercasing all the letters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a1bbc1f3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:54:41.026465Z",
     "iopub.status.busy": "2022-10-12T17:54:41.026051Z",
     "iopub.status.idle": "2022-10-12T17:54:52.272066Z",
     "shell.execute_reply": "2022-10-12T17:54:52.270835Z"
    },
    "papermill": {
     "duration": 11.25788,
     "end_time": "2022-10-12T17:54:52.275032",
     "exception": false,
     "start_time": "2022-10-12T17:54:41.017152",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df['Clean'] = df.apply(lambda row: row['Text'].lower(),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d0a885dc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:54:52.292842Z",
     "iopub.status.busy": "2022-10-12T17:54:52.292438Z",
     "iopub.status.idle": "2022-10-12T17:54:52.555381Z",
     "shell.execute_reply": "2022-10-12T17:54:52.554297Z"
    },
    "papermill": {
     "duration": 0.274806,
     "end_time": "2022-10-12T17:54:52.557922",
     "exception": false,
     "start_time": "2022-10-12T17:54:52.283116",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>@Charlie_Corley @Kristine1G @amyklobuchar @Sty...</td>\n",
       "      <td>@charlie_corley @kristine1g @amyklobuchar @sty...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>#BadBunny: Como dos gotas de agua: Joven se di...</td>\n",
       "      <td>#badbunny: como dos gotas de agua: joven se di...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://t.co/YJNiO0p1JV Flagstar Bank disclose...</td>\n",
       "      <td>https://t.co/yjnio0p1jv flagstar bank disclose...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Rwanda is set to host the headquarters of Unit...</td>\n",
       "      <td>rwanda is set to host the headquarters of unit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>OOPS. I typed her name incorrectly (today’s br...</td>\n",
       "      <td>oops. i typed her name incorrectly (today’s br...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text  \\\n",
       "0  @Charlie_Corley @Kristine1G @amyklobuchar @Sty...   \n",
       "1  #BadBunny: Como dos gotas de agua: Joven se di...   \n",
       "2  https://t.co/YJNiO0p1JV Flagstar Bank disclose...   \n",
       "3  Rwanda is set to host the headquarters of Unit...   \n",
       "4  OOPS. I typed her name incorrectly (today’s br...   \n",
       "\n",
       "                                               Clean  \n",
       "0  @charlie_corley @kristine1g @amyklobuchar @sty...  \n",
       "1  #badbunny: como dos gotas de agua: joven se di...  \n",
       "2  https://t.co/yjnio0p1jv flagstar bank disclose...  \n",
       "3  rwanda is set to host the headquarters of unit...  \n",
       "4  oops. i typed her name incorrectly (today’s br...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['Text','Clean']].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe2ad4b1",
   "metadata": {
    "papermill": {
     "duration": 0.007936,
     "end_time": "2022-10-12T17:54:52.573867",
     "exception": false,
     "start_time": "2022-10-12T17:54:52.565931",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Removing hashtags and mentions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3db56334",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:54:52.592081Z",
     "iopub.status.busy": "2022-10-12T17:54:52.591682Z",
     "iopub.status.idle": "2022-10-12T17:54:52.596413Z",
     "shell.execute_reply": "2022-10-12T17:54:52.595318Z"
    },
    "papermill": {
     "duration": 0.016406,
     "end_time": "2022-10-12T17:54:52.598670",
     "exception": false,
     "start_time": "2022-10-12T17:54:52.582264",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6f41fb06",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:54:52.616688Z",
     "iopub.status.busy": "2022-10-12T17:54:52.616281Z",
     "iopub.status.idle": "2022-10-12T17:55:05.150218Z",
     "shell.execute_reply": "2022-10-12T17:55:05.148490Z"
    },
    "papermill": {
     "duration": 12.546688,
     "end_time": "2022-10-12T17:55:05.153388",
     "exception": false,
     "start_time": "2022-10-12T17:54:52.606700",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df['Clean'] = df.apply(lambda row: re.sub(\"@[A-Za-z0-9_]+\",\"\", row['Clean']),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b23dcef1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:55:05.172468Z",
     "iopub.status.busy": "2022-10-12T17:55:05.171966Z",
     "iopub.status.idle": "2022-10-12T17:55:05.251903Z",
     "shell.execute_reply": "2022-10-12T17:55:05.248461Z"
    },
    "papermill": {
     "duration": 0.09405,
     "end_time": "2022-10-12T17:55:05.255758",
     "exception": false,
     "start_time": "2022-10-12T17:55:05.161708",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>@Charlie_Corley @Kristine1G @amyklobuchar @Sty...</td>\n",
       "      <td>testimony is not evidence in a court of la...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>#BadBunny: Como dos gotas de agua: Joven se di...</td>\n",
       "      <td>#badbunny: como dos gotas de agua: joven se di...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://t.co/YJNiO0p1JV Flagstar Bank disclose...</td>\n",
       "      <td>https://t.co/yjnio0p1jv flagstar bank disclose...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Rwanda is set to host the headquarters of Unit...</td>\n",
       "      <td>rwanda is set to host the headquarters of unit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>OOPS. I typed her name incorrectly (today’s br...</td>\n",
       "      <td>oops. i typed her name incorrectly (today’s br...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text  \\\n",
       "0  @Charlie_Corley @Kristine1G @amyklobuchar @Sty...   \n",
       "1  #BadBunny: Como dos gotas de agua: Joven se di...   \n",
       "2  https://t.co/YJNiO0p1JV Flagstar Bank disclose...   \n",
       "3  Rwanda is set to host the headquarters of Unit...   \n",
       "4  OOPS. I typed her name incorrectly (today’s br...   \n",
       "\n",
       "                                               Clean  \n",
       "0      testimony is not evidence in a court of la...  \n",
       "1  #badbunny: como dos gotas de agua: joven se di...  \n",
       "2  https://t.co/yjnio0p1jv flagstar bank disclose...  \n",
       "3  rwanda is set to host the headquarters of unit...  \n",
       "4  oops. i typed her name incorrectly (today’s br...  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['Text','Clean']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4a8dbcfb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:55:05.283310Z",
     "iopub.status.busy": "2022-10-12T17:55:05.282216Z",
     "iopub.status.idle": "2022-10-12T17:55:17.569314Z",
     "shell.execute_reply": "2022-10-12T17:55:17.567811Z"
    },
    "papermill": {
     "duration": 12.306354,
     "end_time": "2022-10-12T17:55:17.572078",
     "exception": false,
     "start_time": "2022-10-12T17:55:05.265724",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df['Clean'] = df.apply(lambda row: re.sub(\"#[A-Za-z0-9_]+\",\"\", row['Clean']),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "32f38626",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:55:17.591326Z",
     "iopub.status.busy": "2022-10-12T17:55:17.590903Z",
     "iopub.status.idle": "2022-10-12T17:55:17.650680Z",
     "shell.execute_reply": "2022-10-12T17:55:17.649814Z"
    },
    "papermill": {
     "duration": 0.072199,
     "end_time": "2022-10-12T17:55:17.653001",
     "exception": false,
     "start_time": "2022-10-12T17:55:17.580802",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>@Charlie_Corley @Kristine1G @amyklobuchar @Sty...</td>\n",
       "      <td>testimony is not evidence in a court of la...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>#BadBunny: Como dos gotas de agua: Joven se di...</td>\n",
       "      <td>: como dos gotas de agua: joven se disfraza de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://t.co/YJNiO0p1JV Flagstar Bank disclose...</td>\n",
       "      <td>https://t.co/yjnio0p1jv flagstar bank disclose...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Rwanda is set to host the headquarters of Unit...</td>\n",
       "      <td>rwanda is set to host the headquarters of unit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>OOPS. I typed her name incorrectly (today’s br...</td>\n",
       "      <td>oops. i typed her name incorrectly (today’s br...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text  \\\n",
       "0  @Charlie_Corley @Kristine1G @amyklobuchar @Sty...   \n",
       "1  #BadBunny: Como dos gotas de agua: Joven se di...   \n",
       "2  https://t.co/YJNiO0p1JV Flagstar Bank disclose...   \n",
       "3  Rwanda is set to host the headquarters of Unit...   \n",
       "4  OOPS. I typed her name incorrectly (today’s br...   \n",
       "\n",
       "                                               Clean  \n",
       "0      testimony is not evidence in a court of la...  \n",
       "1  : como dos gotas de agua: joven se disfraza de...  \n",
       "2  https://t.co/yjnio0p1jv flagstar bank disclose...  \n",
       "3  rwanda is set to host the headquarters of unit...  \n",
       "4  oops. i typed her name incorrectly (today’s br...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['Text','Clean']].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b14f19cf",
   "metadata": {
    "papermill": {
     "duration": 0.010538,
     "end_time": "2022-10-12T17:55:17.672116",
     "exception": false,
     "start_time": "2022-10-12T17:55:17.661578",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Removing links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7af51fb1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:55:17.693960Z",
     "iopub.status.busy": "2022-10-12T17:55:17.693256Z",
     "iopub.status.idle": "2022-10-12T17:55:30.985372Z",
     "shell.execute_reply": "2022-10-12T17:55:30.982453Z"
    },
    "papermill": {
     "duration": 13.307213,
     "end_time": "2022-10-12T17:55:30.988578",
     "exception": false,
     "start_time": "2022-10-12T17:55:17.681365",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df['Clean'] = df.apply(lambda row: re.sub(r\"http\\S+\",\"\", row['Clean']),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d704f458",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:55:31.012177Z",
     "iopub.status.busy": "2022-10-12T17:55:31.011718Z",
     "iopub.status.idle": "2022-10-12T17:55:43.284032Z",
     "shell.execute_reply": "2022-10-12T17:55:43.282554Z"
    },
    "papermill": {
     "duration": 12.287665,
     "end_time": "2022-10-12T17:55:43.287172",
     "exception": false,
     "start_time": "2022-10-12T17:55:30.999507",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df['Clean'] = df.apply(lambda row: re.sub(r\"www.\\S+\",\"\", row['Clean']),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5d271f91",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:55:43.307957Z",
     "iopub.status.busy": "2022-10-12T17:55:43.307512Z",
     "iopub.status.idle": "2022-10-12T17:55:43.378790Z",
     "shell.execute_reply": "2022-10-12T17:55:43.377840Z"
    },
    "papermill": {
     "duration": 0.085725,
     "end_time": "2022-10-12T17:55:43.382345",
     "exception": false,
     "start_time": "2022-10-12T17:55:43.296620",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>@Charlie_Corley @Kristine1G @amyklobuchar @Sty...</td>\n",
       "      <td>testimony is not evidence in a court of la...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>#BadBunny: Como dos gotas de agua: Joven se di...</td>\n",
       "      <td>: como dos gotas de agua: joven se disfraza de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://t.co/YJNiO0p1JV Flagstar Bank disclose...</td>\n",
       "      <td>flagstar bank discloses a data breach that im...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Rwanda is set to host the headquarters of Unit...</td>\n",
       "      <td>rwanda is set to host the headquarters of unit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>OOPS. I typed her name incorrectly (today’s br...</td>\n",
       "      <td>oops. i typed her name incorrectly (today’s br...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                Text  \\\n",
       "0  @Charlie_Corley @Kristine1G @amyklobuchar @Sty...   \n",
       "1  #BadBunny: Como dos gotas de agua: Joven se di...   \n",
       "2  https://t.co/YJNiO0p1JV Flagstar Bank disclose...   \n",
       "3  Rwanda is set to host the headquarters of Unit...   \n",
       "4  OOPS. I typed her name incorrectly (today’s br...   \n",
       "\n",
       "                                               Clean  \n",
       "0      testimony is not evidence in a court of la...  \n",
       "1  : como dos gotas de agua: joven se disfraza de...  \n",
       "2   flagstar bank discloses a data breach that im...  \n",
       "3  rwanda is set to host the headquarters of unit...  \n",
       "4  oops. i typed her name incorrectly (today’s br...  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['Text','Clean']].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0024adb",
   "metadata": {
    "papermill": {
     "duration": 0.010024,
     "end_time": "2022-10-12T17:55:43.402770",
     "exception": false,
     "start_time": "2022-10-12T17:55:43.392746",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Removing punctuations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "11fcd82f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:55:43.425408Z",
     "iopub.status.busy": "2022-10-12T17:55:43.424854Z",
     "iopub.status.idle": "2022-10-12T17:55:56.685002Z",
     "shell.execute_reply": "2022-10-12T17:55:56.683383Z"
    },
    "papermill": {
     "duration": 13.276061,
     "end_time": "2022-10-12T17:55:56.688550",
     "exception": false,
     "start_time": "2022-10-12T17:55:43.412489",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df['Clean'] = df.apply(lambda row: re.sub('[()!?]',\" \", row['Clean']),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "22db4b8d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:55:56.709063Z",
     "iopub.status.busy": "2022-10-12T17:55:56.708287Z",
     "iopub.status.idle": "2022-10-12T17:56:08.674500Z",
     "shell.execute_reply": "2022-10-12T17:56:08.673268Z"
    },
    "papermill": {
     "duration": 11.978968,
     "end_time": "2022-10-12T17:56:08.677277",
     "exception": false,
     "start_time": "2022-10-12T17:55:56.698309",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df['Clean'] = df.apply(lambda row: re.sub('\\[.*?\\]',\" \", row['Clean']),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "12255521",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:56:08.697086Z",
     "iopub.status.busy": "2022-10-12T17:56:08.695972Z",
     "iopub.status.idle": "2022-10-12T17:56:08.759023Z",
     "shell.execute_reply": "2022-10-12T17:56:08.758186Z"
    },
    "papermill": {
     "duration": 0.075333,
     "end_time": "2022-10-12T17:56:08.761311",
     "exception": false,
     "start_time": "2022-10-12T17:56:08.685978",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>@Charlie_Corley @Kristine1G @amyklobuchar @Sty...</td>\n",
       "      <td>testimony is not evidence in a court of la...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>#BadBunny: Como dos gotas de agua: Joven se di...</td>\n",
       "      <td>: como dos gotas de agua: joven se disfraza de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://t.co/YJNiO0p1JV Flagstar Bank disclose...</td>\n",
       "      <td>flagstar bank discloses a data breach that im...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Rwanda is set to host the headquarters of Unit...</td>\n",
       "      <td>rwanda is set to host the headquarters of unit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>OOPS. I typed her name incorrectly (today’s br...</td>\n",
       "      <td>oops. i typed her name incorrectly  today’s br...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>It sucks for me since I'm focused on the natur...</td>\n",
       "      <td>it sucks for me since i'm focused on the natur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>@en_font Treballar a l’obra a partir dels 19 a...</td>\n",
       "      <td>treballar a l’obra a partir dels 19 anys fins...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>@ShawnTarloff @itsmieu you can also relate thi...</td>\n",
       "      <td>you can also relate this to art too    a lot...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Social Security. Constant political crises dis...</td>\n",
       "      <td>social security. constant political crises dis...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>@FilmThePoliceLA A broken rib can puncture a l...</td>\n",
       "      <td>a broken rib can puncture a lung or lead to a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>@jacobringenwald @aKeithWatts @CountDankulaTV ...</td>\n",
       "      <td>i agree it's a flawed response for a small ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>@NZ_HK_SU @telebusiness My question was rhetor...</td>\n",
       "      <td>my question was rhetorical. we all know russ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Wimbledon: Nick Kyrgios admits spitting toward...</td>\n",
       "      <td>wimbledon: nick kyrgios admits spitting toward...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>is booktwt a thing? if so thats her and she sp...</td>\n",
       "      <td>is booktwt a thing  if so thats her and she sp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>@SaltBurned [When the first sign of surrender ...</td>\n",
       "      <td>[when the first sign of surrender comes, mich...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Text  \\\n",
       "0   @Charlie_Corley @Kristine1G @amyklobuchar @Sty...   \n",
       "1   #BadBunny: Como dos gotas de agua: Joven se di...   \n",
       "2   https://t.co/YJNiO0p1JV Flagstar Bank disclose...   \n",
       "3   Rwanda is set to host the headquarters of Unit...   \n",
       "4   OOPS. I typed her name incorrectly (today’s br...   \n",
       "5   It sucks for me since I'm focused on the natur...   \n",
       "6   @en_font Treballar a l’obra a partir dels 19 a...   \n",
       "7   @ShawnTarloff @itsmieu you can also relate thi...   \n",
       "8   Social Security. Constant political crises dis...   \n",
       "9   @FilmThePoliceLA A broken rib can puncture a l...   \n",
       "10  @jacobringenwald @aKeithWatts @CountDankulaTV ...   \n",
       "11  @NZ_HK_SU @telebusiness My question was rhetor...   \n",
       "12  Wimbledon: Nick Kyrgios admits spitting toward...   \n",
       "13  is booktwt a thing? if so thats her and she sp...   \n",
       "14  @SaltBurned [When the first sign of surrender ...   \n",
       "\n",
       "                                                Clean  \n",
       "0       testimony is not evidence in a court of la...  \n",
       "1   : como dos gotas de agua: joven se disfraza de...  \n",
       "2    flagstar bank discloses a data breach that im...  \n",
       "3   rwanda is set to host the headquarters of unit...  \n",
       "4   oops. i typed her name incorrectly  today’s br...  \n",
       "5   it sucks for me since i'm focused on the natur...  \n",
       "6    treballar a l’obra a partir dels 19 anys fins...  \n",
       "7     you can also relate this to art too    a lot...  \n",
       "8   social security. constant political crises dis...  \n",
       "9    a broken rib can puncture a lung or lead to a...  \n",
       "10     i agree it's a flawed response for a small ...  \n",
       "11    my question was rhetorical. we all know russ...  \n",
       "12  wimbledon: nick kyrgios admits spitting toward...  \n",
       "13  is booktwt a thing  if so thats her and she sp...  \n",
       "14   [when the first sign of surrender comes, mich...  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['Text','Clean']].head(15)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "558506b6",
   "metadata": {
    "papermill": {
     "duration": 0.008763,
     "end_time": "2022-10-12T17:56:08.779172",
     "exception": false,
     "start_time": "2022-10-12T17:56:08.770409",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Filtering non-alphanumeric characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "badba95f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:56:08.799388Z",
     "iopub.status.busy": "2022-10-12T17:56:08.798462Z",
     "iopub.status.idle": "2022-10-12T17:56:28.361854Z",
     "shell.execute_reply": "2022-10-12T17:56:28.360644Z"
    },
    "papermill": {
     "duration": 19.576372,
     "end_time": "2022-10-12T17:56:28.364444",
     "exception": false,
     "start_time": "2022-10-12T17:56:08.788072",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df['Clean'] = df.apply(lambda row: re.sub(\"[^a-z0-9]\",\" \", row['Clean']),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c764a7ef",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:56:28.384483Z",
     "iopub.status.busy": "2022-10-12T17:56:28.383594Z",
     "iopub.status.idle": "2022-10-12T17:56:28.444466Z",
     "shell.execute_reply": "2022-10-12T17:56:28.443210Z"
    },
    "papermill": {
     "duration": 0.073466,
     "end_time": "2022-10-12T17:56:28.446920",
     "exception": false,
     "start_time": "2022-10-12T17:56:28.373454",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>@Charlie_Corley @Kristine1G @amyklobuchar @Sty...</td>\n",
       "      <td>testimony is not evidence in a court of la...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>#BadBunny: Como dos gotas de agua: Joven se di...</td>\n",
       "      <td>como dos gotas de agua  joven se disfraza de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://t.co/YJNiO0p1JV Flagstar Bank disclose...</td>\n",
       "      <td>flagstar bank discloses a data breach that im...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Rwanda is set to host the headquarters of Unit...</td>\n",
       "      <td>rwanda is set to host the headquarters of unit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>OOPS. I typed her name incorrectly (today’s br...</td>\n",
       "      <td>oops  i typed her name incorrectly  today s br...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>It sucks for me since I'm focused on the natur...</td>\n",
       "      <td>it sucks for me since i m focused on the natur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>@en_font Treballar a l’obra a partir dels 19 a...</td>\n",
       "      <td>treballar a l obra a partir dels 19 anys fins...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>@ShawnTarloff @itsmieu you can also relate thi...</td>\n",
       "      <td>you can also relate this to art too    a lot...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Social Security. Constant political crises dis...</td>\n",
       "      <td>social security  constant political crises dis...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>@FilmThePoliceLA A broken rib can puncture a l...</td>\n",
       "      <td>a broken rib can puncture a lung or lead to a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>@jacobringenwald @aKeithWatts @CountDankulaTV ...</td>\n",
       "      <td>i agree it s a flawed response for a small ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>@NZ_HK_SU @telebusiness My question was rhetor...</td>\n",
       "      <td>my question was rhetorical  we all know russ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Wimbledon: Nick Kyrgios admits spitting toward...</td>\n",
       "      <td>wimbledon  nick kyrgios admits spitting toward...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>is booktwt a thing? if so thats her and she sp...</td>\n",
       "      <td>is booktwt a thing  if so thats her and she sp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>@SaltBurned [When the first sign of surrender ...</td>\n",
       "      <td>when the first sign of surrender comes  mich...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>@co_fuckoff J'espere que ça passera le plus vi...</td>\n",
       "      <td>j espere que  a passera le plus vite possible</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>@roipaee @joe98100959 @Formula1game @redbullra...</td>\n",
       "      <td>i wrote to them now waiting for a respons...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>🤩 Power back on (1855749):  North VI in Ladysm...</td>\n",
       "      <td>power back on  1855749    north vi in ladysm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>At the end we consume fictional stories to enj...</td>\n",
       "      <td>at the end we consume fictional stories to enj...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>#Duranto Express A1 coach AC not working prope...</td>\n",
       "      <td>express a1 coach ac not working properly in  ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Text  \\\n",
       "0   @Charlie_Corley @Kristine1G @amyklobuchar @Sty...   \n",
       "1   #BadBunny: Como dos gotas de agua: Joven se di...   \n",
       "2   https://t.co/YJNiO0p1JV Flagstar Bank disclose...   \n",
       "3   Rwanda is set to host the headquarters of Unit...   \n",
       "4   OOPS. I typed her name incorrectly (today’s br...   \n",
       "5   It sucks for me since I'm focused on the natur...   \n",
       "6   @en_font Treballar a l’obra a partir dels 19 a...   \n",
       "7   @ShawnTarloff @itsmieu you can also relate thi...   \n",
       "8   Social Security. Constant political crises dis...   \n",
       "9   @FilmThePoliceLA A broken rib can puncture a l...   \n",
       "10  @jacobringenwald @aKeithWatts @CountDankulaTV ...   \n",
       "11  @NZ_HK_SU @telebusiness My question was rhetor...   \n",
       "12  Wimbledon: Nick Kyrgios admits spitting toward...   \n",
       "13  is booktwt a thing? if so thats her and she sp...   \n",
       "14  @SaltBurned [When the first sign of surrender ...   \n",
       "15  @co_fuckoff J'espere que ça passera le plus vi...   \n",
       "16  @roipaee @joe98100959 @Formula1game @redbullra...   \n",
       "17  🤩 Power back on (1855749):  North VI in Ladysm...   \n",
       "18  At the end we consume fictional stories to enj...   \n",
       "19  #Duranto Express A1 coach AC not working prope...   \n",
       "\n",
       "                                                Clean  \n",
       "0       testimony is not evidence in a court of la...  \n",
       "1     como dos gotas de agua  joven se disfraza de...  \n",
       "2    flagstar bank discloses a data breach that im...  \n",
       "3   rwanda is set to host the headquarters of unit...  \n",
       "4   oops  i typed her name incorrectly  today s br...  \n",
       "5   it sucks for me since i m focused on the natur...  \n",
       "6    treballar a l obra a partir dels 19 anys fins...  \n",
       "7     you can also relate this to art too    a lot...  \n",
       "8   social security  constant political crises dis...  \n",
       "9    a broken rib can puncture a lung or lead to a...  \n",
       "10     i agree it s a flawed response for a small ...  \n",
       "11    my question was rhetorical  we all know russ...  \n",
       "12  wimbledon  nick kyrgios admits spitting toward...  \n",
       "13  is booktwt a thing  if so thats her and she sp...  \n",
       "14    when the first sign of surrender comes  mich...  \n",
       "15   j espere que  a passera le plus vite possible     \n",
       "16       i wrote to them now waiting for a respons...  \n",
       "17    power back on  1855749    north vi in ladysm...  \n",
       "18  at the end we consume fictional stories to enj...  \n",
       "19   express a1 coach ac not working properly in  ...  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['Text','Clean']].head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3115f435",
   "metadata": {
    "papermill": {
     "duration": 0.009041,
     "end_time": "2022-10-12T17:56:28.465328",
     "exception": false,
     "start_time": "2022-10-12T17:56:28.456287",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Tokenization\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4f258e27",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:56:28.486733Z",
     "iopub.status.busy": "2022-10-12T17:56:28.486320Z",
     "iopub.status.idle": "2022-10-12T17:56:44.884601Z",
     "shell.execute_reply": "2022-10-12T17:56:44.883499Z"
    },
    "papermill": {
     "duration": 16.412289,
     "end_time": "2022-10-12T17:56:44.887599",
     "exception": false,
     "start_time": "2022-10-12T17:56:28.475310",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df['Clean'] = df.apply(lambda row: row['Clean'].split(),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6167dfb1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:56:44.908811Z",
     "iopub.status.busy": "2022-10-12T17:56:44.908332Z",
     "iopub.status.idle": "2022-10-12T17:56:44.987863Z",
     "shell.execute_reply": "2022-10-12T17:56:44.986283Z"
    },
    "papermill": {
     "duration": 0.093399,
     "end_time": "2022-10-12T17:56:44.990990",
     "exception": false,
     "start_time": "2022-10-12T17:56:44.897591",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>@Charlie_Corley @Kristine1G @amyklobuchar @Sty...</td>\n",
       "      <td>[testimony, is, not, evidence, in, a, court, o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>#BadBunny: Como dos gotas de agua: Joven se di...</td>\n",
       "      <td>[como, dos, gotas, de, agua, joven, se, disfra...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://t.co/YJNiO0p1JV Flagstar Bank disclose...</td>\n",
       "      <td>[flagstar, bank, discloses, a, data, breach, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Rwanda is set to host the headquarters of Unit...</td>\n",
       "      <td>[rwanda, is, set, to, host, the, headquarters,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>OOPS. I typed her name incorrectly (today’s br...</td>\n",
       "      <td>[oops, i, typed, her, name, incorrectly, today...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>It sucks for me since I'm focused on the natur...</td>\n",
       "      <td>[it, sucks, for, me, since, i, m, focused, on,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>@en_font Treballar a l’obra a partir dels 19 a...</td>\n",
       "      <td>[treballar, a, l, obra, a, partir, dels, 19, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>@ShawnTarloff @itsmieu you can also relate thi...</td>\n",
       "      <td>[you, can, also, relate, this, to, art, too, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Social Security. Constant political crises dis...</td>\n",
       "      <td>[social, security, constant, political, crises...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>@FilmThePoliceLA A broken rib can puncture a l...</td>\n",
       "      <td>[a, broken, rib, can, puncture, a, lung, or, l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>@jacobringenwald @aKeithWatts @CountDankulaTV ...</td>\n",
       "      <td>[i, agree, it, s, a, flawed, response, for, a,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>@NZ_HK_SU @telebusiness My question was rhetor...</td>\n",
       "      <td>[my, question, was, rhetorical, we, all, know,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Wimbledon: Nick Kyrgios admits spitting toward...</td>\n",
       "      <td>[wimbledon, nick, kyrgios, admits, spitting, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>is booktwt a thing? if so thats her and she sp...</td>\n",
       "      <td>[is, booktwt, a, thing, if, so, thats, her, an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>@SaltBurned [When the first sign of surrender ...</td>\n",
       "      <td>[when, the, first, sign, of, surrender, comes,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>@co_fuckoff J'espere que ça passera le plus vi...</td>\n",
       "      <td>[j, espere, que, a, passera, le, plus, vite, p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>@roipaee @joe98100959 @Formula1game @redbullra...</td>\n",
       "      <td>[i, wrote, to, them, now, waiting, for, a, res...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>🤩 Power back on (1855749):  North VI in Ladysm...</td>\n",
       "      <td>[power, back, on, 1855749, north, vi, in, lady...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>At the end we consume fictional stories to enj...</td>\n",
       "      <td>[at, the, end, we, consume, fictional, stories...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>#Duranto Express A1 coach AC not working prope...</td>\n",
       "      <td>[express, a1, coach, ac, not, working, properl...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 Text  \\\n",
       "0   @Charlie_Corley @Kristine1G @amyklobuchar @Sty...   \n",
       "1   #BadBunny: Como dos gotas de agua: Joven se di...   \n",
       "2   https://t.co/YJNiO0p1JV Flagstar Bank disclose...   \n",
       "3   Rwanda is set to host the headquarters of Unit...   \n",
       "4   OOPS. I typed her name incorrectly (today’s br...   \n",
       "5   It sucks for me since I'm focused on the natur...   \n",
       "6   @en_font Treballar a l’obra a partir dels 19 a...   \n",
       "7   @ShawnTarloff @itsmieu you can also relate thi...   \n",
       "8   Social Security. Constant political crises dis...   \n",
       "9   @FilmThePoliceLA A broken rib can puncture a l...   \n",
       "10  @jacobringenwald @aKeithWatts @CountDankulaTV ...   \n",
       "11  @NZ_HK_SU @telebusiness My question was rhetor...   \n",
       "12  Wimbledon: Nick Kyrgios admits spitting toward...   \n",
       "13  is booktwt a thing? if so thats her and she sp...   \n",
       "14  @SaltBurned [When the first sign of surrender ...   \n",
       "15  @co_fuckoff J'espere que ça passera le plus vi...   \n",
       "16  @roipaee @joe98100959 @Formula1game @redbullra...   \n",
       "17  🤩 Power back on (1855749):  North VI in Ladysm...   \n",
       "18  At the end we consume fictional stories to enj...   \n",
       "19  #Duranto Express A1 coach AC not working prope...   \n",
       "\n",
       "                                                Clean  \n",
       "0   [testimony, is, not, evidence, in, a, court, o...  \n",
       "1   [como, dos, gotas, de, agua, joven, se, disfra...  \n",
       "2   [flagstar, bank, discloses, a, data, breach, t...  \n",
       "3   [rwanda, is, set, to, host, the, headquarters,...  \n",
       "4   [oops, i, typed, her, name, incorrectly, today...  \n",
       "5   [it, sucks, for, me, since, i, m, focused, on,...  \n",
       "6   [treballar, a, l, obra, a, partir, dels, 19, a...  \n",
       "7   [you, can, also, relate, this, to, art, too, a...  \n",
       "8   [social, security, constant, political, crises...  \n",
       "9   [a, broken, rib, can, puncture, a, lung, or, l...  \n",
       "10  [i, agree, it, s, a, flawed, response, for, a,...  \n",
       "11  [my, question, was, rhetorical, we, all, know,...  \n",
       "12  [wimbledon, nick, kyrgios, admits, spitting, t...  \n",
       "13  [is, booktwt, a, thing, if, so, thats, her, an...  \n",
       "14  [when, the, first, sign, of, surrender, comes,...  \n",
       "15  [j, espere, que, a, passera, le, plus, vite, p...  \n",
       "16  [i, wrote, to, them, now, waiting, for, a, res...  \n",
       "17  [power, back, on, 1855749, north, vi, in, lady...  \n",
       "18  [at, the, end, we, consume, fictional, stories...  \n",
       "19  [express, a1, coach, ac, not, working, properl...  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[['Text','Clean']].head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "927823c2",
   "metadata": {
    "papermill": {
     "duration": 0.009584,
     "end_time": "2022-10-12T17:56:45.010591",
     "exception": false,
     "start_time": "2022-10-12T17:56:45.001007",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "fe9322b1",
   "metadata": {
    "papermill": {
     "duration": 0.009671,
     "end_time": "2022-10-12T17:56:45.030965",
     "exception": false,
     "start_time": "2022-10-12T17:56:45.021294",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c3af1333",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:56:45.054455Z",
     "iopub.status.busy": "2022-10-12T17:56:45.053935Z",
     "iopub.status.idle": "2022-10-12T17:56:45.065001Z",
     "shell.execute_reply": "2022-10-12T17:56:45.063760Z"
    },
    "papermill": {
     "duration": 0.025463,
     "end_time": "2022-10-12T17:56:45.067471",
     "exception": false,
     "start_time": "2022-10-12T17:56:45.042008",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def remove_emoji(string):\n",
    "        emoji_pattern = re.compile(\"[\"\n",
    "                           u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
    "                           u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
    "                           u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
    "                           u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
    "                           u\"\\U00002702-\\U000027B0\"\n",
    "                           u\"\\U000024C2-\\U0001F251\"\n",
    "                           \"]+\", flags=re.UNICODE)\n",
    "        return emoji_pattern.sub(r'', string) \n",
    "\n",
    "\n",
    "def clean_tweet(tweet):\n",
    "    if type(tweet) == np.float:\n",
    "        return \"\"\n",
    "    temp = tweet.lower()\n",
    "    temp = re.sub(\"'\", \"\", temp) # to avoid removing contractions in english\n",
    "    temp = re.sub(\"@[A-Za-z0-9_]+\",\"\", temp)\n",
    "    temp = re.sub(\"#\",\"\", temp)\n",
    "    temp = remove_emoji(temp)\n",
    "    temp = re.sub(r'http\\S+', '', temp)\n",
    "    temp = re.sub('[()!?]', ' ', temp)\n",
    "    temp = re.sub('\\[.*?\\]',' ', temp)\n",
    "    temp = re.sub(\"[^a-z0-9]\",\" \", temp)\n",
    "    temp = temp.split()\n",
    "    return temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3528d6eb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:56:45.090006Z",
     "iopub.status.busy": "2022-10-12T17:56:45.089601Z",
     "iopub.status.idle": "2022-10-12T17:56:45.095236Z",
     "shell.execute_reply": "2022-10-12T17:56:45.094076Z"
    },
    "papermill": {
     "duration": 0.019908,
     "end_time": "2022-10-12T17:56:45.098282",
     "exception": false,
     "start_time": "2022-10-12T17:56:45.078374",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "tweet = \"I am Python Programmer🙈🙉🙊 don't ask me to hack facebook account🙏\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "292a6a3d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-10-12T17:56:45.120947Z",
     "iopub.status.busy": "2022-10-12T17:56:45.120547Z",
     "iopub.status.idle": "2022-10-12T17:56:45.132445Z",
     "shell.execute_reply": "2022-10-12T17:56:45.131249Z"
    },
    "papermill": {
     "duration": 0.027014,
     "end_time": "2022-10-12T17:56:45.135758",
     "exception": false,
     "start_time": "2022-10-12T17:56:45.108744",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:14: DeprecationWarning: `np.float` is a deprecated alias for the builtin `float`. To silence this warning, use `float` by itself. Doing this will not modify any behavior and is safe. If you specifically wanted the numpy scalar type, use `np.float64` here.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['i',\n",
       " 'am',\n",
       " 'python',\n",
       " 'programmer',\n",
       " 'dont',\n",
       " 'ask',\n",
       " 'me',\n",
       " 'to',\n",
       " 'hack',\n",
       " 'facebook',\n",
       " 'account']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_tweet(tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bea2161",
   "metadata": {
    "papermill": {
     "duration": 0.009698,
     "end_time": "2022-10-12T17:56:45.155607",
     "exception": false,
     "start_time": "2022-10-12T17:56:45.145909",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 144.373714,
   "end_time": "2022-10-12T17:56:47.786837",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-10-12T17:54:23.413123",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}