Feature_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Font:\n",
    "\n",
    "https://github.com/dipanjanS/practical-machine-learning-with-python/blob/master/bonus%20content/nlp%20proven%20approach/NLP%20Strategy%20I%20-%20Processing%20and%20Understanding%20Text.ipynb\n",
    "\n",
    "https://www.analyticsvidhya.com/blog/2018/02/the-different-methods-deal-text-data-predictive-python/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "seed_urls = ['https://inshorts.com/en/read/technology',\n",
    "             'https://inshorts.com/en/read/sports',\n",
    "             'https://inshorts.com/en/read/world']\n",
    "\n",
    "def build_dataset(seed_urls):\n",
    "    news_data = []\n",
    "    for url in seed_urls:\n",
    "        news_category = url.split('/')[-1]\n",
    "        data = requests.get(url)\n",
    "        soup = BeautifulSoup(data.content, 'html.parser')\n",
    "        \n",
    "        news_articles = [{'news_headline': headline.find('span', \n",
    "                                                         attrs={\"itemprop\": \"headline\"}).string,\n",
    "                          'news_article': article.find('div', \n",
    "                                                       attrs={\"itemprop\": \"articleBody\"}).string,\n",
    "                          'news_category': news_category}\n",
    "                         \n",
    "                            for headline, article in \n",
    "                             zip(soup.find_all('div', \n",
    "                                               class_=[\"news-card-title news-right-box\"]),\n",
    "                                 soup.find_all('div', \n",
    "                                               class_=[\"news-card-content news-right-box\"]))\n",
    "                        ]\n",
    "        news_data.extend(news_articles)\n",
    "        \n",
    "    df =  pd.DataFrame(news_data)\n",
    "    df = df[['news_headline', 'news_article', 'news_category']]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_headline</th>\n",
       "      <th>news_article</th>\n",
       "      <th>news_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Google takes blame for Aadhaar helpline number...</td>\n",
       "      <td>After Aadhaar governing body UIDAI denied aski...</td>\n",
       "      <td>technology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Defence Minister gives 11 open challenges to s...</td>\n",
       "      <td>Defence Minister Nirmala Sitharaman launched t...</td>\n",
       "      <td>technology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Microsoft once invested $150 million in near-b...</td>\n",
       "      <td>In 1997, ousted CEO Steve Jobs returned to App...</td>\n",
       "      <td>technology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Helpline number in a mobile's contact list can...</td>\n",
       "      <td>Aadhaar's governing body UIDAI on Sunday said,...</td>\n",
       "      <td>technology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Google Maps location sharing to show battery p...</td>\n",
       "      <td>Tech giant Google has started showing users' b...</td>\n",
       "      <td>technology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Cybercriminals use fake Income Tax refund SMS ...</td>\n",
       "      <td>A cybercrime racket is under investigation for...</td>\n",
       "      <td>technology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Government panel proposes localisation of clou...</td>\n",
       "      <td>A panel working on Indian government's cloud c...</td>\n",
       "      <td>technology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Facebook begins testing its dating feature</td>\n",
       "      <td>Two months after announcing its dating feature...</td>\n",
       "      <td>technology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Apple's value equals GDP of 16th largest econo...</td>\n",
       "      <td>US consumer technology giant Apple's $1 trilli...</td>\n",
       "      <td>technology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Google limits employees’ access to China searc...</td>\n",
       "      <td>Tech giant Google has limited its employees' a...</td>\n",
       "      <td>technology</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       news_headline  \\\n",
       "0  Google takes blame for Aadhaar helpline number...   \n",
       "1  Defence Minister gives 11 open challenges to s...   \n",
       "2  Microsoft once invested $150 million in near-b...   \n",
       "3  Helpline number in a mobile's contact list can...   \n",
       "4  Google Maps location sharing to show battery p...   \n",
       "5  Cybercriminals use fake Income Tax refund SMS ...   \n",
       "6  Government panel proposes localisation of clou...   \n",
       "7         Facebook begins testing its dating feature   \n",
       "8  Apple's value equals GDP of 16th largest econo...   \n",
       "9  Google limits employees’ access to China searc...   \n",
       "\n",
       "                                        news_article news_category  \n",
       "0  After Aadhaar governing body UIDAI denied aski...    technology  \n",
       "1  Defence Minister Nirmala Sitharaman launched t...    technology  \n",
       "2  In 1997, ousted CEO Steve Jobs returned to App...    technology  \n",
       "3  Aadhaar's governing body UIDAI on Sunday said,...    technology  \n",
       "4  Tech giant Google has started showing users' b...    technology  \n",
       "5  A cybercrime racket is under investigation for...    technology  \n",
       "6  A panel working on Indian government's cloud c...    technology  \n",
       "7  Two months after announcing its dating feature...    technology  \n",
       "8  US consumer technology giant Apple's $1 trilli...    technology  \n",
       "9  Tech giant Google has limited its employees' a...    technology  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news_df = build_dataset(seed_urls)\n",
    "news_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "world         25\n",
       "sports        25\n",
       "technology    23\n",
       "Name: news_category, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news_df.news_category.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Basic feature extraction using text data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Number of words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word_count(label, column):\n",
    "    news_df[label] = news_df[column].apply(lambda x: len(str(x).split(\" \")))\n",
    "    return news_df[[column, label]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_headline</th>\n",
       "      <th>news_headline_word_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Google takes blame for Aadhaar helpline number...</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Defence Minister gives 11 open challenges to s...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Microsoft once invested $150 million in near-b...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Helpline number in a mobile's contact list can...</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Google Maps location sharing to show battery p...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       news_headline  news_headline_word_count\n",
       "0  Google takes blame for Aadhaar helpline number...                        10\n",
       "1  Defence Minister gives 11 open challenges to s...                         8\n",
       "2  Microsoft once invested $150 million in near-b...                         8\n",
       "3  Helpline number in a mobile's contact list can...                        11\n",
       "4  Google Maps location sharing to show battery p...                         8"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_count('news_headline_word_count', 'news_headline').head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_article</th>\n",
       "      <th>news_article_word_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>After Aadhaar governing body UIDAI denied aski...</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Defence Minister Nirmala Sitharaman launched t...</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>In 1997, ousted CEO Steve Jobs returned to App...</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Aadhaar's governing body UIDAI on Sunday said,...</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Tech giant Google has started showing users' b...</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        news_article  news_article_word_count\n",
       "0  After Aadhaar governing body UIDAI denied aski...                       60\n",
       "1  Defence Minister Nirmala Sitharaman launched t...                       54\n",
       "2  In 1997, ousted CEO Steve Jobs returned to App...                       60\n",
       "3  Aadhaar's governing body UIDAI on Sunday said,...                       60\n",
       "4  Tech giant Google has started showing users' b...                       59"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_count('news_article_word_count', 'news_article').head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Number of characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def char_count(label, column):\n",
    "    news_df[label] = news_df[column].str.len()\n",
    "    return news_df[[column, label]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_article</th>\n",
       "      <th>news_article_char_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>After Aadhaar governing body UIDAI denied aski...</td>\n",
       "      <td>397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Defence Minister Nirmala Sitharaman launched t...</td>\n",
       "      <td>397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>In 1997, ousted CEO Steve Jobs returned to App...</td>\n",
       "      <td>359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Aadhaar's governing body UIDAI on Sunday said,...</td>\n",
       "      <td>357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Tech giant Google has started showing users' b...</td>\n",
       "      <td>350</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        news_article  news_article_char_count\n",
       "0  After Aadhaar governing body UIDAI denied aski...                      397\n",
       "1  Defence Minister Nirmala Sitharaman launched t...                      397\n",
       "2  In 1997, ousted CEO Steve Jobs returned to App...                      359\n",
       "3  Aadhaar's governing body UIDAI on Sunday said,...                      357\n",
       "4  Tech giant Google has started showing users' b...                      350"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_count('news_article_char_count', 'news_article').head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_headline</th>\n",
       "      <th>news_headline_char_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Google takes blame for Aadhaar helpline number...</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Defence Minister gives 11 open challenges to s...</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Microsoft once invested $150 million in near-b...</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Helpline number in a mobile's contact list can...</td>\n",
       "      <td>66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Google Maps location sharing to show battery p...</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       news_headline  news_headline_char_count\n",
       "0  Google takes blame for Aadhaar helpline number...                        64\n",
       "1  Defence Minister gives 11 open challenges to s...                        53\n",
       "2  Microsoft once invested $150 million in near-b...                        59\n",
       "3  Helpline number in a mobile's contact list can...                        66\n",
       "4  Google Maps location sharing to show battery p...                        55"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_count('news_headline_char_count', 'news_headline').head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Average word length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def avg_word(sentence):\n",
    "    words = sentence.split()\n",
    "    return (sum(len(word) for word in words )/len(words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>news_article</th>\n",
       "      <th>avg_word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>After Aadhaar governing body UIDAI denied aski...</td>\n",
       "      <td>5.633333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Defence Minister Nirmala Sitharaman launched t...</td>\n",
       "      <td>6.370370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>In 1997, ousted CEO Steve Jobs returned to App...</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Aadhaar's governing body UIDAI on Sunday said,...</td>\n",
       "      <td>4.966667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Tech giant Google has started showing users' b...</td>\n",
       "      <td>5.034483</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        news_article  avg_word\n",
       "0  After Aadhaar governing body UIDAI denied aski...  5.633333\n",
       "1  Defence Minister Nirmala Sitharaman launched t...  6.370370\n",
       "2  In 1997, ousted CEO Steve Jobs returned to App...  5.000000\n",
       "3  Aadhaar's governing body UIDAI on Sunday said,...  4.966667\n",
       "4  Tech giant Google has started showing users' b...  5.034483"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news_df['avg_word'] = news_df['news_article'].apply(lambda x: avg_word(x))\n",
    "news_df[['news_article', 'avg_word']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}