Word2Vec_Scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TRAINING WORD2VEC FROM SCRATCH"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing Jupyter notebook from utils.ipynb\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import numpy as np\n",
    "import import_ipynb\n",
    "import pandas as pd\n",
    "from utils import get_batches\n",
    "from nltk.tokenize import TweetTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('npr.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Article</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>In the Washington of 2016, even when the polic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Donald Trump has used Twitter  —   his prefe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Donald Trump is unabashedly praising Russian...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Updated at 2:50 p. m. ET, Russian President Vl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>From photography, illustration and video, to d...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             Article\n",
       "0  In the Washington of 2016, even when the polic...\n",
       "1    Donald Trump has used Twitter  —   his prefe...\n",
       "2    Donald Trump is unabashedly praising Russian...\n",
       "3  Updated at 2:50 p. m. ET, Russian President Vl...\n",
       "4  From photography, illustration and video, to d..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_cleaning(text):\n",
    "    \n",
    "    ### Lowercasing ###\n",
    "    text = text.lower()\n",
    "    \n",
    "    ### Using correct apostrophe ###\n",
    "    apostrophe = re.compile(r\"’\")\n",
    "    text = apostrophe.sub(r\"'\", text)\n",
    "    \n",
    "    ### Removing hashtags (#brexit etc) \n",
    "    hashtag_words = re.compile(r\"#[A-Za-z0-9]+\")\n",
    "    text = hashtag_words.sub(r\"\", text)\n",
    "    \n",
    "    ### Converting punctuation into fullstop ###\n",
    "    text =  re.sub(r'[,!?;-]', '.', text)\n",
    "    text = re.sub(r\"\\.+\", '.', text)\n",
    "    \n",
    "    text = re.sub(r\"[^a-z. ]+\", \"\", text)\n",
    "    \n",
    "    text = TweetTokenizer().tokenize(text)\n",
    "    \n",
    "    return text\n",
    "\n",
    "df['Article'] = df['Article'].map(lambda x: data_cleaning(x)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Article</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[in, the, washington, of, ., even, when, the, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[donald, trump, has, used, twitter, his, prefe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[donald, trump, is, unabashedly, praising, rus...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[updated, at, p, ., m, ., et, ., russian, pres...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[from, photography, ., illustration, and, vide...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             Article\n",
       "0  [in, the, washington, of, ., even, when, the, ...\n",
       "1  [donald, trump, has, used, twitter, his, prefe...\n",
       "2  [donald, trump, is, unabashedly, praising, rus...\n",
       "3  [updated, at, p, ., m, ., et, ., russian, pres...\n",
       "4  [from, photography, ., illustration, and, vide..."
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Corpus Creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_corpus(df):\n",
    "    \n",
    "    corpus = []\n",
    "    \n",
    "    for x in df['Article']:\n",
    "        for i in x:\n",
    "            corpus.append(i)\n",
    "             \n",
    "    return corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = create_corpus(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10173847"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = sorted(set(corpus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "99915"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mapping words to indices and indices to words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dict(list_of_tokens):\n",
    "    \n",
    "    words = sorted(list(set(list_of_tokens)))\n",
    "    n = len(words)\n",
    "    idx = 0\n",
    "    \n",
    "    word2Ind = {}\n",
    "    Ind2word = {}\n",
    "    for k in words:\n",
    "        word2Ind[k] = idx\n",
    "        Ind2word[idx] = k\n",
    "        idx += 1\n",
    "    \n",
    "    return word2Ind, Ind2word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary Size:  99915\n"
     ]
    }
   ],
   "source": [
    "word2Ind, Ind2word = get_dict(corpus)\n",
    "\n",
    "V = len(word2Ind)\n",
    "print(\"Vocabulary Size: \", V)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Model Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Input Layer --> Hidden Layer --> Output Layer\n",
    "\n",
    "You will now initialize two matrices and two vectors. \n",
    "- The first matrix ($W_1$) is of dimension $N \\times V$, where $V$ is the number of words in your vocabulary and $N$ is the dimension of your word vector.\n",
    "- The second matrix ($W_2$) is of dimension $V \\times N$. \n",
    "- Vector $b_1$ has dimensions $N\\times 1$\n",
    "- Vector $b_2$ has dimensions  $V\\times 1$. \n",
    "- $b_1$ and $b_2$ are the bias vectors of the linear layers from matrices $W_1$ and $W_2$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize_model(N,V, random_seed=1):\n",
    "    \n",
    "    np.random.seed(random_seed)\n",
    "    \n",
    "    W1 = np.random.rand(N,V)\n",
    "    W2 = np.random.rand(V,N)\n",
    "    b1 = np.random.rand(N,1)\n",
    "    b2 = np.random.rand(V,1)\n",
    "    \n",
    "    return W1, W2, b1, b2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Softmax Function\n",
    "\n",
    "$$ \\text{softmax}(z_i) = \\frac{e^{z_i} }{\\sum_{i=0}^{V-1} e^{z_i} }  \\tag{5} $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(z):\n",
    "    \n",
    "    e_z = np.exp(z)\n",
    "    yhat = e_z/np.sum(e_z,axis=0)\n",
    "    \n",
    "    return yhat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Forward Propagation\n",
    "\n",
    "Implement the forward propagation $z$. <br>\n",
    "\n",
    "\\begin{align}\n",
    " h &= W_1 \\  X + b_1  \\tag{1} \\\\\n",
    " a &= ReLU(h)  \\tag{2} \\\\\n",
    " z &= W_2 \\  a + b_2   \\tag{3} \\\\\n",
    "\\end{align}\n",
    "\n",
    "For that, you will use as activation the Rectified Linear Unit (ReLU) given by:\n",
    "\n",
    "$$f(h)=\\max (0,h) \\tag{6}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_prop(x, W1, W2, b1, b2):\n",
    "    \n",
    "    h = np.dot(W1,x) + b1\n",
    "    \n",
    "    h = np.maximum(0,h)\n",
    "    \n",
    "    z = np.dot(W2,h) + b2\n",
    "    \n",
    "    return z, h"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cost function\n",
    "\n",
    "- Implementing the *cross-entropy* cost function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_cost(y, yhat, batch_size):\n",
    "    \n",
    "    logprobs = np.multiply(np.log(yhat),y) + np.multiply(np.log(1-yhat), 1 - y)\n",
    "    cost = -1/batch_size * np.sum(logprobs)\n",
    "    cost = np.squeeze(cost)\n",
    "    return cost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Backpropagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def back_prop(x, yhat, y, h, W1, W2, b1, b2, batch_size):\n",
    "    \n",
    "    l1 = np.dot(W2.T,(yhat-y))\n",
    "    l1 = np.maximum(0,l1)\n",
    "    \n",
    "    grad_W1 = (1/batch_size)*np.dot(l1,x.T)\n",
    "    grad_W2 = (1/batch_size)*np.dot(yhat-y,h.T)\n",
    "    grad_b1 = np.sum((1/batch_size)*np.dot(l1,x.T), axis=1, keepdims=True)\n",
    "    grad_b2 = np.sum((1/batch_size)*np.dot(yhat-y,h.T), axis=1, keepdims=True)\n",
    "    \n",
    "    return grad_W1, grad_W2, grad_b1, grad_b2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient Descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent(data, word2Ind, N, V, epochs, alpha=0.03):\n",
    "    \n",
    "    W1, W2, b1, b2 = initialize_model(N,V, random_seed=282)\n",
    "    batch_size = 128\n",
    "    iters = 0\n",
    "    C = 2\n",
    "    for x, y in get_batches(data, word2Ind, V, C, batch_size):\n",
    "        # Get z and h\n",
    "        z, h = forward_prop(x,W1,W2,b1,b2)\n",
    "        # Get yhat\n",
    "        yhat = softmax(z)\n",
    "        # Get cost\n",
    "        cost = compute_cost(y, yhat, batch_size)\n",
    "        if ( (iters+1) % 10 == 0):\n",
    "            print(f\"Epochs: {iters + 1} cost: {cost:.6f}\")\n",
    "        # Get gradients\n",
    "        grad_W1, grad_W2, grad_b1, grad_b2 = back_prop(x, yhat, y, h, W1, W2, b1, b2, batch_size)\n",
    "        \n",
    "        # Update weights and biases\n",
    "        W1 -= alpha*grad_W1\n",
    "        W2 -= alpha*grad_W2\n",
    "        b1 -= alpha*grad_b1\n",
    "        b2 -= alpha*grad_b2\n",
    "        \n",
    "        iters += 1 \n",
    "        if iters == epochs: \n",
    "            break\n",
    "        if iters % 100 == 0:\n",
    "            alpha *= 0.66\n",
    "            \n",
    "    return W1, W2, b1, b2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Call gradient_descent\n",
      "Epochs: 10 cost: 0.035759\n",
      "Epochs: 20 cost: 0.016522\n",
      "Epochs: 30 cost: 0.010792\n",
      "Epochs: 40 cost: 0.008024\n",
      "Epochs: 50 cost: 0.006389\n",
      "Epochs: 60 cost: 0.005310\n",
      "Epochs: 70 cost: 0.004543\n",
      "Epochs: 80 cost: 0.003970\n",
      "Epochs: 90 cost: 0.003526\n",
      "Epochs: 100 cost: 0.003172\n",
      "Epochs: 110 cost: 0.002965\n",
      "Epochs: 120 cost: 0.002792\n",
      "Epochs: 130 cost: 0.002638\n",
      "Epochs: 140 cost: 0.002501\n",
      "Epochs: 150 cost: 0.002377\n",
      "Epochs: 160 cost: 0.002264\n",
      "Epochs: 170 cost: 0.002162\n",
      "Epochs: 180 cost: 0.002069\n",
      "Epochs: 190 cost: 0.001983\n",
      "Epochs: 200 cost: 0.001905\n",
      "Wall time: 11min 9s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "C = 2\n",
    "N = 100\n",
    "word2Ind, Ind2word = get_dict(corpus)\n",
    "V = len(word2Ind)\n",
    "epochs = 200\n",
    "print(\"Call gradient_descent\")\n",
    "W1, W2, b1, b2 = gradient_descent(corpus, word2Ind, N, V, epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}