Word2Vec_Embedding_Generation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e86ae20e",
   "metadata": {},
   "source": [
    "# Task 2: Word2Vec - Data Preparation and Word2Vec Embeddings\n",
    "-------------------------------------------------------------------------------\n",
    "\n",
    "This notebook includes the word2vec embedding creation and preparation of the data using the embedding model followed by an examination of resulting embeddings.\n",
    "\n",
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c5ae8a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#For dataset I/O\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle, csv\n",
    "from sklearn.utils import shuffle\n",
    "import random\n",
    "import project2Lib\n",
    "\n",
    "## for plotting\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.manifold import TSNE\n",
    "\n",
    "#for text pre-processing\n",
    "from nltk.tokenize import word_tokenize\n",
    "import nltk\n",
    "\n",
    "#for word clouds\n",
    "from wordcloud import WordCloud\n",
    "\n",
    "# Word2Vec\n",
    "from gensim.models import Word2Vec, KeyedVectors\n",
    "from gensim.models.callbacks import CallbackAny2Vec\n",
    "\n",
    "\n",
    "#For Keras Deep Learning Models\n",
    "from tensorflow.keras import models, layers, preprocessing\n",
    "from tensorflow.keras import backend as K\n",
    "\n",
    "np.random.seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee68329f",
   "metadata": {},
   "source": [
    "## Loading preprocessed data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f54e9ad0",
   "metadata": {},
   "outputs": [],
   "source": [
    "suffix = \"\"\n",
    "mode = 1\n",
    "\n",
    "if   mode==0:\n",
    "    suffix = \"lemmatization_noph\"\n",
    "    \n",
    "elif mode==1:\n",
    "    suffix = \"lemmatization\"\n",
    "    \n",
    "elif mode==2:\n",
    "    suffix = \"_noph\"\n",
    "\n",
    "elif mode==3:\n",
    "    suffix = \"_\"\n",
    "    \n",
    "elif mode==4:\n",
    "    suffix = \"stemming_noph\"\n",
    "    \n",
    "elif mode==5:\n",
    "    suffix = \"stemming\"\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b0e0f55",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_csv( f'PreprocessedData/train_{suffix}.csv').dropna()\n",
    "dev_data   = pd.read_csv( f'PreprocessedData/dev_{suffix}.csv'  ).dropna()\n",
    "test_data  = pd.read_csv( f'PreprocessedData/test_{suffix}.csv' ).dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da8f0038",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eb663040",
   "metadata": {},
   "source": [
    "## Splitting sentences into tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2c01205",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data[\"tokens\"] = train_data['preprocess'].apply(lambda x: nltk.word_tokenize(x))\n",
    "dev_data[\"tokens\"]   = dev_data['preprocess'].apply(lambda x: nltk.word_tokenize(x))\n",
    "test_data[\"tokens\"]  = test_data['preprocess'].apply(lambda x: nltk.word_tokenize(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b057654a",
   "metadata": {},
   "source": [
    "### Inspecting sentence length distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afa3cc63",
   "metadata": {},
   "outputs": [],
   "source": [
    "def words_per_sent(txt):\n",
    "    # split text into words and count them.\n",
    "    return len(txt.split()) \n",
    "\n",
    "# apply to our dataframe\n",
    "train_data['sent_len'] = train_data[\"tokens\"].apply(lambda x: len(x))\n",
    "#dev_data['num_words'].hist()\n",
    "\n",
    "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 8))\n",
    "train_data['sent_len'].hist(ax=ax1, bins=20)\n",
    "ax1.set_title('Sentence Lenght Distribution')\n",
    "\n",
    "\n",
    "#dev_data.plot( x='sent_len', y='target', kind='bar', ax=ax[1])\n",
    "ax2.set_title('Sentence Lenght By Label')\n",
    "\n",
    "sns.boxplot(x=\"label\", y=\"sent_len\", data=train_data, ax=ax2)\n",
    "\n",
    "ax2.set_xticks(np.arange(5) , labels=[\"BACKGROUND\", \"OBJECTIVE\", \"METHODS\", \"RESULTS\", \"CONCLUSIONS\"])\n",
    "\n",
    "plt.show()\n",
    "\n",
    "print(f\"Maximum sentence length: {max(train_data['sent_len'])}\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7a1b189",
   "metadata": {},
   "source": [
    "### We see that while variance in sentence length is different for different labels, the average sentence length does not appear informative. \n",
    "- including sentence length as a variable could help a model decide that e.g. longer sentences are more likely to be \"RESULTS\" than \"CONCLUSIONS\" but the effect does not appear significant other than for outliers.\n",
    "- the great majority of sentences are shorter than 50 let alone 100 words/tokens. When processing sentences sewuentially we will need a maximum length to pad sequences to. A value such as 100 or 150 will be a good compromise between preserving information from sentences with outlier length and keeping the dataset size manageable. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fe2e590",
   "metadata": {},
   "source": [
    "## Visualising most frequent words for each sentence type\n",
    "\n",
    "We visualize the most frequent words found in sentences of each type in word clouds. Top 10 most frequent words are not visualised as they are similar for most labels. Both tf-idf and Word2Vec vectorizers have settings to deal with too common / uninformative words so this change is not needed for those models. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8bb9c81c",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# ignoring most frequent 10 words\n",
    "ignore_top = 10\n",
    "\n",
    "\n",
    "labels = ['BACKGROUND', 'OBJECTIVE','METHODS', 'RESULTS','CONCLUSIONS']\n",
    "fig, axs = plt.subplots(3, 2, figsize=(15,25))\n",
    "\n",
    "for i in range(5): \n",
    "    \n",
    "    ax = axs[i//2, i%2]\n",
    "    freqs = pd.Series(np.concatenate( train_data.loc[train_data.label == i, 'tokens'].\n",
    "                                     values ) ).value_counts()[ignore_top:]\n",
    "    \n",
    "    # generate word cloud of words with highest counts\n",
    "    wordcloud = WordCloud(height=400, max_words=100, background_color=\"white\").generate_from_frequencies(freqs) \n",
    "    ax.set_title(labels[i], fontsize=20 )\n",
    "    ax.imshow(wordcloud, interpolation='bilinear') \n",
    "    ax.axis(\"off\") \n",
    "axs[2, 1].remove()\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa81e1b0",
   "metadata": {},
   "source": [
    "## Interpretation of Word Clouds\n",
    "\n",
    "Looking at each word cloud, we can identify certain words that seem intuitive for the respective sentence types.\n",
    "\n",
    "### Background and Objective\n",
    "\n",
    "These two labels display some similarities in word distribution, as we will also observe in the t-SNE plot of the averaged sentence embeddings later in the notebook. The word \"aim\" appears frequently in both while not appearing in the word clouds of other labels. However, these two labels are still differentiated by other words such as \"whether\", \"assess\", \"investigate\" which are more common for the OBJECTIVE label. We could perhaps attribute this to these words being used in establishing a formal hypothesis, which we would expect to see as an objective sentence.\n",
    "\n",
    "\n",
    "### Methods\n",
    "\n",
    "We see that words such as \"day\", \"month\", \"year\", \"placebo\",\"measure\" and \"trial\" are prominent for this label, which is highly intuitive given that a research paper methodology based on clinical trials will likely have a scheduling / timing component as well as possibly a standard procedure for drug adminsitration. \n",
    "\n",
    "\n",
    "### Results and Conclusion\n",
    "\n",
    "For these two labels we observe quantity comparison / interpretation related words such as \"increase\", \"mean\", \"high\", \"rate\", \"reduce\". It appears intuitive that a RESULTS sentence would uncover or confirm a relationship between quantities, possibly using numerical data and its description. A conclusion sentence may be referring to the previously declared results or may be restating them. Tt appears difficult to find an intuitive way to distinguish the word cloud for these two labels. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56756b8d",
   "metadata": {},
   "source": [
    "# Word2Vec Model\n",
    "\n",
    "A pretrained Gnesim Word2Vec model with 200 dimensional embeddings is given in the repository. After running the constants cell, you can choose to either skip the model creation cell and load keyed vectors directly, or run the model and then load the keyed vectors from the new saved model.\n",
    "\n",
    "## Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ebb2c14",
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs=15\n",
    "vector_dim = 200\n",
    "window = 10\n",
    "min_count = 10\n",
    "save_name = f'./TrainedModels/w2v_{vector_dim}_{suffix}.bin'\n",
    "\n",
    "#choosing skip-gram over CBOW\n",
    "sg = 1\n",
    "#for using hierarchical softmax in word2vec model\n",
    "hs = 1\n",
    "\n",
    "# callback function for observing loss after each epoch\n",
    "\n",
    "class callback(CallbackAny2Vec):\n",
    "    '''Prints loss after each epoch.'''\n",
    "\n",
    "    def __init__(self):\n",
    "        self.epoch = 0\n",
    "        self.accum_loss = 0\n",
    "\n",
    "    def on_epoch_end(self, model):\n",
    "        current_loss = model.get_latest_training_loss()\n",
    "        model.running_training_loss=0\n",
    "\n",
    "        print(f'Loss at epoch {self.epoch}: {current_loss}')\n",
    "        self.epoch += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa201f4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# shuffling data as Gensim does not do it automatically\n",
    "shuffled_token_sents = train_data[\"tokens\"].copy().sample(frac=1).values.tolist()\n",
    "\n",
    "wv_model = Word2Vec(shuffled_token_sents, workers = 20, \n",
    "               vector_size=vector_dim,  # vector dim    \n",
    "               min_count =  min_count, # min word count filter \n",
    "               window = window , # context window      \n",
    "               sg = sg,\n",
    "               hs = hs,\n",
    "               callbacks=[callback()],\n",
    "               epochs = epochs,\n",
    "               compute_loss=True\n",
    "               )\n",
    "\n",
    "\n",
    "print(\"Model vocabulary size: \" + str(len(wv_model.wv.key_to_index)))\n",
    "  \n",
    "# Save Model\n",
    "wv_model.init_sims(replace=True)\n",
    "wv_model.wv.save_word2vec_format(save_name, binary=True)\n",
    "kv = KeyedVectors.load_word2vec_format(save_name, binary=True) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b73dd30",
   "metadata": {},
   "source": [
    "## To load existing model keyed vectors:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77b0234a",
   "metadata": {},
   "outputs": [],
   "source": [
    "kv = KeyedVectors.load_word2vec_format(save_name, binary=True) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3616be56",
   "metadata": {},
   "source": [
    "# Analysing semantic relationships between embeddings \n",
    "\n",
    "### Observing similarity relationships"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed436968",
   "metadata": {},
   "outputs": [],
   "source": [
    "kv.most_similar(positive=\"doctor\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "507a9b02",
   "metadata": {},
   "outputs": [],
   "source": [
    "kv.most_similar(positive=\"patient\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4830515b",
   "metadata": {},
   "outputs": [],
   "source": [
    "kv.most_similar(positive=\"cancer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dce1b66a",
   "metadata": {},
   "outputs": [],
   "source": [
    "kv.most_similar(positive=\"diabetes\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec25c546",
   "metadata": {},
   "source": [
    "Now, to compare word similarity performance to previous studies using Word2Vec, we use the word aspirin that was used in the following paper:\n",
    "\n",
    "Miñarro-Giménez, J. A., Marín-Alonso, O., and Samwald, M., “Applying deep learning techniques on medical corpora from the World Wide Web: a prototypical system and evaluation”, <i>arXiv e-prints</i>, 2015.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0a1d1ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "kv.most_similar(positive=\"aspirin\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63b2273b",
   "metadata": {},
   "source": [
    "The top results appear to be mostly the same words with variations in cosine similarity value."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b91ca920",
   "metadata": {},
   "source": [
    "## Observing Analogy Relationships\n",
    "-------------------------------------------------\n",
    "In this section we will inspect whether binary semantic relationships between words is captured by Word2Vec embeddings and whether these relationships are directly applicable to similar expected semantic relationships between other word pairs.\n",
    "\n",
    "As the model was trained on medical research paper abstracts, the vocabulary does not lend itself to a wide variety of analogies as a non-medical corpus may. However, we still expect to uncover reasonable examples of intuitive relationships. Simple relationships may be captured from words that are common in general language structures, however data may not be diverse enough to capture relationships such as \"king is to man as queen is to woman\".\n",
    "\n",
    "_**Word 1** is to **Word 2** as **New Word 1** is to **New Word 2**_\n",
    "\n",
    "w2v( **Word 1** ) - w2v( **New Word 1** )  =  w2v( **Word 2** ) - w2v( **New Word 2** )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b48a8ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "def analogy(kv, word1, word2, new_word1):\n",
    "    return kv.most_similar(negative=[word1],positive=[word2, new_word1])\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7011d395",
   "metadata": {},
   "outputs": [],
   "source": [
    "analogy(kv, \"good\",\"bad\", \"successful\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5909cd13",
   "metadata": {},
   "outputs": [],
   "source": [
    "analogy(kv, \"healthy\",\"ill\", \"recover\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6aaf5039",
   "metadata": {},
   "outputs": [],
   "source": [
    "analogy(kv, \"significantly\",\"slightly\", \"severe\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1baad7d8",
   "metadata": {},
   "source": [
    "In all three of the exampels above, an opposite / dialectic relationship is captured in a significant portion of response words "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4677c27d",
   "metadata": {},
   "outputs": [],
   "source": [
    "analogy(kv, \"breakfast\",\"morning\", \"dinner\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d4e985a",
   "metadata": {},
   "outputs": [],
   "source": [
    "analogy(kv, \"fever\",\"paracetamol\", \"schizophrenia\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11bf28cc",
   "metadata": {},
   "source": [
    "These two examples demosntrate different relationships based on associations. Time and meal type relationship is captured in the first one while condition and a corresponding treatment option relationship is captured in the last example."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e67d6931",
   "metadata": {},
   "source": [
    "# t-SNE Plot of Word2Vec Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69904829",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tsne_word2vec(kv, num_vecs = 20000):\n",
    "\n",
    "    labels = list(kv.key_to_index.keys())[:num_vecs]\n",
    "    \n",
    "    tsne_model = TSNE(perplexity=30, n_components=2, verbose=1,\n",
    "                      init='pca', n_iter=300, random_state=1)\n",
    "    data = tsne_model.fit_transform(kv.vectors[:num_vecs])\n",
    "\n",
    "\n",
    "    x = data[:,0]\n",
    "    y = data[:,1]\n",
    "    \n",
    "\n",
    "    to_annotate = set(random.sample(range(0, len(x)), 100))\n",
    "        \n",
    "    plt.figure(figsize=(16, 16)) \n",
    "    for i in range(len(x)):\n",
    "        plt.scatter(x[i],y[i], cmap='viridis')\n",
    "        if i in to_annotate:\n",
    "            plt.annotate(labels[i],\n",
    "                         xy=(x[i]+0.25, y[i]),\n",
    "                         xytext=(5, 2),\n",
    "                         textcoords='offset points',\n",
    "                         ha='right',\n",
    "                         va='bottom',\n",
    "                         size='medium',\n",
    "                         backgroundcolor=\"white\"\n",
    "                        )\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a7b3aba",
   "metadata": {},
   "outputs": [],
   "source": [
    "tsne_word2vec(kv)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "198fbde7",
   "metadata": {},
   "source": [
    "It is difficult to make intuitive interpretations based on the word embedding t-SNE produced here, partially since we annotate a random sample of embedded words. Interpreting analogies and most similar words to single words is a better way of obserbving semantic quality of the model.  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d6b83f46",
   "metadata": {},
   "source": [
    "# Preparing data for classifiers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bfe7b5b3",
   "metadata": {},
   "source": [
    "## To get embedding matrix lookup index sequences for each sentence - for sequential classifers\n",
    "\n",
    "Instead of saving the Word2Vec Embeddings fro each word of each sentence directly in the dataset, we save the vectors' lookup indices corresponding the Word2Vec model's vector matrix. Later, this matrix will be placed in a Keras embedding layer that will retrieve the word vectors on the run. This way we save memory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9628bab",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_data   = project2Lib.words_to_idx(dev_data,   kv, save_name = f\"PreprocessedData/dev_{suffix}_w2v\")\n",
    "test_data  = project2Lib.words_to_idx(test_data,  kv, save_name = f\"PreprocessedData/test_{suffix}_w2v\")\n",
    "train_data = project2Lib.words_to_idx(train_data, kv, save_name = f\"PreprocessedData/train_{suffix}_w2v\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98e11746",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read data\n",
    "#train_data = pd.read_pickle (f\"PreprocessedData/train_{suffix}_w2v\")\n",
    "#dev_data = pd.read_pickle (f\"PreprocessedData/dev_{suffix}_w2v\")\n",
    "#test_data = pd.read_pickle (f\"PreprocessedData/test_{suffix}_w2v\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "049aba2a",
   "metadata": {},
   "source": [
    "## Averaging words to obtain sentence embeddings - for non sequential models:\n",
    "\n",
    "For classifiers that cannot not process word embeddings sequentially, we need some type of aggregation to obtain sentence embeddings. One way of doing this is averaging word embeddings, which we carry out below. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b5121f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_data   = project2Lib.vectorize_dataset(dev_data,   kv, save_name = f\"PreprocessedData/dev_{suffix}_w2v\")\n",
    "test_data  = project2Lib.vectorize_dataset(test_data,  kv, save_name = f\"PreprocessedData/test_{suffix}_w2v\")\n",
    "train_data = project2Lib.vectorize_dataset(train_data, kv, save_name = f\"PreprocessedData/train_{suffix}_w2v\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f685121",
   "metadata": {},
   "source": [
    "# Visualizing average sentence vectors with t-SNE\n",
    "\n",
    "To get an intuition about how well the average sentence vectors cluster or whether averaging retains enough discriminative information, we observe the t-SNE plot of our averaged training sentence vectors for each sentence label. To have a reasonably limited computation time, we run t-sne on the dev set instead of the training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d365314d",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def tsne_sentence_vec(df):\n",
    "\n",
    "    tsne_model = TSNE(perplexity=30, n_components=2, init='pca', \n",
    "                      n_iter=500, random_state=1, learning_rate='auto', verbose=1)\n",
    "    \n",
    "    data = tsne_model.fit_transform(np.stack(df[\"avg_vectors\"].values))\n",
    "\n",
    "    df['tsne-1'] = data[:,0]\n",
    "    df['tsne-2'] = data[:,1]\n",
    "    \n",
    "    plt.figure(figsize=(16,16))\n",
    "    ax = sns.scatterplot(\n",
    "        x='tsne-1', y='tsne-2',\n",
    "        hue=\"label\",\n",
    "        palette=sns.color_palette(\"hls\", 5),\n",
    "        data=df,\n",
    "        legend=True,\n",
    "        alpha=0.9\n",
    "    )\n",
    "    handles, labels  =  ax.get_legend_handles_labels()\n",
    "    ax.legend(handles=handles, title='Classes', loc='upper right', labels=[\"BACKGROUND\", \"OBJECTIVE\", \"METHODS\", \"RESULTS\", \"CONCLUSIONS\"])\n",
    "    \n",
    "    fig = plt.gcf()\n",
    "    fig.savefig(f'tsne_avg_sent_{suffix}.png')\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0da3849a",
   "metadata": {},
   "outputs": [],
   "source": [
    "tsne_sentence_vec(dev_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86e6c785",
   "metadata": {},
   "source": [
    "On the averaged sentence vector t-SNE plot we can make the following observations:\n",
    "- RESULTS  and METHODS sentences are both a large portion of the data and form more visibly seperable regions compared to other sentence types\n",
    "\n",
    "- BACKGROUND and CONCLUSION appear to be clustering in very similar regions\n",
    "\n",
    "- OBJECTIVE does not display an apparent clustering behavior other than being relatively sepearble from RESULTS and to a lesser degree, from METHODS\n",
    "\n",
    "- We know that conclusions tend to come last in general English language while background information is likley to be towards the beginning, therefore the seperablitiy issue we observe can be mitigated by introducing the relative position of the sentence in the abstract as a feature. Indeed, for both averaged sentence vector based models and sequence classificaiton models in the other notebooks, we will see that a significant performance boost will come with this change. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89a4bb65",
   "metadata": {},
   "source": [
    "# For comparison with smaller dataset:\n",
    "\n",
    "The dataset preparation and word2Vec related steps above are repeated below for the samller 20k dataset, later to be used in the classifiers as well. We aim to identify whether working with the larger dataset makes a noticable difference on classifier performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "907c7287",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_small = pd.read_csv( f'PreprocessedData/train_{suffix}_small.csv').dropna()\n",
    "dev_data_small   = pd.read_csv( f'PreprocessedData/dev_{suffix}_small.csv'  ).dropna()\n",
    "test_data_small  = pd.read_csv( f'PreprocessedData/test_{suffix}_small.csv' ).dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77bb5b31",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_small[\"tokens\"] = train_data_small['preprocess'].apply(lambda x: nltk.word_tokenize(x))\n",
    "dev_data_small[\"tokens\"]   = dev_data_small['preprocess'].apply(lambda x: nltk.word_tokenize(x))\n",
    "test_data_small[\"tokens\"]  = test_data_small['preprocess'].apply(lambda x: nltk.word_tokenize(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78e3009d",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_name_small = f'./TrainedModels/w2v_{vector_dim}_{suffix}_small.bin'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1370cac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# shuffling data as Gnesim does not do it automatically\n",
    "shuffled_token_sents_small = train_data_small[\"tokens\"].copy().sample(frac=1).values.tolist()\n",
    "\n",
    "wv_model_small = Word2Vec(shuffled_token_sents_small, workers = 20, \n",
    "               vector_size=vector_dim,  # vector dim    \n",
    "               min_count =  min_count, # min word count filter \n",
    "               window = window , # context window      \n",
    "               sg = sg,\n",
    "               hs = hs,\n",
    "               callbacks=[callback()],\n",
    "               epochs = epochs,\n",
    "               compute_loss=True\n",
    "               )\n",
    "\n",
    "\n",
    "print(\"Model vocabulary size: \" + str(len(wv_model_small.wv.key_to_index)))\n",
    "  \n",
    "# Save Model\n",
    "wv_model_small.init_sims(replace=True)\n",
    "wv_model_small.wv.save_word2vec_format(save_name_small, binary=True)\n",
    "kv_small = KeyedVectors.load_word2vec_format(save_name_small, binary=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52a0bc9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_data_small   = project2Lib.words_to_idx(dev_data_small  , kv_small, save_name = f\"PreprocessedData/dev_{suffix}_w2v_small\")\n",
    "test_data_small  = project2Lib.words_to_idx(test_data_small,  kv_small, save_name = f\"PreprocessedData/test_{suffix}_w2v_small\")\n",
    "train_data_small = project2Lib.words_to_idx(train_data_small, kv_small, save_name = f\"PreprocessedData/train_{suffix}_w2v_small\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e735e768",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_data_small   = project2Lib.vectorize_dataset(dev_data_small,   kv_small, save_name = f\"PreprocessedData/dev_{suffix}_w2v_small\")\n",
    "test_data_small  = project2Lib.vectorize_dataset(test_data_small,  kv_small, save_name = f\"PreprocessedData/test_{suffix}_w2v_small\")\n",
    "train_data_small = project2Lib.vectorize_dataset(train_data_small, kv_small, save_name = f\"PreprocessedData/train_{suffix}_w2v_small\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3cbf6a52",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}