Word2Vec_Sequential_Classifiers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7ac1a14a",
   "metadata": {},
   "source": [
    "# Task 2: Word2Vec - Sequence Classification Approach\n",
    "\n",
    "\n",
    "In this notebook, word embeddings produced by Word2Vec are not aggregated into a single sentence emedding, but instead kept as a sequence of embeddings. Therefore, sequence classification models are applied.\n",
    "\n",
    "\n",
    "----------------------------------------------\n",
    "\n",
    "## Importing Libraries\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8638affd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#For dataset I/O\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle, csv\n",
    "from sklearn.utils import shuffle\n",
    "import project2Lib\n",
    "\n",
    "## for plotting\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "#for text pre-processing\n",
    "import re, string\n",
    "import nltk\n",
    "\n",
    "# Word2Vec\n",
    "from gensim.models import Word2Vec, KeyedVectors\n",
    "\n",
    "\n",
    "#For Keras Deep Learning Models\n",
    "from tensorflow.keras import models, layers, preprocessing, Sequential,  losses, Model\n",
    "from tensorflow.keras import backend as K\n",
    "\n",
    "#from tensorflow import keras\n",
    "from tensorflow.keras import layers, backend as K\n",
    "from tensorflow.keras.models import Sequential\n",
    "import tensorflow as tf\n",
    "\n",
    "from tensorflow.keras.utils import Sequence\n",
    "from tensorflow.keras.layers import LSTM, Dense, Masking, Embedding, Input\n",
    "import numpy as np\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\n",
    "\n",
    "\n",
    "#For Peformance Metrics\n",
    "from sklearn.metrics import classification_report, f1_score, accuracy_score, plot_confusion_matrix, confusion_matrix, ConfusionMatrixDisplay\n",
    "\n",
    "np.random.seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0cfe0457",
   "metadata": {},
   "source": [
    "### Checking for GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33bbabac",
   "metadata": {},
   "outputs": [],
   "source": [
    "device_name = tf.test.gpu_device_name()\n",
    "if device_name != \"/device:GPU:0\":\n",
    "  device_name = \"/cpu:0\"\n",
    "print('Found device at: {}'.format(device_name))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef51fcae",
   "metadata": {},
   "source": [
    "## Loading preprocessed data\n",
    "\n",
    "### Choosing one of the preprocessing options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af2913b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "suffix = \"\"\n",
    "mode = 1\n",
    "\n",
    "if   mode==0:\n",
    "    suffix = \"lemmatization_noph\"\n",
    "    \n",
    "elif mode==1:\n",
    "    suffix = \"lemmatization\"\n",
    "    \n",
    "elif mode==2:\n",
    "    suffix = \"_noph\"\n",
    "\n",
    "elif mode==3:\n",
    "    suffix = \"_\"\n",
    "    \n",
    "elif mode==4:\n",
    "    suffix = \"stemming_noph\"\n",
    "    \n",
    "elif mode==5:\n",
    "    suffix = \"stemming\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cab3644c",
   "metadata": {},
   "outputs": [],
   "source": [
    "    \n",
    "# read data\n",
    "train_data = pd.read_pickle (f\"PreprocessedData/train_{suffix}_w2v.pkl\")\n",
    "dev_data = pd.read_pickle (f\"PreprocessedData/dev_{suffix}_w2v.pkl\")\n",
    "test_data = pd.read_pickle (f\"PreprocessedData/test_{suffix}_w2v.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16b28f48",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2fca215",
   "metadata": {},
   "source": [
    "## To load existing model's keyed vectors:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "915c8a55",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load saved model\n",
    "kv = KeyedVectors.load_word2vec_format(f\"./TrainedModels/w2v_200_{suffix}.bin\", binary=True) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "006119f3",
   "metadata": {},
   "source": [
    "## To prepare data:\n",
    "\n",
    "the X_train_lines data is extracted for models that use the relative line number as an auxiliary input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2364a67f",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_idx = np.stack(train_data[\"idx\"].values)\n",
    "X_train_lines = np.reshape(train_data[\"line_relative\"].values, (-1, 1))\n",
    "Y_train = train_data['label'].values\n",
    "\n",
    "X_dev_idx = np.stack(dev_data[\"idx\"].values)\n",
    "X_dev_lines = np.reshape(dev_data[\"line_relative\"].values, (-1, 1))\n",
    "Y_dev = dev_data['label'].values\n",
    "\n",
    "X_test_idx = np.stack(test_data[\"idx\"].values)\n",
    "X_test_lines = np.reshape(test_data[\"line_relative\"].values, (-1, 1))\n",
    "Y_test = test_data['label'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0076e276",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data[\"idx\"].values[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e8ad4b0",
   "metadata": {},
   "source": [
    "Here we can see that our prepared data is padded to a standard sequence length. Sequential classifiers will be set to ignore the zero entries."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3b88a14",
   "metadata": {},
   "source": [
    "# Keras Based Sequential Classifiers\n",
    "\n",
    "In this section, models that process Word2Vec embeddings of individual words sequentially to classify a sentence are explored. In most models, word embeddings are first processed by a Bidirectional LSTM, the rest of the model varies.  \n",
    "\n",
    "-----------------------\n",
    "\n",
    "# Bidirectional LSTM\n",
    "\n",
    "## Without Line Numbers\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d975bdbe",
   "metadata": {},
   "source": [
    "### Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4ab23c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "dimension=kv.vector_size\n",
    "units=32\n",
    "lr = 0.01\n",
    "max_sent_len = len(X_train_idx[0])\n",
    "epochs = 50\n",
    "dropout= 0.2\n",
    "\n",
    "save_name = f\"./TrainedModels/biLSTM_noline_{suffix}a\" + \".h5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af884b28",
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.device(device_name):\n",
    "    model = Sequential()\n",
    "\n",
    "    embed_layer = Embedding(input_dim=   kv.vectors.shape[0]+1, \n",
    "                            output_dim=  kv.vectors.shape[1], \n",
    "                            weights=     [np.vstack((np.zeros((1, kv.vectors.shape[1])),kv.vectors))],             \n",
    "                            input_length=max_sent_len,\n",
    "                            mask_zero=   True,\n",
    "                            trainable=   False)\n",
    "    model.add(embed_layer)\n",
    "    model.add(layers.Bidirectional(\n",
    "            LSTM(\n",
    "                units=units,\n",
    "                activation='tanh'\n",
    "            )) )\n",
    "\n",
    "    model.add(layers.Dense(units, activation='relu'))\n",
    "    model.add(layers.Dense(units, activation='relu'))\n",
    "    model.add(layers.Dense(5, activation='softmax'))\n",
    "\n",
    "    model.compile(loss=losses.sparse_categorical_crossentropy, \n",
    "                              optimizer=\"adam\", metrics=['accuracy'])\n",
    "\n",
    "\n",
    "    K.set_value(model.optimizer.learning_rate, lr)\n",
    "\n",
    "    early = EarlyStopping(monitor=\"val_loss\", patience=10, verbose=1)\n",
    "    redonplat = ReduceLROnPlateau(monitor=\"val_loss\", patience=5, verbose=1)\n",
    "    checkpoint = ModelCheckpoint( filepath=save_name, save_weights_only=True, \n",
    "                                                 monitor='val_accuracy', mode='max', save_best_only=True)\n",
    "\n",
    "    callbacks_list = [early, redonplat,checkpoint]  \n",
    "\n",
    "    model.fit(X_train_idx, Y_train, epochs=epochs, batch_size=1000, \n",
    "                                    verbose=1, validation_data=(X_dev_idx,Y_dev), callbacks=callbacks_list )\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab7d6f03",
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred = model.predict(X_test_idx)\n",
    "Y_pred = np.argmax(Y_pred, axis=-1)\n",
    "del model\n",
    "\n",
    "\n",
    "print(\"Accuracy: \" ,accuracy_score(Y_test, Y_pred))\n",
    "print(\"F1 Score: \" ,f1_score(Y_test, Y_pred, average='weighted') )\n",
    "cm = confusion_matrix(Y_test, Y_pred, normalize = \"true\")\n",
    "cmd = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45bb85d8",
   "metadata": {},
   "source": [
    "-------------------------------------------------------------------------------------------\n",
    "\n",
    "## With Line Numbers\n",
    "\n",
    "In this section, a version of the Bidirectional LSTM based model that takes in relative line number as an auxiliary input is implemented. Since for all previous models, line number improves performance significantly, we expect this to be the case for this model too.\n",
    "\n",
    "### Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93fc58e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "dimension=kv.vector_size\n",
    "units=32\n",
    "lr = 0.001\n",
    "max_sent_len = len(X_train_idx[0])\n",
    "epochs = 40\n",
    "\n",
    "save_name = f\"./TrainedModels/biLSTM_line_{suffix}\" + \".h5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b148346",
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.device(device_name):\n",
    "    \n",
    "    input_sequence = Input(shape=(max_sent_len,), name='input_sequence')\n",
    "    input_sent_num = Input(shape=(1,), name='input_sent_num')\n",
    "\n",
    "\n",
    "    # ---- the sequential processing section\n",
    "\n",
    "    x = Embedding(input_dim=  kv.vectors.shape[0]+1, \n",
    "                            output_dim=  kv.vectors.shape[1], \n",
    "                            weights=     [np.vstack((np.zeros((1, kv.vectors.shape[1])),kv.vectors))],             \n",
    "                            input_length=max_sent_len,\n",
    "                            mask_zero=   True,\n",
    "                            trainable=   False)(input_sequence)\n",
    "    \n",
    "    x = layers.Bidirectional( LSTM( units=units, activation='tanh'))(x)\n",
    "    combined = layers.concatenate([x, input_sent_num])\n",
    "    \n",
    "    y = layers.Dense(units, activation='relu')(combined)\n",
    "    y = layers.Dense(units, activation='relu')(y)\n",
    "    output = layers.Dense(5, activation='softmax')(y)\n",
    "    \n",
    "    aux_line_model = Model(inputs=[input_sequence, input_sent_num], outputs=output)\n",
    "    \n",
    "    print(aux_line_model.summary())\n",
    "\n",
    "    aux_line_model.compile(loss=losses.sparse_categorical_crossentropy, \n",
    "                              optimizer=\"adam\", metrics=['accuracy'])\n",
    "\n",
    "\n",
    "    K.set_value(aux_line_model.optimizer.learning_rate, lr)\n",
    "\n",
    "    early = EarlyStopping(monitor=\"val_loss\", patience=10, verbose=1)\n",
    "    redonplat = ReduceLROnPlateau(monitor=\"val_loss\", patience=5, verbose=1)\n",
    "    checkpoint = ModelCheckpoint( filepath=save_name, save_weights_only=True, \n",
    "                                                 monitor='val_accuracy', mode='max', save_best_only=True)\n",
    "\n",
    "    callbacks_list = [early, redonplat,checkpoint] \n",
    "    \n",
    "    aux_line_model.fit({'input_sequence': X_train_idx, 'input_sent_num': X_train_lines}, Y_train, \n",
    "              epochs=epochs, batch_size=1000,  verbose=1, \n",
    "              validation_data=({'input_sequence': X_dev_idx, 'input_sent_num': X_dev_lines},Y_dev), \n",
    "              callbacks=callbacks_list)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b2d670d",
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred = aux_line_model.predict({'input_sequence': X_test_idx, 'input_sent_num': X_test_lines})\n",
    "Y_pred = np.argmax(Y_pred, axis=-1)\n",
    "del aux_line_model\n",
    "\n",
    "print(\"Accuracy: \" ,accuracy_score(Y_test, Y_pred))\n",
    "print(\"F1 Score: \" ,f1_score(Y_test, Y_pred, average='weighted') )\n",
    "cm = confusion_matrix(Y_test, Y_pred, normalize = \"true\")\n",
    "cmd = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4cf42b11",
   "metadata": {},
   "source": [
    "-----------------------\n",
    "\n",
    "# Bidirectional LSTM + Conv1D layers\n",
    "\n",
    "This model combines an initial Bidirectional LSTM that aggregates word vectors into a single embedding with an attention layer and one dimensional convolutions. Given that Bidirectional LSTMs are commonly used in text classification, and covolutions are an effective way to get capacity, this model was deemed worth investigating. LSTM cell size was limited by computation time, larger hidden state sizes could create marginal improvements.\n",
    "\n",
    "## Without Line Numbers\n",
    "\n",
    "### Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a025ef8",
   "metadata": {},
   "outputs": [],
   "source": [
    "dropout=0\n",
    "units=32 # = filters\n",
    "max_sent_len = len(X_train_idx[0])\n",
    "conv_layers = 3\n",
    "lr = 0.01\n",
    "epochs = 60\n",
    "\n",
    "save_name = f\"./TrainedModels/biLSTM_conv_{suffix}\" + \".h5\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1cccf8ae",
   "metadata": {},
   "source": [
    "### Attention Module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c902aa0",
   "metadata": {},
   "outputs": [],
   "source": [
    "class attention(keras.layers.Layer):\n",
    "    def __init__(self,return_sequences=False):\n",
    "        #super(attention,self).__init__(**kwargs)\n",
    "        self.return_sequences = return_sequences\n",
    "\n",
    "        super(attention,self).__init__()\n",
    " \n",
    "    def build(self,input_shape):\n",
    "        self.W=self.add_weight(name='attention_weight', shape=(input_shape[-1],1), \n",
    "                               initializer='random_normal', trainable=True)\n",
    "        self.b=self.add_weight(name='attention_bias', shape=(input_shape[1],1), \n",
    "                               initializer='zeros', trainable=True)        \n",
    "        super(attention, self).build(input_shape)\n",
    " \n",
    "    def call(self,x):\n",
    "\n",
    "        e = K.tanh(K.dot(x,self.W)+self.b)\n",
    "        e = K.squeeze(e, axis=-1)   \n",
    "        alpha = K.softmax(e)\n",
    "        alpha = K.expand_dims(alpha, axis=-1)\n",
    "        \n",
    "        #final context vector\n",
    "        context = x * alpha\n",
    "        \n",
    "        if self.return_sequences:\n",
    "            return context\n",
    "        \n",
    "        context = K.sum(context, axis=1)\n",
    "        return context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86021aff",
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.device(device_name):\n",
    "    blstm_conv_model = Sequential()\n",
    "\n",
    "    embed_layer = Embedding(input_dim=kv.vectors.shape[0]+1, \n",
    "                            output_dim=kv.vectors.shape[1], \n",
    "                            weights=[np.vstack((np.zeros((1, kv.vectors.shape[1])),kv.vectors))],             \n",
    "                            input_length=max_sent_len,\n",
    "                            mask_zero=True,\n",
    "                            trainable=False)\n",
    "    blstm_conv_model.add(embed_layer)\n",
    "    blstm_conv_model.add(layers.Bidirectional(\n",
    "            LSTM(\n",
    "                units=32,\n",
    "                activation='tanh',\n",
    "                return_sequences = True,\n",
    "            )) )\n",
    "    blstm_conv_model.add(attention(return_sequences = True))\n",
    "    \n",
    "    for _ in range(conv_layers):\n",
    "\n",
    "        blstm_conv_model.add(keras.layers.Conv1D(filters=units, kernel_size=3, strides=1, padding=\"valid\", activation='relu'))\n",
    "        blstm_conv_model.add(keras.layers.Dropout(rate=dropout))\n",
    "        blstm_conv_model.add(keras.layers.MaxPool1D(pool_size=2, strides=2, padding=\"valid\"))\n",
    "    \n",
    "    blstm_conv_model.add(keras.layers.Flatten())\n",
    "    \n",
    "    blstm_conv_model.add(layers.Dense(units, activation='relu'))\n",
    "    blstm_conv_model.add(layers.Dense(5, activation='softmax'))\n",
    "    \n",
    "        \n",
    "    print(blstm_conv_model.summary())\n",
    "\n",
    "    blstm_conv_model.compile(loss=losses.sparse_categorical_crossentropy, \n",
    "                              optimizer=\"adam\", metrics=['accuracy'])\n",
    "\n",
    "\n",
    "    K.set_value(blstm_conv_model.optimizer.learning_rate, lr)\n",
    "\n",
    "    early = EarlyStopping(monitor=\"val_loss\", patience=10, verbose=1)\n",
    "    redonplat = ReduceLROnPlateau(monitor=\"val_loss\", patience=5, verbose=1)\n",
    "    checkpoint = ModelCheckpoint( filepath=save_name, save_weights_only=True, \n",
    "                                                 monitor='val_accuracy', mode='max', save_best_only=True)\n",
    "    \n",
    "    \n",
    "    callbacks_list = [early, redonplat,checkpoint] \n",
    "\n",
    "    blstm_conv_model.fit(X_train_idx, Y_train, epochs=epochs, batch_size=1000, \n",
    "                                    verbose=1, validation_data=(X_dev_idx,Y_dev), callbacks=callbacks_list )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "118e75d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred = blstm_conv_model.predict(X_test_idx)\n",
    "Y_pred = np.argmax(Y_pred, axis=-1)\n",
    "del blstm_conv_model\n",
    "\n",
    "print(\"Accuracy: \" ,accuracy_score(Y_test, Y_pred))\n",
    "print(\"F1 Score: \" ,f1_score(Y_test, Y_pred, average='weighted') )\n",
    "cm = confusion_matrix(Y_test, Y_pred, normalize = \"true\")\n",
    "cmd = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc87a52c",
   "metadata": {},
   "source": [
    "# Classification with Small Dataset\n",
    "\n",
    "We run the versions of Bidirectional LSTM + fully connected layers on the small 20k dataset as well.\n",
    "\n",
    "## Bidirectional LSTM without line numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f21146ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read data\n",
    "train_data_small = pd.read_pickle (f\"PreprocessedData/train_{suffix}_w2v_small.pkl\")\n",
    "dev_data_small = pd.read_pickle (f\"PreprocessedData/dev_{suffix}_w2v_small.pkl\")\n",
    "test_data_small = pd.read_pickle (f\"PreprocessedData/test_{suffix}_w2v_small.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e727e693",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_idx_small = np.stack(train_data_small[\"idx\"].values)\n",
    "X_train_lines_small = np.reshape(train_data_small[\"line_relative\"].values, (-1, 1))\n",
    "Y_train_small = train_data_small['label'].values\n",
    "\n",
    "X_dev_idx_small = np.stack(dev_data_small[\"idx\"].values)\n",
    "X_dev_lines_small = np.reshape(dev_data_small[\"line_relative\"].values, (-1, 1))\n",
    "Y_dev_small = dev_data_small['label'].values\n",
    "\n",
    "X_test_idx_small = np.stack(test_data_small[\"idx\"].values)\n",
    "X_test_lines_small = np.reshape(test_data_small[\"line_relative\"].values, (-1, 1))\n",
    "Y_test_small = test_data_small['label'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8541af7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load saved model\n",
    "kv_small = KeyedVectors.load_word2vec_format(f\"./TrainedModels/w2v_200_{suffix}_small.bin\", binary=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16dbaebd",
   "metadata": {},
   "outputs": [],
   "source": [
    "dimension=kv_small.vector_size\n",
    "units=32\n",
    "lr = 0.001\n",
    "max_sent_len = len(X_train_idx_small[0])\n",
    "epochs = 40\n",
    "\n",
    "save_name_small = f\"./TrainedModels/biLSTM_noline_{suffix}_small\" + \".h5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0b29ad5",
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.device(device_name):\n",
    "    model_small = Sequential()\n",
    "\n",
    "    embed_layer = Embedding(input_dim=   kv_small.vectors.shape[0]+1, \n",
    "                            output_dim=  kv_small.vectors.shape[1], \n",
    "                            weights=     [np.vstack((np.zeros((1, kv_small.vectors.shape[1])),kv_small.vectors))],             \n",
    "                            input_length=max_sent_len,\n",
    "                            mask_zero=   True,\n",
    "                            trainable=   False)\n",
    "    model_small.add(embed_layer)\n",
    "    model_small.add(layers.Bidirectional(\n",
    "            LSTM(\n",
    "                units=units,\n",
    "                activation='tanh'\n",
    "            )) )\n",
    "\n",
    "    model_small.add(layers.Dense(units, activation='relu'))\n",
    "    model_small.add(layers.Dense(units, activation='relu'))\n",
    "    model_small.add(layers.Dense(5, activation='softmax'))\n",
    "\n",
    "    model_small.compile(loss=losses.sparse_categorical_crossentropy, \n",
    "                              optimizer=\"adam\", metrics=['accuracy'])\n",
    "\n",
    "\n",
    "    K.set_value(model_small.optimizer.learning_rate, lr)\n",
    "\n",
    "    early = EarlyStopping(monitor=\"val_loss\", patience=10, verbose=1)\n",
    "    redonplat = ReduceLROnPlateau(monitor=\"val_loss\", patience=5, verbose=1)\n",
    "    checkpoint = ModelCheckpoint( filepath=save_name_small, save_weights_only=True, \n",
    "                                                 monitor='val_accuracy', mode='max', save_best_only=True)\n",
    "\n",
    "    callbacks_list = [early, redonplat,checkpoint]  \n",
    "\n",
    "    model_small.fit(X_train_idx_small, Y_train_small, epochs=epochs, batch_size=1000, \n",
    "                                    verbose=1, validation_data=(X_dev_idx_small,Y_dev_small), callbacks=callbacks_list )\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f8cdaf5",
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred_small = model_small.predict(X_test_idx_small)\n",
    "Y_pred_small = np.argmax(Y_pred_small, axis=-1)\n",
    "\n",
    "\n",
    "print(\"Accuracy: \" ,accuracy_score(Y_test_small, Y_pred_small))\n",
    "print(\"F1 Score: \" ,f1_score(Y_test_small, Y_pred_small, average='weighted') )\n",
    "cm = confusion_matrix(Y_test_small, Y_pred_small, normalize = \"true\")\n",
    "cmd = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4dd6e14",
   "metadata": {},
   "source": [
    "## Bidirectional LSTM with line numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5055b1f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_name_small = f\"./TrainedModels/biLSTM_line_{suffix}_small\" + \".h5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21d81386",
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.device(device_name):\n",
    "    \n",
    "    input_sequence = Input(shape=(max_sent_len,), name='input_sequence')\n",
    "    input_sent_num = Input(shape=(1,), name='input_sent_num')\n",
    "\n",
    "\n",
    "    # ---- the sequential processing section\n",
    "\n",
    "    x = Embedding(input_dim=  kv_small.vectors.shape[0]+1, \n",
    "                            output_dim=  kv_small.vectors.shape[1], \n",
    "                            weights=     [np.vstack((np.zeros((1, kv_small.vectors.shape[1])),kv_small.vectors))],             \n",
    "                            input_length=max_sent_len,\n",
    "                            mask_zero=   True,\n",
    "                            trainable=   False)(input_sequence)\n",
    "    \n",
    "    x = layers.Bidirectional( LSTM( units=units, activation='tanh'))(x)\n",
    "    combined = layers.concatenate([x, input_sent_num])\n",
    "    \n",
    "    y = layers.Dense(units, activation='relu')(combined)\n",
    "    y = layers.Dense(units, activation='relu')(y)\n",
    "    output = layers.Dense(5, activation='softmax')(y)\n",
    "    \n",
    "    line_model_small = Model(inputs=[input_sequence, input_sent_num], outputs=output)\n",
    "    \n",
    "    print(line_model_small.summary())\n",
    "\n",
    "    line_model_small.compile(loss=losses.sparse_categorical_crossentropy, \n",
    "                              optimizer=\"adam\", metrics=['accuracy'])\n",
    "\n",
    "\n",
    "    K.set_value(line_model_small.optimizer.learning_rate, lr)\n",
    "\n",
    "    early = EarlyStopping(monitor=\"val_loss\", patience=10, verbose=1)\n",
    "    redonplat = ReduceLROnPlateau(monitor=\"val_loss\", patience=5, verbose=1)\n",
    "    checkpoint = ModelCheckpoint( filepath=save_name_small, save_weights_only=True, \n",
    "                                                 monitor='val_accuracy', mode='max', save_best_only=True)\n",
    "\n",
    "    callbacks_list = [early, redonplat,checkpoint] \n",
    "    \n",
    "    line_model_small.fit({'input_sequence': X_train_idx_small, 'input_sent_num': X_train_lines_small}, Y_train_small, \n",
    "              epochs=epochs, batch_size=1000,  verbose=1, \n",
    "              validation_data=({'input_sequence': X_dev_idx_small, 'input_sent_num': X_dev_lines_small},Y_dev_small), \n",
    "              callbacks=callbacks_list)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d22229fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred_small = line_model_small.predict({'input_sequence': X_test_idx_small, 'input_sent_num': X_test_lines_small})\n",
    "Y_pred_small = np.argmax(Y_pred_small, axis=-1)\n",
    "\n",
    "\n",
    "print(\"Accuracy: \" ,accuracy_score(Y_test_small, Y_pred_small))\n",
    "print(\"F1 Score: \" ,f1_score(Y_test_small, Y_pred_small, average='weighted') )\n",
    "cm = confusion_matrix(Y_test_small, Y_pred_small, normalize = \"true\")\n",
    "cmd = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df12baa2",
   "metadata": {},
   "source": [
    "# Classification using Difference of  Averaged Sentence Embeddings\n",
    "\n",
    "Since memory related problems restricted our ability to use neighbouring sentences' word embeddings and aggreagate results, we opted for using averaged sentence embeddings in a way that may carry some sequential information as well. This idea was inspired from the analogy examples explored in the Word2Vec_Embedding_Generation notebook.\n",
    "\n",
    "For each sentence, the averaged sentence embeddings of the previous and next sentences are substracted from its embedding to get an embedding representing semantic change. \n",
    "\n",
    "This model does not converge - averaged embeddings may not be semantically representative/ discriminative enough for their difference to carry enough information."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "292a8bf1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#getting differenced vectors\n",
    "X_dev_diffed, Y_dev_diffed = project2Lib.get_diffed_vecs(dev_data)\n",
    "X_test_diffed, Y_test_diffed  = project2Lib.get_diffed_vecs(test_data)\n",
    "X_train_diffed, Y_train_diffed  = project2Lib.get_diffed_vecs(train_data)\n",
    "\n",
    "#deallocating large datasets to free up memory\n",
    "del X_train_idx\n",
    "del Y_train\n",
    "del X_dev_idx\n",
    "del Y_dev\n",
    "del X_test_idx\n",
    "del Y_test\n",
    "\n",
    "del train_data\n",
    "del dev_data\n",
    "del test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05869d31",
   "metadata": {},
   "outputs": [],
   "source": [
    "dimension=kv.vector_size\n",
    "units=32\n",
    "lr = 0.001\n",
    "epochs = 40\n",
    "dropout=0.2\n",
    "\n",
    "save_name = f\"./TrainedModels/diffed_{suffix}\" + \".h5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bcee490a",
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.device(device_name):\n",
    "    diff_model = Sequential()\n",
    "\n",
    "    diff_model.add(layers.Conv1D(input_shape=(dimension,1), filters=units//2, kernel_size=3, strides=1, padding=\"valid\", activation='relu'))\n",
    "    diff_model.add(layers.Dropout(rate=dropout))\n",
    "    diff_model.add(layers.MaxPool1D(pool_size=2, strides=2, padding=\"valid\"))\n",
    "    diff_model.add(layers.Conv1D(filters=dimension, kernel_size=3, strides=1, padding=\"valid\", activation='relu'))\n",
    "    diff_model.add(layers.Dropout(rate=dropout))\n",
    "    diff_model.add(layers.MaxPool1D(pool_size=2, strides=2, padding=\"valid\"))\n",
    "    diff_model.add(layers.Conv1D(filters=dimension, kernel_size=3, strides=1, padding=\"valid\", activation='relu'))\n",
    "    diff_model.add(layers.Dropout(rate=dropout))\n",
    "    diff_model.add(layers.MaxPool1D(pool_size=2, strides=2, padding=\"valid\"))\n",
    "    diff_model.add(layers.Flatten())\n",
    "    diff_model.add(layers.Dense(32, activation='relu'))\n",
    "    diff_model.add(layers.Dense(5, activation='softmax'))\n",
    "    \n",
    "    print(diff_model.summary())\n",
    "\n",
    "    diff_model.compile(loss=losses.sparse_categorical_crossentropy, \n",
    "                              optimizer=\"adam\", metrics=['accuracy'])\n",
    "\n",
    "\n",
    "    K.set_value(diff_model.optimizer.learning_rate, lr)\n",
    "\n",
    "    early = EarlyStopping(monitor=\"val_loss\", patience=10, verbose=1)\n",
    "    redonplat = ReduceLROnPlateau(monitor=\"val_loss\", patience=5, verbose=1)\n",
    "    checkpoint = ModelCheckpoint( filepath=save_name, save_weights_only=True, \n",
    "                                                 monitor='val_accuracy', mode='max', save_best_only=True)\n",
    "\n",
    "    callbacks_list = [early, redonplat,checkpoint]   \n",
    "\n",
    "    diff_model.fit(X_train_diffed.reshape(-1,dimension,1), Y_train_diffed, epochs=epochs, batch_size=1000, \n",
    "                                    verbose=1, validation_data=(X_dev_diffed.reshape(-1,dimension,1),Y_dev_diffed), callbacks=callbacks_list )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d050cad9",
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred = diff_model.predict(X_test_diffed.reshape(-1,dimension,1))\n",
    "Y_pred = np.argmax(Y_pred, axis=-1)\n",
    "\n",
    "\n",
    "print(\"Accuracy: \" ,accuracy_score(Y_test, Y_pred))\n",
    "print(\"F1 Score: \" ,f1_score(Y_test, Y_pred, average='weighted') )\n",
    "cm = confusion_matrix(Y_test, Y_pred, normalize = \"true\")\n",
    "cmd = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b3409c54",
   "metadata": {},
   "source": [
    "# CNN on Concatenated Embeddings - WAY TOO SLOW\n",
    "\n",
    "The idea behind this model is to find an alterantive to sequential processing. Word embeddings are concatenated and the padded sections of the sequence are masked in the matrix. The embedded sentence is therefore in the form of a 2D matrix that will not be treated sequentially but given to convolutional layers. \n",
    "\n",
    "- Model is too slow to run effectively, its design must have issues causing an unforeseen computational bottleneck"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e95a16a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "dropout=0\n",
    "units=32 # = filters\n",
    "max_sent_len = len(X_train_idx[0])\n",
    "lr = 0.001\n",
    "epochs = 40\n",
    "dimension=kv.vector_size\n",
    "\n",
    "save_name = f\"./TrainedModels/conv2d_{suffix}\" + \".h5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "963d25b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.device(device_name):\n",
    "    cnn_model = Sequential()\n",
    "\n",
    "    embed_layer = Embedding(input_dim=kv.vectors.shape[0]+1, \n",
    "                            output_dim=kv.vectors.shape[1], \n",
    "                            weights=[np.vstack((np.zeros((1, kv.vectors.shape[1])),kv.vectors))],             \n",
    "                            input_length=max_sent_len,\n",
    "                            mask_zero=True,\n",
    "                            trainable=False)\n",
    "    cnn_model.add(embed_layer)\n",
    "\n",
    "    cnn_model.add(layers.Reshape((max_sent_len,dimension,1),input_shape=(max_sent_len,dimension)))\n",
    "\n",
    "    cnn_model.add(layers.Conv2D(8, kernel_size=5,    activation=\"relu\"))\n",
    "    #cnn_model.add(layers.Dropout(rate=dropout))\n",
    "    cnn_model.add(layers.MaxPool2D(pool_size=2,  ))\n",
    "    cnn_model.add(layers.Conv2D(8, kernel_size=3,   activation=\"relu\"))\n",
    "    cnn_model.add(layers.Dropout(rate=dropout))\n",
    "    cnn_model.add(layers.MaxPool2D(pool_size=2, ))\n",
    "    cnn_model.add(layers.Conv2D(16, kernel_size=3,  activation=\"relu\"))\n",
    "    #cnn_model.add(layers.Dropout(rate=dropout))\n",
    "    cnn_model.add(layers.MaxPool2D(pool_size=2, ))\n",
    "    cnn_model.add(layers.Conv2D(16, kernel_size=3,   activation=\"relu\"))\n",
    "    cnn_model.add(layers.Dropout(rate=dropout))\n",
    "    cnn_model.add(layers.MaxPool2D(pool_size=2, ))\n",
    "    cnn_model.add(layers.Conv2D(16, kernel_size=3,   activation=\"relu\"))\n",
    "    cnn_model.add(layers.Dropout(rate=dropout))\n",
    "    cnn_model.add(layers.MaxPool2D(pool_size=2, ))\n",
    "\n",
    "    \n",
    "    cnn_model.add(keras.layers.Flatten())\n",
    "    \n",
    "    cnn_model.add(layers.Dense(units, activation='relu'))\n",
    "    cnn_model.add(layers.Dense(5, activation='softmax'))\n",
    "    \n",
    "        \n",
    "    print(cnn_model.summary())\n",
    "\n",
    "    cnn_model.compile(loss=losses.sparse_categorical_crossentropy, \n",
    "                              optimizer=\"adam\", metrics=['accuracy'])\n",
    "\n",
    "\n",
    "    K.set_value(cnn_model.optimizer.learning_rate, lr)\n",
    "\n",
    "    early = EarlyStopping(monitor=\"val_loss\", patience=10, verbose=1)\n",
    "    redonplat = ReduceLROnPlateau(monitor=\"val_loss\", patience=5, verbose=1)\n",
    "    checkpoint = ModelCheckpoint( filepath=save_name, save_weights_only=True, \n",
    "                                                 monitor='val_accuracy', mode='max', save_best_only=True)\n",
    "    \n",
    "    \n",
    "    callbacks_list = [early, redonplat,checkpoint] \n",
    "\n",
    "    cnn_model.fit(X_train_idx, Y_train, epochs=epochs, batch_size=500, \n",
    "                                    verbose=1, validation_data=(X_dev_idx,Y_dev), callbacks=callbacks_list )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2488ec5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred = cnn_model.predict(X_test_idx)\n",
    "Y_pred = np.argmax(Y_pred, axis=-1)\n",
    "\n",
    "\n",
    "print(\"Accuracy: \" ,accuracy_score(Y_test, Y_pred))\n",
    "print(\"F1 Score: \" ,f1_score(Y_test, Y_pred, average='weighted') )\n",
    "cm = confusion_matrix(Y_test, Y_pred, normalize = \"true\")\n",
    "cmd = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
    "cmd.plot()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}