Word2Vec_Averaged_Embedding_Approach.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0d5d4901",
   "metadata": {},
   "source": [
    "# Task 2: Word2Vec - Average Sentence Embedding Approach to Sentence Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5581275",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle, csv\n",
    "import project2Lib\n",
    "\n",
    "## for plotting data distribution\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "#for text pre-processing\n",
    "import re, string\n",
    "import nltk\n",
    "from gensim.models import Word2Vec\n",
    "import gensim.models\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "import xgboost as xgb\n",
    "\n",
    "#For Peformance Metrics\n",
    "from sklearn.metrics import classification_report, f1_score, accuracy_score, plot_confusion_matrix, confusion_matrix, ConfusionMatrixDisplay\n",
    "\n",
    "\n",
    "from tensorflow.keras import models, layers, preprocessing, Sequential\n",
    "from tensorflow.keras import backend as K\n",
    "from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint\n",
    "\n",
    "\n",
    "np.random.seed(1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b67325b5",
   "metadata": {},
   "source": [
    "## Loading preprocessed data\n",
    "\n",
    "### Choosing one of the preprocessing options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26f8c9d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "suffix = \"\"\n",
    "mode = 1\n",
    "\n",
    "if   mode==0:\n",
    "    suffix = \"lemmatization_noph\"\n",
    "    \n",
    "elif mode==1:\n",
    "    suffix = \"lemmatization\"\n",
    "    \n",
    "elif mode==2:\n",
    "    suffix = \"_noph\"\n",
    "\n",
    "elif mode==3:\n",
    "    suffix = \"_\"\n",
    "    \n",
    "elif mode==4:\n",
    "    suffix = \"stemming_noph\"\n",
    "    \n",
    "elif mode==5:\n",
    "    suffix = \"stemming\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9f63229",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read data\n",
    "train_data = pd.read_pickle (f\"PreprocessedData/train_{suffix}_w2v.pkl\")\n",
    "dev_data = pd.read_pickle (f\"PreprocessedData/dev_{suffix}_w2v.pkl\")\n",
    "test_data = pd.read_pickle (f\"PreprocessedData/test_{suffix}_w2v.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "59be032f",
   "metadata": {},
   "source": [
    "## To prepare data:\n",
    "\n",
    "the X_train_lines data is extracted for models that use the relative line number as an auxiliary input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65060540",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_vec = np.stack(train_data[\"avg_vectors\"].values)\n",
    "X_train_vec_line = np.concatenate( ( np.reshape(train_data[\"line_relative\"].values, (-1, 1)) ,X_train_vec) ,axis=1) \n",
    "Y_train = train_data['label'].values\n",
    "\n",
    "X_dev_vec = np.stack(dev_data[\"avg_vectors\"].values)\n",
    "X_dev_vec_line = np.concatenate( ( np.reshape(dev_data[\"line_relative\"].values, (-1, 1)) ,X_dev_vec) ,axis=1) \n",
    "Y_dev = dev_data['label'].values\n",
    "\n",
    "X_test_vec = np.stack(test_data[\"avg_vectors\"].values)\n",
    "X_test_vec_line = np.concatenate( ( np.reshape(test_data[\"line_relative\"].values, (-1, 1)) ,X_test_vec) ,axis=1) \n",
    "Y_test = test_data['label'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "59b70933",
   "metadata": {},
   "source": [
    "# Classifiers Based on Average Sentence Vectors"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e246430c",
   "metadata": {},
   "source": [
    "## Logistic Regression\n",
    "\n",
    "### Without Line Numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7026f91a",
   "metadata": {},
   "outputs": [],
   "source": [
    "logisticRegr1 = LogisticRegression(verbose=1, n_jobs=-1)\n",
    "logisticRegr1.fit(X_train_vec, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b99e3e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_name = f'./TrainedModels/logreg_w2v_noline_{suffix}.sav'\n",
    "pickle.dump(logisticRegr1, open(save_name, 'wb'))\n",
    "\n",
    "Y_pred = logisticRegr1.predict(X_test_vec)\n",
    "\n",
    "print(\"Accuracy: \" ,accuracy_score(Y_test, Y_pred))\n",
    "print(\"F1 Score: \" ,f1_score(Y_test, Y_pred, average='weighted') )\n",
    "cm = confusion_matrix(Y_test, Y_pred, normalize = \"true\")\n",
    "cmd = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "381de017",
   "metadata": {},
   "source": [
    "### With Line Numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d25bb3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "logisticRegr2 = LogisticRegression(verbose=1, n_jobs=-1)\n",
    "logisticRegr2.fit(X_train_vec_line, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60b0a32e",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_name = f'./TrainedModels/logreg_w2v_line_{suffix}.sav'\n",
    "pickle.dump(logisticRegr1, open(save_name, 'wb'))\n",
    "\n",
    "Y_pred = logisticRegr2.predict(X_test_vec_line)\n",
    "\n",
    "print(\"Accuracy: \" ,accuracy_score(Y_test, Y_pred))\n",
    "print(\"F1 Score: \" ,f1_score(Y_test, Y_pred, average='weighted') )\n",
    "cm = confusion_matrix(Y_test, Y_pred, normalize = \"true\")\n",
    "cmd = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a49ea2e",
   "metadata": {},
   "source": [
    "## Random Forest\n",
    "\n",
    "### Without Line Numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6b29f77",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = RandomForestClassifier(verbose=1, n_jobs=8)\n",
    "rf_model = rf.fit(X_train_vec, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3b7ea67",
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred = rf_model.predict(X_test_vec)\n",
    "\n",
    "print(\"Accuracy: \" ,accuracy_score(Y_test, Y_pred))\n",
    "print(\"F1 Score: \" ,f1_score(Y_test, Y_pred, average='weighted') )\n",
    "cm = confusion_matrix(Y_test, Y_pred, normalize = \"true\")\n",
    "cmd = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5819e684",
   "metadata": {},
   "source": [
    "### With Line Numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e4d84ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = RandomForestClassifier(verbose=1, n_jobs=-1)\n",
    "rf_model2 = rf.fit(X_train_vec_line, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff0fcaa0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "Y_pred = rf_model2.predict(X_test_vec_line)\n",
    "\n",
    "print(\"Accuracy: \" ,accuracy_score(Y_test, Y_pred))\n",
    "print(\"F1 Score: \" ,f1_score(Y_test, Y_pred, average='weighted') )\n",
    "cm = confusion_matrix(Y_test, Y_pred, normalize = \"true\")\n",
    "cmd = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e8d46c5",
   "metadata": {},
   "source": [
    "## XGBoost\n",
    "\n",
    "### Without Line Numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc09f4ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "xg_class = xgb.XGBClassifier()\n",
    "xg_model = xg_class.fit(X_train_vec, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48789ce4",
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred = xg_model.predict(X_test_vec)\n",
    "\n",
    "print(\"Accuracy: \" ,accuracy_score(Y_test, Y_pred))\n",
    "print(\"F1 Score: \" ,f1_score(Y_test, Y_pred, average='weighted') )\n",
    "cm = confusion_matrix(Y_test, Y_pred, normalize = \"true\")\n",
    "cmd = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db505b80",
   "metadata": {},
   "source": [
    "### With Line Numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b4ea670",
   "metadata": {},
   "outputs": [],
   "source": [
    "xg_class = xgb.XGBClassifier()\n",
    "xg_model = xg_class.fit(X_train_vec_line, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e8b2beb",
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_pred = xg_model.predict(X_test_vec_line)\n",
    "\n",
    "print(\"Accuracy: \" ,accuracy_score(Y_test, Y_pred))\n",
    "print(\"F1 Score: \" ,f1_score(Y_test, Y_pred, average='weighted') )\n",
    "cm = confusion_matrix(Y_test, Y_pred, normalize = \"true\")\n",
    "cmd = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d88772e",
   "metadata": {},
   "source": [
    "# Classification Using Small Dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adc9396c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read data\n",
    "train_data_small = pd.read_pickle (f\"PreprocessedData/train_{suffix}_w2v_small.pkl\")\n",
    "dev_data_small = pd.read_pickle (f\"PreprocessedData/dev_{suffix}_w2v_small.pkl\")\n",
    "test_data_small = pd.read_pickle (f\"PreprocessedData/test_{suffix}_w2v_small.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "baae069c",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_small[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ddf0c0e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_vec_small = np.stack(train_data_small[\"avg_vectors\"].values)\n",
    "#X_train_vec_line_small = np.concatenate( ( np.reshape(train_data_small[\"line_relative\"].values, (-1, 1)) ,X_train_vec_small) ,axis=1) \n",
    "Y_train_small = train_data_small['label'].values\n",
    "\n",
    "X_dev_vec_small = np.stack(dev_data_small[\"avg_vectors\"].values)\n",
    "#X_dev_vec_line_small = np.concatenate( ( np.reshape(dev_data_small[\"line_relative\"].values, (-1, 1)) ,X_dev_vec_small) ,axis=1) \n",
    "Y_dev_small = dev_data_small['label'].values\n",
    "\n",
    "X_test_vec_small = np.stack(test_data_small[\"avg_vectors\"].values)\n",
    "#X_test_vec_line_small = np.concatenate( ( np.reshape(test_data_small[\"line_relative\"].values, (-1, 1)) ,X_test_vec_small) ,axis=1) \n",
    "Y_test_small = test_data_small['label'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da97c6ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "logisticRegr1_small = LogisticRegression(verbose=1, n_jobs=-1)\n",
    "logisticRegr1_small.fit(X_train_vec_small, Y_train_small)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7c15085",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_name = f'./TrainedModels/logreg_w2v_noline_{suffix}_small.sav'\n",
    "pickle.dump(logisticRegr1_small, open(save_name, 'wb'))\n",
    "\n",
    "Y_pred_small = logisticRegr1_small.predict(X_test_vec_small)\n",
    "\n",
    "print(\"Accuracy: \" ,accuracy_score(Y_test_small, Y_pred_small))\n",
    "print(\"F1 Score: \" ,f1_score(Y_test_small, Y_pred_small, average='weighted') )\n",
    "cm = confusion_matrix(Y_test_small, Y_pred_small, normalize = \"true\")\n",
    "cmd = ConfusionMatrixDisplay(confusion_matrix=cm)\n",
    "cmd.plot()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}