TFIDF_BaselineModel.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "461427ce",
   "metadata": {},
   "source": [
    "# Task 1: Baseline Model\n",
    "\n",
    "In this notebook we develop and evaluate a baseline model for classifying sentences of the PubMed RCT dataset. First we test different models and parameters of the tf-idf embedding. Then for our best performing model we test if balancing the classes and if .. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "932c885b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import project2Lib \n",
    "from pathlib import Path\n",
    "\n",
    "# bag of words\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "#for model-building\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.metrics import classification_report, f1_score, accuracy_score, plot_confusion_matrix, confusion_matrix, ConfusionMatrixDisplay\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "from keras import activations\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "import tensorflow as tf\n",
    "\n",
    "from scipy.sparse import hstack, csr_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77b0640a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_nicely(n, df):\n",
    "    \n",
    "    for index, row in df.iterrows():\n",
    "        print(\"----------------------\")\n",
    "        r = []\n",
    "        for i in range(len(df.columns)):\n",
    "            r.append(row[i])\n",
    "        print(r)\n",
    "        \n",
    "        if(index >=n):\n",
    "            break\n",
    "def read_preprocessed_data(option):\n",
    "    datasets = [\"train\", \"dev\", \"test\"]\n",
    "    filepaths = []\n",
    "    for i in datasets: \n",
    "        filepaths.append(f\"./PreprocessedData/{i}_{option}.csv\")\n",
    "\n",
    "    train_data = pd.read_csv(filepaths[0], index_col = 0)\n",
    "    train_data = train_data.fillna('')\n",
    "    dev_data = pd.read_csv(filepaths[1], index_col = 0)\n",
    "    dev_data = dev_data.fillna('')\n",
    "    test_data = pd.read_csv(filepaths[2], index_col = 0)\n",
    "    test_data = test_data.fillna('')\n",
    "    \n",
    "    return train_data, dev_data, test_data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81e5f2fe",
   "metadata": {},
   "source": [
    "## Evaluate different models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "edf0ccdd",
   "metadata": {},
   "source": [
    "First, we test different (simple) models directly on tfidf without any further preprocessing. We optimize some parameters regarding preprocessing. More specifically, we vary tfidf options (varying number of maximal features and ngram_range),  balancing the classes vs not balancing and using the relative line number of a sentence or not."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f578dcd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = {\n",
    "    \"preprocessed_options\" : [\"lemmatization_noph.csv\", \"lemmatization.csv\"],\n",
    "    \"max_features\": [5000,15000,30000,50000, None], \n",
    "    \"ngram_range\" : [(1,1), (1,2)],\n",
    "    \"classifier\": [MultinomialNB(), LogisticRegression(random_state=321, max_iter = 500), RandomForestClassifier(max_depth=40, random_state=123)]\n",
    "}\n",
    "\n",
    "results = {}\n",
    "\n",
    "for preprocessed_option in parameters['preprocessed_options']:\n",
    "    # read preprocessed dataset\n",
    "    datasets = [\"train\", \"dev\", \"test\"]\n",
    "    filepaths = []\n",
    "    for i in datasets: \n",
    "        filepaths.append(f\"./PreprocessedData/{i}_{preprocessed_option}\")\n",
    "    \n",
    "    train_data = pd.read_csv(filepaths[0], index_col = 0)\n",
    "    train_data = train_data.fillna('')\n",
    "    dev_data = pd.read_csv(filepaths[1], index_col = 0)\n",
    "    dev_data = dev_data.fillna('')\n",
    "    test_data = pd.read_csv(filepaths[2], index_col = 0)\n",
    "    test_data = test_data.fillna('')\n",
    "\n",
    "    Y_train = train_data[\"label\"].to_numpy()\n",
    "    Y_dev = dev_data[\"label\"].to_numpy()\n",
    "    Y_test = test_data[\"label\"].to_numpy()\n",
    "    X_train = train_data[\"sentence\"].values\n",
    "    X_dev = dev_data[\"sentence\"].values\n",
    "    X_test = test_data[\"sentence\"].values\n",
    "    \n",
    "    \n",
    "    for max_feature in parameters[\"max_features\"]:\n",
    "        for ngram in parameters[\"ngram_range\"]:\n",
    "\n",
    "            tfidf_vectorizer = TfidfVectorizer(use_idf=True, max_features=max_feature, ngram_range=ngram)\n",
    "            X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train)\n",
    "            X_dev_vectors_tfidf = tfidf_vectorizer.transform(X_dev)\n",
    "\n",
    "            for classifier in parameters[\"classifier\"]:\n",
    "                clf = classifier\n",
    "                clf.fit(X_train_vectors_tfidf, Y_train)\n",
    "                y_hat_dev = clf.predict(X_dev_vectors_tfidf)\n",
    "                f1 = f1_score(Y_dev, y_hat_dev,average=\"weighted\")\n",
    "\n",
    "                key = f\"{preprocessed_option}_max_feature{max_feature}_ngram{ngram}_\" + str(classifier)\n",
    "\n",
    "                results[key] = f1\n",
    "                print(f\"{key}____{f1}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76050f73",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read preprocessed data\n",
    "datasets = [\"train\", \"dev\", \"test\"]\n",
    "filepaths = []\n",
    "for i in datasets: \n",
    "    filepaths.append(f\"./PreprocessedData/{i}_lemmatization_noph.csv\")\n",
    "    \n",
    "train_data = pd.read_csv(filepaths[0], index_col = 0)\n",
    "train_data = train_data.fillna('')\n",
    "dev_data = pd.read_csv(filepaths[1], index_col = 0)\n",
    "dev_data = dev_data.fillna('')\n",
    "test_data = pd.read_csv(filepaths[2], index_col = 0)\n",
    "test_data = test_data.fillna('')\n",
    "\n",
    "Y_train = train_data[\"label\"].to_numpy()\n",
    "Y_dev = dev_data[\"label\"].to_numpy()\n",
    "Y_test = test_data[\"label\"].to_numpy()\n",
    "X_train = train_data[\"preprocess\"].values\n",
    "X_dev = dev_data[\"preprocess\"].values\n",
    "X_test = test_data[\"preprocess\"].values\n",
    "\n",
    "\n",
    "# train tfidf vectorizer\n",
    "tfidf_vectorizer = TfidfVectorizer(use_idf=True, max_features=50000, ngram_range=(1,2))\n",
    "X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train)\n",
    "X_dev_vectors_tfidf = tfidf_vectorizer.transform(X_dev)\n",
    "\n",
    "clf = LogisticRegression(random_state=321, max_iter = 500)\n",
    "clf.fit(X_train_vectors_tfidf, Y_train)\n",
    "y_hat_dev = clf.predict(X_dev_vectors_tfidf)\n",
    "f1 = f1_score(Y_dev, y_hat_dev,average=\"weighted\")\n",
    "print(f\"F1 score: {f1}\")\n",
    "cm = confusion_matrix(Y_dev, y_hat_dev)\n",
    "cmd = ConfusionMatrixDisplay(cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "444a56f1",
   "metadata": {},
   "source": [
    "Based on this grid-search we conclude that **logistic regression** is the best performing model among the tested model. \n",
    "\n",
    "The differences between removing the placeholder @ for numbers and keeping it is very small with the preprocessed data without the placeholder performing slightly better. Using a tf-idf with all features and 1-grams and 2-grams performs best and has (weigthed) **f1-score = 0.80288** when evaluating the trained model on the dev-dataset. In the corresponding tf-idf embedding, there are 5289618 features. It is worth mentioning that, using logistic regression and 5000 featrues obtains an **f1-score = 0.79512**. Because of the small performance gap, but much larger complexity gap in the data, we continue working with the max_features=5000 option.\n",
    "\n",
    "\n",
    "-> F1 score keep ph: 0.7958474671704888\n",
    "\n",
    "### Complex Model: XGBoost\n",
    "Additionally, we further test XGBoost on a smaller subset of tfidf features with max_features = 300 (due to computational limits). However they perform worse than simple models (f1 = 0.6471) and model training takes significantly more time. Therefore we focus on logistic regression for the baseline model. The code for reproducing the results is in the following cell. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25ba74ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "\n",
    "tfidf_vectorizer = TfidfVectorizer(use_idf=True, max_features=300, ngram_range=(1,2))\n",
    "X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train)\n",
    "X_dev_vectors_tfidf = tfidf_vectorizer.transform(X_dev)\n",
    "X_test_vectors_tfidf = tfidf_vectorizer.transform(X_test)\n",
    "\n",
    "model = XGBClassifier(max_depth=10,random_state=42, n_estimators = 100)\n",
    "model.fit(X_train_vectors_tfidf, Y_train)\n",
    "\n",
    "y_hat_dev = model.predict(X_dev_vectors_tfidf)\n",
    "f1 = f1_score(Y_dev, y_hat_dev,average=\"weighted\")\n",
    "print(f\"F1 score: {f1}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b8fa820",
   "metadata": {},
   "source": [
    "### Solving class imbalance\n",
    "\n",
    "The training data is inbalanced. The largest class \"RESULTS\" has 786,527 observations while the smallest class \"OBJECTIVE\" has only 191,408 observations. Therefore we try over- and undersampling to balance the data and compare the performance compared to the original data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ebb3655d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read preprocessed data \n",
    "train_data, dev_data, test_data = read_preprocessed_data(\"lemmatization_noph\")\n",
    "\n",
    "# resample training data so classes are more balanced\n",
    "class_mapping={\n",
    "    \"0\" :(350000,True),\n",
    "    \"1\" :(350000,True),\n",
    "    \"2\" :(550000,False),\n",
    "    \"3\" :(550000,False),\n",
    "    \"4\" :(339714,False),\n",
    "}\n",
    "train_data_balanced = project2Lib.balance_data(train_data, class_mapping)\n",
    "\n",
    "# extract X and Y from dataframe\n",
    "Y_train = train_data_balanced[\"label\"].to_numpy()\n",
    "Y_dev = dev_data[\"label\"].to_numpy()\n",
    "Y_test = test_data[\"label\"].to_numpy()\n",
    "X_train = train_data_balanced[\"preprocess\"].values\n",
    "X_dev = dev_data[\"preprocess\"].values\n",
    "X_test = test_data[\"preprocess\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e0a39f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# train tfidf vectorizer\n",
    "tfidf_vectorizer = TfidfVectorizer(use_idf=True, max_features=50000, ngram_range=(1,2))\n",
    "X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train)\n",
    "X_dev_vectors_tfidf = tfidf_vectorizer.transform(X_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b35d023e",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = LogisticRegression(random_state=321, max_iter = 500)\n",
    "\n",
    "clf.fit(X_train_vectors_tfidf, Y_train)\n",
    "y_hat_dev = clf.predict(X_dev_vectors_tfidf)\n",
    "f1 = f1_score(Y_dev, y_hat_dev,average=\"weighted\")\n",
    "print(f\"F1 score: {f1}\")\n",
    "cm = confusion_matrix(Y_dev, y_hat_dev)\n",
    "cmd = ConfusionMatrixDisplay(cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c584dcf2",
   "metadata": {},
   "source": [
    "The f1 score = 0.7935124908036201 is slightly worse than without balancing the data. While balancing results in the minory classes to have more true positives (from 1400 to 1564/ from 1413 to 1559) the model is slightly worse for the majority classes since they have less true positives (from 8573 to 8440/ from 8567 to 8291/ from 3135 to 3067). We conclude that wether balancing the data is useful depends on the use case of the classification costs of misclassification for each class. Since the performance metric for this project is f1 score, we will proceed without balancing the data because this lead to slightly better results."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "218ddb1e",
   "metadata": {},
   "source": [
    "### Different preprocessing options\n",
    "We further compare the performance of our model (logistic regression) on the dataset using lemmatization with preprocessed data set using stemming and no lemmatization/stemming.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d64a4b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read preprocessed data with stemming\n",
    "train_data, dev_data, test_data = read_preprocessed_data(\"stemming_noph\")\n",
    "\n",
    "# extract X and Y from dataframe\n",
    "Y_train = train_data[\"label\"].to_numpy()\n",
    "Y_dev = dev_data[\"label\"].to_numpy()\n",
    "Y_test = test_data[\"label\"].to_numpy()\n",
    "X_train = train_data[\"preprocess\"].values\n",
    "X_dev = dev_data[\"preprocess\"].values\n",
    "X_test = test_data[\"preprocess\"].values\n",
    "\n",
    "# train tfidf vectorizer\n",
    "tfidf_vectorizer = TfidfVectorizer(use_idf=True, max_features=50000, ngram_range=(1,2))\n",
    "X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train)\n",
    "X_dev_vectors_tfidf = tfidf_vectorizer.transform(X_dev)\n",
    "\n",
    "clf = LogisticRegression(random_state=321, max_iter = 500)\n",
    "clf.fit(X_train_vectors_tfidf, Y_train)\n",
    "y_hat_dev = clf.predict(X_dev_vectors_tfidf)\n",
    "f1 = f1_score(Y_dev, y_hat_dev,average=\"weighted\")\n",
    "print(f\"F1 score: {f1}\")\n",
    "cm = confusion_matrix(Y_dev, y_hat_dev)\n",
    "cmd = ConfusionMatrixDisplay(cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c81d8689",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read preprocessed data without no stemming and no lemmatization\n",
    "train_data, dev_data, test_data = read_preprocessed_data(\"_noph\")\n",
    "\n",
    "# extract X and Y from dataframe\n",
    "Y_train = train_data[\"label\"].to_numpy()\n",
    "Y_dev = dev_data[\"label\"].to_numpy()\n",
    "Y_test = test_data[\"label\"].to_numpy()\n",
    "X_train = train_data[\"preprocess\"].values\n",
    "X_dev = dev_data[\"preprocess\"].values\n",
    "X_test = test_data[\"preprocess\"].values\n",
    "\n",
    "# train tfidf vectorizer\n",
    "tfidf_vectorizer = TfidfVectorizer(use_idf=True, max_features=50000, ngram_range=(1,2))\n",
    "X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train)\n",
    "X_dev_vectors_tfidf = tfidf_vectorizer.transform(X_dev)\n",
    "\n",
    "clf = LogisticRegression(random_state=321, max_iter = 500)\n",
    "clf.fit(X_train_vectors_tfidf, Y_train)\n",
    "y_hat_dev = clf.predict(X_dev_vectors_tfidf)\n",
    "f1 = f1_score(Y_dev, y_hat_dev,average=\"weighted\")\n",
    "print(f\"F1 score: {f1}\")\n",
    "cm = confusion_matrix(Y_dev, y_hat_dev)\n",
    "cmd = ConfusionMatrixDisplay(cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15e5e585",
   "metadata": {},
   "outputs": [],
   "source": [
    "# compare to no preprocessing \n",
    "# extract X and Y from dataframe\n",
    "Y_train = train_data[\"label\"].to_numpy()\n",
    "Y_dev = dev_data[\"label\"].to_numpy()\n",
    "Y_test = test_data[\"label\"].to_numpy()\n",
    "X_train = train_data[\"sentence\"].values\n",
    "X_dev = dev_data[\"sentence\"].values\n",
    "X_test = test_data[\"sentence\"].values\n",
    "\n",
    "# train tfidf vectorizer\n",
    "tfidf_vectorizer = TfidfVectorizer(use_idf=True, max_features=50000, ngram_range=(1,2))\n",
    "X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train)\n",
    "X_dev_vectors_tfidf = tfidf_vectorizer.transform(X_dev)\n",
    "\n",
    "clf = LogisticRegression(random_state=321, max_iter = 500)\n",
    "clf.fit(X_train_vectors_tfidf, Y_train)\n",
    "y_hat_dev = clf.predict(X_dev_vectors_tfidf)\n",
    "f1 = f1_score(Y_dev, y_hat_dev,average=\"weighted\")\n",
    "print(f\"F1 score: {f1}\")\n",
    "cm = confusion_matrix(Y_dev, y_hat_dev)\n",
    "cmd = ConfusionMatrixDisplay(cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e7a1992",
   "metadata": {},
   "source": [
    "Stemming: 0.791065835354926\n",
    "\n",
    "NO Stemming/ No Lemmaitzation: 0.8087533127068136\n",
    "\n",
    "No preprocessing at all: F1 score: 0.8452712703418799\n",
    "\n",
    "\n",
    "-> Suprising, but because of this we train final models on data with no preprocessing. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92fa4c14",
   "metadata": {},
   "source": [
    "## Train the final model\n",
    "\n",
    "Note: Sine we want to use the baselinemodel in knowledge distillation, we use keras from now on, because its API is better compatible for that."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5522c70",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read preprocessed dataset\n",
    "datasets = [\"train\", \"dev\", \"test\"]\n",
    "filepaths = []\n",
    "for i in datasets: \n",
    "    filepaths.append(f\"./PreprocessedData/{i}_lemmatization_noph.csv\")\n",
    "    \n",
    "train_data = pd.read_csv(filepaths[0], index_col = 0)\n",
    "train_data = train_data.fillna('')\n",
    "dev_data = pd.read_csv(filepaths[1], index_col = 0)\n",
    "dev_data = dev_data.fillna('')\n",
    "test_data = pd.read_csv(filepaths[2], index_col = 0)\n",
    "test_data = test_data.fillna('')\n",
    "\n",
    "Y_train = train_data[\"label\"].to_numpy()\n",
    "Y_dev = dev_data[\"label\"].to_numpy()\n",
    "Y_test = test_data[\"label\"].to_numpy()\n",
    "X_train = train_data[\"sentence\"].values\n",
    "X_dev = dev_data[\"sentence\"].values\n",
    "X_test = test_data[\"sentence\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb443dc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cca710c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_vectorizer = TfidfVectorizer(use_idf=True, max_features=50000, ngram_range=(1,2))\n",
    "X_train_vectors_tfidf = tfidf_vectorizer.fit_transform(X_train)\n",
    "X_dev_vectors_tfidf = tfidf_vectorizer.transform(X_dev)\n",
    "X_test_vectors_tfidf =  tfidf_vectorizer.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "002d992a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# sort indices of sparse matrix otherwise keras gives an error\n",
    "X_train_vectors_tfidf.sort_indices()\n",
    "X_dev_vectors_tfidf.sort_indices()\n",
    "X_test_vectors_tfidf.sort_indices()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1b53b08",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_logistic_regression(numb_classes=5, numb_features=50000):\n",
    "    number_of_classes = numb_classes\n",
    "    number_of_features = numb_features\n",
    "    lr = Sequential()\n",
    "    lr.add(Dense(number_of_classes,activation = activations.softmax,input_dim = number_of_features))\n",
    "    lr.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['acc'])\n",
    "    return lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4fefa2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train logistic regression model\n",
    "lr = get_logistic_regression()\n",
    "file_path = f\"./TrainedModels/tfidf_lr.h5\"\n",
    "checkpoint = ModelCheckpoint(file_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')    \n",
    "early = EarlyStopping(monitor='val_acc', patience=3)\n",
    "callbacks_list = [checkpoint, early]\n",
    "lr.fit(X_train_vectors_tfidf, Y_train, epochs=40, validation_data=(X_dev_vectors_tfidf, Y_dev), batch_size=1024, verbose=2, callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75bf58c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load weights and make final prediction on test set\n",
    "lr2 = get_logistic_regression()\n",
    "lr2.load_weights(\"./TrainedModels/tfidf_lr.h5\")\n",
    "\n",
    "y_hat_test = lr2.predict(X_test_vectors_tfidf)\n",
    "y_hat_test = np.argmax(y_hat_test, axis=-1)\n",
    "f1 = f1_score(Y_test, y_hat_test,average=\"weighted\")\n",
    "acc = accuracy_score(Y_test, y_hat_test)\n",
    "print(f\"F1 score: {f1}\")\n",
    "print(f\"Acc score: {acc}\")\n",
    "\n",
    "\n",
    "cm=confusion_matrix(Y_test,y_hat_test,normalize=\"true\")\n",
    "cmd = ConfusionMatrixDisplay(cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47f139bd",
   "metadata": {},
   "source": [
    "Log Regression: \n",
    "\n",
    "F1 score: 0.8462603682499152\n",
    "\n",
    "Acc score: 0.8482012680975146"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ffb6a073",
   "metadata": {},
   "outputs": [],
   "source": [
    "# analyze the weights\n",
    "\n",
    "def print_min_and_max(weights, mapping, num_features):\n",
    "    max_vals = (-weights).argsort()[:num_features]\n",
    "    min_vals = (weights).argsort()[:num_features]\n",
    "\n",
    "    print(\"-----------Max-Values------------\")\n",
    "    max_words = []\n",
    "    for i in max_vals:\n",
    "        if(i>=50000):\n",
    "            continue\n",
    "        t = i+1\n",
    "        max_words.append(mapping[i])\n",
    "    print(max_words)\n",
    "    min_words = []\n",
    "    print(\"-----------Min-Values------------\")\n",
    "    for i in min_vals:\n",
    "        if(i>=50000):\n",
    "            continue\n",
    "        min_words.append(mapping[i])\n",
    "    print(min_words)\n",
    "    return max_words, min_words\n",
    "    \n",
    "weights = lr2.get_weights()\n",
    "weights = weights[0]\n",
    "mapping = tfidf_vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d86fafa8",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# for class 0 ()\n",
    "n = 50\n",
    "max_0, min_0 = print_min_and_max(weights[:,0], mapping, n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71e8ca47",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "max_1, min_1 = print_min_and_max(weights[:,1], mapping, n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e63f3587",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "max_2, min_2 = print_min_and_max(weights[:,2], mapping, n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb8d0894",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "max_3, min_3 = print_min_and_max(weights[:,3], mapping, n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6c4497b1",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "max_4, min_4 = print_min_and_max(weights[:,4], mapping, n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4d65120",
   "metadata": {},
   "outputs": [],
   "source": [
    "overlap_max_01 = len(set(max_0) & set(max_1))\n",
    "overlap_min_01 = len(set(min_0) & set(min_1))\n",
    "print(f\"Overlap between max weight features {overlap_max_01/ len(max_0)}\")\n",
    "print(f\"Overlap between min weight features {overlap_min_01/ len(min_0)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f9cf8bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "overlap_max_03 = len(set(max_0) & set(max_3))\n",
    "overlap_min_03 = len(set(min_0) & set(min_3))\n",
    "print(f\"Overlap between max weight features {overlap_max_03/ len(max_0)}\")\n",
    "print(f\"Overlap between min weight features {overlap_min_03/ len(min_0)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5910b021",
   "metadata": {},
   "source": [
    "### Train on small dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c901f37",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read preprocessed dataset\n",
    "datasets = [\"train\", \"dev\", \"test\"]\n",
    "filepaths = []\n",
    "for i in datasets: \n",
    "    filepaths.append(f\"./PreprocessedData/{i}_lemmatization_small.csv\")\n",
    "    \n",
    "train_data_small = pd.read_csv(filepaths[0], index_col = 0)\n",
    "train_data_small = train_data_small.fillna('')\n",
    "dev_data_small = pd.read_csv(filepaths[1], index_col = 0)\n",
    "dev_data_small = dev_data_small.fillna('')\n",
    "test_data_small = pd.read_csv(filepaths[2], index_col = 0)\n",
    "test_data_small = test_data_small.fillna('')\n",
    "Y_train_small = train_data_small[\"label\"].to_numpy()\n",
    "Y_dev_small = dev_data_small[\"label\"].to_numpy()\n",
    "Y_test_small = test_data_small[\"label\"].to_numpy()\n",
    "X_train_small = train_data_small[\"sentence\"].values\n",
    "X_dev_small = dev_data_small[\"sentence\"].values\n",
    "X_test_small = test_data_small[\"sentence\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84961418",
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_vectorizer_small = TfidfVectorizer(use_idf=True, ngram_range=(1,2))\n",
    "X_train_vectors_tfidf_small = tfidf_vectorizer_small.fit_transform(X_train_small)\n",
    "X_dev_vectors_tfidf_small = tfidf_vectorizer_small.transform(X_dev_small)\n",
    "X_test_vectors_tfidf_small =  tfidf_vectorizer_small.transform(X_test_small)\n",
    "\n",
    "# sort indices of sparse matrix otherwise keras gives an error\n",
    "X_train_vectors_tfidf_small.sort_indices()\n",
    "X_dev_vectors_tfidf_small.sort_indices()\n",
    "X_test_vectors_tfidf_small.sort_indices()\n",
    "\n",
    "# Train logistic regression model\n",
    "#lr = get_logistic_regression(5, X_train_vectors_tfidf_small.shape[1])\n",
    "#file_path = f\"./TrainedModels/tfidf_small_lr.h5\"\n",
    "#checkpoint = ModelCheckpoint(file_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')    \n",
    "#early = EarlyStopping(monitor='val_acc', patience=3)\n",
    "#callbacks_list = [checkpoint, early]\n",
    "#lr.fit(X_train_vectors_tfidf_small, Y_train_small, epochs=40, validation_data=(X_dev_vectors_tfidf_small, Y_dev_small), batch_size=1024, verbose=2, callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92e892ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load weights and make final prediction on test set\n",
    "lr2 = get_logistic_regression(5, X_train_vectors_tfidf_small.shape[1])\n",
    "lr2.load_weights(\"./TrainedModels/tfidf_small_lr.h5\")\n",
    "\n",
    "y_hat_test_small = lr2.predict(X_test_vectors_tfidf_small)\n",
    "y_hat_test_small = np.argmax(y_hat_test_small, axis=-1)\n",
    "f1 = f1_score(Y_test_small, y_hat_test_small,average=\"weighted\")\n",
    "acc = accuracy_score(Y_test_small, y_hat_test_small)\n",
    "print(f\"F1 score: {f1}\")\n",
    "print(f\"Acc score: {acc}\")\n",
    "\n",
    "\n",
    "cm=confusion_matrix(Y_test_small,y_hat_test_small,normalize=\"true\")\n",
    "cmd = ConfusionMatrixDisplay(cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "498bcf47",
   "metadata": {},
   "source": [
    "## Further adjustments\n",
    "\n",
    "So far we only used the individual sentences for prediction. Since multiple sentences belong to an abstract we use the additionaly the relative position of a sentence in the abstract."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08358c3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79d1dcf0",
   "metadata": {},
   "outputs": [],
   "source": [
    "l = train_data.iloc[:,2].values.shape[0]\n",
    "helper = csr_matrix(train_data[\"line_relative\"].values).reshape((l,1))\n",
    "X_train_vectors_tfidf = csr_matrix(hstack([X_train_vectors_tfidf, helper]))\n",
    "l = dev_data.iloc[:,2].values.shape[0]\n",
    "helper = csr_matrix(dev_data[\"line_relative\"].values).reshape((l,1))\n",
    "X_dev_vectors_tfidf = csr_matrix(hstack([X_dev_vectors_tfidf, helper]))\n",
    "l = test_data.iloc[:,2].values.shape[0]\n",
    "helper = csr_matrix(test_data[\"line_relative\"].values).reshape((l,1))\n",
    "X_test_vectors_tfidf = csr_matrix(hstack([X_test_vectors_tfidf, helper]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31e52269",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_vectors_tfidf.sort_indices()\n",
    "X_dev_vectors_tfidf.sort_indices()\n",
    "X_test_vectors_tfidf.sort_indices()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef16d756",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = get_logistic_regression(5,50001)\n",
    "file_path = f\"./TrainedModels/tfidf_lr_linenumber.h5\"\n",
    "checkpoint = ModelCheckpoint(file_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')    \n",
    "early = EarlyStopping(monitor='val_acc', patience=3)\n",
    "callbacks_list = [checkpoint, early]\n",
    "lr.fit(X_train_vectors_tfidf, Y_train, epochs=40, validation_data=(X_dev_vectors_tfidf, Y_dev), batch_size=1024, verbose=2, callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3272a22",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load weights and make final prediction on test set\n",
    "lr2 = get_logistic_regression(5,50001)\n",
    "lr2.load_weights(\"./TrainedModels/tfidf_lr_linenumber.h5\")\n",
    "\n",
    "y_hat_test = lr2.predict(X_test_vectors_tfidf)\n",
    "y_hat_test = np.argmax(y_hat_test, axis=-1)\n",
    "f1 = f1_score(Y_test, y_hat_test,average=\"weighted\")\n",
    "acc = accuracy_score(Y_test, y_hat_test)\n",
    "print(f\"F1 score: {f1}\")\n",
    "print(f\"Accuracy score: {acc}\")\n",
    "\n",
    "\n",
    "cm = confusion_matrix(Y_test, y_hat_test, normalize=\"true\")\n",
    "cmd = ConfusionMatrixDisplay(cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10ea5ad1",
   "metadata": {},
   "source": [
    "F1 score: 0.8930709607539945\n",
    "\n",
    "Accuracy score: 0.8937374970331943"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c72c00f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = lr2.get_weights()\n",
    "weights = weights[0]\n",
    "mapping = tfidf_vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7aa9f96",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# for class 0 ()\n",
    "n = 15\n",
    "max_0, min_0 = print_min_and_max(weights[:,0], mapping, n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1eabada",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "max_1, min_1 = print_min_and_max(weights[:,1], mapping, n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed72287b",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "max_2, min_2 = print_min_and_max(weights[:,2], mapping, n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba3bcc42",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "max_3, min_3 = print_min_and_max(weights[:,3], mapping, n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa9bc228",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "max_4, min_4 = print_min_and_max(weights[:,4], mapping, n)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "367756f6",
   "metadata": {},
   "source": [
    "### Train on small dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4608d354",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data_small"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12fa2495",
   "metadata": {},
   "outputs": [],
   "source": [
    "l = train_data_small.iloc[:,2].values.shape[0]\n",
    "helper = csr_matrix(train_data_small[\"line_relative\"].values).reshape((l,1))\n",
    "X_train_vectors_tfidf_small = csr_matrix(hstack([X_train_vectors_tfidf_small, helper]))\n",
    "l = dev_data_small.iloc[:,2].values.shape[0]\n",
    "helper = csr_matrix(dev_data_small[\"line_relative\"].values).reshape((l,1))\n",
    "X_dev_vectors_tfidf_small = csr_matrix(hstack([X_dev_vectors_tfidf_small, helper]))\n",
    "l = test_data_small.iloc[:,2].values.shape[0]\n",
    "helper = csr_matrix(test_data_small[\"line_relativeline\"].values).reshape((l,1))\n",
    "X_test_vectors_tfidf_small = csr_matrix(hstack([X_test_vectors_tfidf_small, helper]))\n",
    "\n",
    "X_train_vectors_tfidf_small.sort_indices()\n",
    "X_dev_vectors_tfidf_small.sort_indices()\n",
    "X_test_vectors_tfidf_small.sort_indices()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49d2d7a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = get_logistic_regression(5, X_train_vectors_tfidf_small.shape[1])\n",
    "file_path = f\"./TrainedModels/tfidf_small_lr_linenumber.h5\"\n",
    "checkpoint = ModelCheckpoint(file_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')    \n",
    "early = EarlyStopping(monitor='val_acc', patience=3)\n",
    "callbacks_list = [checkpoint, early]\n",
    "lr.fit(X_train_vectors_tfidf_small, Y_train_small, epochs=50, validation_data=(X_dev_vectors_tfidf_small, Y_dev_small), batch_size=1024, verbose=2, callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ef7325f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load weights and make final prediction on test set\n",
    "lr2 = get_logistic_regression(5,X_train_vectors_tfidf_small.shape[1])\n",
    "lr2.load_weights(\"./TrainedModels/tfidf_small_lr_linenumber.h5\")\n",
    "\n",
    "y_hat_test_small = lr2.predict(X_test_vectors_tfidf_small)\n",
    "y_hat_test_small = np.argmax(y_hat_test_small, axis=-1)\n",
    "f1 = f1_score(Y_test_small, y_hat_test_small,average=\"weighted\")\n",
    "acc = accuracy_score(Y_test_small, y_hat_test_small)\n",
    "print(f\"F1 score: {f1}\")\n",
    "print(f\"Accuracy score: {acc}\")\n",
    "\n",
    "\n",
    "cm = confusion_matrix(Y_test_small, y_hat_test_small, normalize=\"true\")\n",
    "cmd = ConfusionMatrixDisplay(cm)\n",
    "cmd.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d68b7e5a",
   "metadata": {},
   "source": [
    "F1 score: 0.8591555015009821\n",
    "\n",
    "Accuracy score: 0.8613572258171561"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}