NLP.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NLP.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "WHGM4ET3HZH1"
      },
      "source": [
        "import xml.etree.ElementTree as ET\n",
        "import enum\n",
        "import os\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "import string\n",
        "import torch\n",
        "from transformers import DistilBertForTokenClassification, DistilBertTokenizerFast\n",
        "from torch.utils.data import Subset\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import f1_score, precision_score, recall_score\n",
        "from tqdm import tqdm\n",
        "from torch.utils.data import DataLoader"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l0zTBsokJOVj"
      },
      "source": [
        "# Code for ABSA\n",
        "\n",
        "This will be the code for the aspect-based sentiment detection task.\n",
        "\n",
        "## Parsing the train/test data\n",
        "\n",
        "Our first step will be parsing the train/test data. \n",
        "It comes as an XML file. We parse it into some custom\n",
        "data classes. We have four polarities, aspects and text. For\n",
        "the aspects, we will later use a set data structure\n",
        "to calculate precision/recall scores, so we need to provide\n",
        "a hash and equality operator."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lloBLlOmMjaZ"
      },
      "source": [
        "# We have four possible polarities\n",
        "\n",
        "class Polarity(enum.IntEnum):\n",
        "  NEUTRAL = 0,\n",
        "  POSITIVE = 1,\n",
        "  NEGATIVE = 2,\n",
        "  CONFLICT = 3\n",
        "\n",
        "polarities = {\n",
        "    \"positive\" : Polarity.POSITIVE,\n",
        "    \"negative\" : Polarity.NEGATIVE,\n",
        "    \"conflict\" : Polarity.CONFLICT,\n",
        "    \"neutral\" : Polarity.NEUTRAL\n",
        "}\n",
        "\n",
        "strings_polarities = {\n",
        "   Polarity.POSITIVE : \"positive\", \n",
        "   Polarity.NEGATIVE : \"negative\",\n",
        "   Polarity.CONFLICT : \"conflict\",\n",
        "   Polarity.NEUTRAL : \"neutral\"\n",
        "}\n",
        "\n",
        "class ABSATrainExample:\n",
        "  def __init__(self, text, aspects):\n",
        "    self.text = text\n",
        "    self.aspects = aspects\n",
        "    \n",
        "class ABSAAspect:\n",
        "  def __init__(self, polarity, from_, to):\n",
        "    self.polarity = polarity\n",
        "    self.end = to\n",
        "    self.start = from_\n",
        "\n",
        "  def __eq__(self, other):\n",
        "    if isinstance(other, ABSAAspect):\n",
        "      return self.polarity == other.polarity and \\\n",
        "       self.start == other.start and \\\n",
        "       self.end == other.end\n",
        "    else:\n",
        "      return False\n",
        "\n",
        "  def __ne__(self, other):\n",
        "    return (not self.__eq__(other))\n",
        "\n",
        "  def __hash__(self):\n",
        "    return hash(self.start + self.end)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7sHqVr3PJNY6"
      },
      "source": [
        "## Parsing the file\n",
        "\n",
        "Each training example comes with a sentence and a list of aspects. We parse\n",
        "both into our data structures. We save token ids, rather than the text, for later processing.\n",
        "\n",
        "When parsing the files, we have to take tokenization into account. We use the tokenization provided by the Huggingface library; because we need to later transform the examples into label sequences, we need to be able to associate specific terms to specific tokens. To do that, we split the data into words first and pass the ```split_into_words``` argument to the Tokenizer.\n",
        "\n",
        "We then locate the aspects in the token sequence and mark the positions. This\n",
        "allows for the creation of label sequences later on."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XA40hQcnLTLA"
      },
      "source": [
        "def locate_subsequence(query, base):\n",
        "    l = len(query)\n",
        "    for i in range(len(base)):\n",
        "        if base[i:i+l] == query:\n",
        "            return i, i + l\n",
        "    print(\"Not found\")\n",
        "    raise RuntimeError(\"Token sequence not found\")\n",
        "\n",
        "def parse_file(filename, tokenizer):\n",
        "  tree = ET.parse(filename)\n",
        "  sentences = tree.getroot()\n",
        "  train_examples = []\n",
        "  sentence_texts = []\n",
        "  for sentence in sentences:\n",
        "    # Extract text and clean it\n",
        "    text = sentence.find(\"text\").text\n",
        "    # Remove punctuation\n",
        "    text = text.translate(str.maketrans(string.punctuation, ' ' * len(string.punctuation))).lower()\n",
        "    # Split into words\n",
        "    # We do the splitting so we can locate the tokens later\n",
        "    text_split = text.split(' ')\n",
        "    # Add to list of texts\n",
        "    sentence_texts.append(text_split)\n",
        "\n",
        "  # Tokenize texts, adding padding and truncating to max length\n",
        "  tok_results = tokenizer(sentence_texts, is_split_into_words=True,\n",
        "                          truncation=True, padding=True,\n",
        "                          return_attention_mask=False)\n",
        "  \n",
        "  num_fail = 0\n",
        "  for text_tokenized, sentence in zip(tok_results['input_ids'], sentences):\n",
        "    # Extract aspects\n",
        "    aspects = sentence.find(\"aspectTerms\")\n",
        "    aspectList = []\n",
        "    if aspects is not None:\n",
        "      for aspect in aspects:\n",
        "        try:\n",
        "          # Clean aspect term, just like the text\n",
        "          term = aspect.get(\"term\")\n",
        "          if not term:\n",
        "            term = aspect.get(\"aspectTerm\")\n",
        "          term = term.translate(str.maketrans(string.punctuation, ' ' * len(string.punctuation))).lower()\n",
        "          term = term.split(' ')\n",
        "\n",
        "          # Tokenize and locate in text for later processing\n",
        "          term_tokenized = tokenizer(term, is_split_into_words=True)['input_ids']\n",
        "          start, end = locate_subsequence(term_tokenized[1:len(term_tokenized) - 1], text_tokenized)\n",
        "          polarity = polarities[aspect.get(\"polarity\")]\n",
        "\n",
        "          aspectList.append(ABSAAspect(polarity, start, end))\n",
        "        except Exception as e:\n",
        "          print(term)\n",
        "          print(text_tokenized)\n",
        "          print(term_tokenized[1:len(term_tokenized) - 1])\n",
        "          num_fail += 1\n",
        "    train_examples.append(ABSATrainExample(text_tokenized, aspectList))\n",
        "  return train_examples, num_fail"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4rVy80F7K-hz"
      },
      "source": [
        "## Converting training examples to labels\n",
        "\n",
        "Here, we take the training examples and convert them to label sequences."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B-cckHc0hpZG"
      },
      "source": [
        "# Take training examples and convert them into sequence labels\n",
        "# We use a BIO scheme\n",
        "# First, define classes\n",
        "\n",
        "class SequenceLabel(enum.IntEnum):\n",
        "  OUTSIDE = 0,\n",
        "  BEGIN_POS = 1,\n",
        "  BEGIN_NEG = 2,\n",
        "  BEGIN_CON = 3,\n",
        "  BEGIN_NEU = 4,\n",
        "  INSIDE_POS = 5,\n",
        "  INSIDE_NEG = 6,\n",
        "  INSIDE_CON = 7,\n",
        "  INSIDE_NEU = 8\n",
        "\n",
        "NUM_LABELS = 9\n",
        "\n",
        "polarities_to_labels = {\n",
        "    Polarity.POSITIVE : [SequenceLabel.BEGIN_POS, SequenceLabel.INSIDE_POS],\n",
        "    Polarity.NEGATIVE : [SequenceLabel.BEGIN_NEG, SequenceLabel.INSIDE_NEG],\n",
        "    Polarity.NEUTRAL : [SequenceLabel.BEGIN_NEU, SequenceLabel.INSIDE_NEU],\n",
        "    Polarity.CONFLICT : [SequenceLabel.BEGIN_CON, SequenceLabel.INSIDE_CON]\n",
        "}\n",
        "\n",
        "def example_to_labels(example):\n",
        "  aspects = example.aspects\n",
        "  text_tokens = example.text\n",
        "  initial_labels = [0] * len(text_tokens)\n",
        "  for aspect in aspects:\n",
        "    label_beg, label_end = polarities_to_labels[aspect.polarity]\n",
        "    aspect_begin = aspect.start\n",
        "    aspect_end = aspect.end\n",
        "    initial_labels[aspect_begin] = int(label_beg)\n",
        "    for i in range(aspect_begin + 1, aspect_end):\n",
        "      initial_labels[i] = int(label_end)\n",
        "  return initial_labels"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oP3rUyOHLItO"
      },
      "source": [
        "## Creation of dataset for PyTorch\n",
        "\n",
        "We make use of the PyTorch Dataset API to later train our models. We create a Pytorch dataset from our datapoints."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ws-RQnkmC9Rh"
      },
      "source": [
        "class ABSADataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, tokens, labels):\n",
        "      self.tokens = tokens\n",
        "      self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "      item = {'tokens' : torch.tensor(self.tokens[idx])}\n",
        "      item['labels'] = torch.tensor(self.labels[idx])\n",
        "      return item\n",
        "\n",
        "    def __len__(self):\n",
        "      return len(self.labels)\n",
        "\n",
        "def create_datasets_tokenizer(tokenizer):\n",
        "  laptops_trial, fail_lt = parse_file(\"Laptop_Test.xml\", tokenizer)\n",
        "  restaurants_trial, fail_rt = parse_file(\"Restaurants_Test.xml\", tokenizer)\n",
        "  laptops_train, fail_ltr = parse_file(\"Laptop_Train.xml\", tokenizer)\n",
        "  restaurants_train, fail_rtr = parse_file(\"Restaurants_Train.xml\", tokenizer)\n",
        "\n",
        "\n",
        "  labels_laptops_trial = [example_to_labels(example) for example in laptops_trial]\n",
        "  tokens_laptops_trial = [example.text for example in laptops_trial]\n",
        "  labels_laptops_train = [example_to_labels(example) for example in laptops_train]\n",
        "  tokens_laptops_train = [example.text for example in laptops_train]\n",
        "\n",
        "  labels_restaurants_trial = [example_to_labels(example) for example in restaurants_trial]\n",
        "  tokens_restaurants_trial = [example.text for example in restaurants_trial]\n",
        "  labels_restaurants_train = [example_to_labels(example) for example in restaurants_train]\n",
        "  tokens_restaurants_train = [example.text for example in restaurants_train]\n",
        "\n",
        "  laptops_train_ds = ABSADataset(tokens_laptops_train, labels_laptops_train)\n",
        "  laptops_trial_ds = ABSADataset(tokens_laptops_trial, labels_laptops_trial)\n",
        "\n",
        "  restaurants_train_ds = ABSADataset(tokens_restaurants_train, labels_restaurants_train)\n",
        "  restaurants_trial_ds = ABSADataset(tokens_restaurants_trial, labels_restaurants_trial)\n",
        "\n",
        "  return laptops_train_ds, laptops_trial_ds, restaurants_train_ds, restaurants_trial_ds"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gehjfAuaLfoE"
      },
      "source": [
        "## Evaluation metrics\n",
        "\n",
        "Here, we provide the evaluation metrics. We have the F1 score, precision and recall. We compute these grouped by polarity. For the precision and recall, we retrieve true positives, false positives and false negatives by building a set of true and predicted examples. For each predicted example, we check if it is the set of true examples; if it is, it is a true positive. If it is not, it is a false positive. We remove true positives from the set of true examples; once we have gone through all predicted examples, if there are any true examples left in the set, they are false negatives.\n",
        "\n",
        "To do this, we also need to write functions to convert label sequences to our aspect data structure. We added support for hashing above.\n",
        "\n",
        "We allow for the calculation of both the macro and micro F1 score, precision and recall."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Ef-Net5TExp"
      },
      "source": [
        "\n",
        "# We add the inside labels because starting with an inside label is incorrect\n",
        "beg_labels_polarity = {\n",
        "    int(SequenceLabel.BEGIN_POS) : Polarity.POSITIVE,\n",
        "    int(SequenceLabel.BEGIN_NEU) : Polarity.NEUTRAL,\n",
        "    int(SequenceLabel.BEGIN_NEG) : Polarity.NEGATIVE,\n",
        "    int(SequenceLabel.BEGIN_CON) : Polarity.CONFLICT,\n",
        "}\n",
        "\n",
        "in_labels = {\n",
        "    int(SequenceLabel.INSIDE_POS),\n",
        "    int(SequenceLabel.INSIDE_NEU),\n",
        "    int(SequenceLabel.INSIDE_NEG),\n",
        "    int(SequenceLabel.INSIDE_CON),\n",
        "}\n",
        "\n",
        "def labels_to_aspects(labels_seqs):\n",
        "  aspects = []\n",
        "  for label_seq in labels_seqs:\n",
        "    i = 0\n",
        "    while i < len(label_seq):\n",
        "      cur_label = label_seq[i]\n",
        "      if cur_label in beg_labels_polarity:\n",
        "        polarity = beg_labels_polarity[cur_label]\n",
        "        start = i\n",
        "        end = i + 1\n",
        "        i += 1\n",
        "        while label_seq[i] in in_labels:\n",
        "          i += 1\n",
        "          end += 1\n",
        "        aspects.append(ABSAAspect(polarity, start, end))\n",
        "      else:\n",
        "        i += 1\n",
        "  return aspects\n",
        "\n",
        "def group_by_polarities(aspect_list):\n",
        "  aspects = {\n",
        "      Polarity.POSITIVE : [],\n",
        "      Polarity.NEUTRAL : [],\n",
        "      Polarity.NEGATIVE : [],\n",
        "      Polarity.CONFLICT : []\n",
        "  }\n",
        "  \n",
        "  for aspect in aspect_list:\n",
        "    aspects[aspect.polarity].append(aspect)\n",
        "  return aspects\n",
        "\n",
        "def calculate_f1(true_aspects, pred_aspects, micro=False):\n",
        "  precision = []\n",
        "  recalls = []\n",
        "  f1_scores = []\n",
        "\n",
        "  tp_total = 0\n",
        "  fp_total = 0\n",
        "  fn_total = 0\n",
        "\n",
        "  true_aspects_grouped = group_by_polarities(true_aspects)\n",
        "  pred_aspects_grouped = group_by_polarities(pred_aspects)\n",
        "\n",
        "  for key, value in true_aspects_grouped.items():\n",
        "    pred_aspects_for_item = pred_aspects_grouped[key]\n",
        "    true_aspect_set = set(value)\n",
        "    pred_aspect_set = set(pred_aspects_for_item)\n",
        "\n",
        "    tp = 0\n",
        "    fp = 0\n",
        "\n",
        "    while pred_aspect_set:\n",
        "      aspect = pred_aspect_set.pop()\n",
        "      if aspect in true_aspect_set:\n",
        "        true_aspect_set.remove(aspect)\n",
        "        tp += 1\n",
        "      else:\n",
        "        fp += 1\n",
        "    fn = len(true_aspect_set)\n",
        "\n",
        "    tp_total += tp\n",
        "    fp_total += fp\n",
        "    fn_total += fn\n",
        "\n",
        "    if tp + fp > 0 and tp + fn > 0:\n",
        "      prec = tp / (tp + fp)\n",
        "      recall = tp / (tp + fn)\n",
        "      if prec + recall > 0:\n",
        "        f1 = 2 * ((prec * recall) / (prec + recall))\n",
        "      else:\n",
        "        f1 = 0\n",
        "\n",
        "      precision.append(prec)\n",
        "      recalls.append(recall)\n",
        "      f1_scores.append(f1)\n",
        "    else:\n",
        "      precision.append(0)\n",
        "      recalls.append(0)\n",
        "      f1_scores.append(0)\n",
        "\n",
        "  if micro:\n",
        "    prec = tp_total / (tp_total + fp_total)\n",
        "    recall = tp_total / (tp_total + fn_total)\n",
        "    if prec + recall > 0:\n",
        "      f1 = 2 * ((prec * recall) / (prec + recall))\n",
        "    else:\n",
        "      f1 = 0\n",
        "    return f1, prec, recall\n",
        "\n",
        "  else:\n",
        "    return np.mean(np.array(f1_scores)), np.mean(np.array(precision)), \\\n",
        "     np.mean(np.array(recalls))\n",
        "\n",
        "\n",
        "def eval_model(model, trial_loader, device, micro=False):\n",
        "    model.to(device)\n",
        "    model.eval()\n",
        "    f1_vals = []\n",
        "    aspects_real_full = []\n",
        "    aspects_pred_full = []\n",
        "    with torch.no_grad():\n",
        "      for batch in trial_loader:\n",
        "        input_ids = batch['tokens'].to(device)\n",
        "        outputs = model(input_ids).logits.cpu().numpy()\n",
        "\n",
        "        labels_pred = np.argmax(outputs, axis=-1)\n",
        "        labels_real = batch['labels'].numpy()\n",
        "\n",
        "        aspects_pred = labels_to_aspects(labels_pred)\n",
        "        aspects_real = labels_to_aspects(labels_real)\n",
        "\n",
        "        aspects_real_full += aspects_real\n",
        "        aspects_pred_full += aspects_pred\n",
        "\n",
        "    f1_score, precision, recall = calculate_f1(aspects_real_full,\n",
        "                                               aspects_pred_full,\n",
        "                                               micro=micro)\n",
        "\n",
        "    return f1_score, precision, recall"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UJbimZtGMTVF"
      },
      "source": [
        "## Code for training the model\n",
        "\n",
        "This is some code for training the model. Nothing too fancy; we add some rudimentary support for early stopping."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "duAqIOJhhvLT"
      },
      "source": [
        "def train_model(model, train_loader, optim, num_epochs,\n",
        "                device, early_stopping=False, val_loader=None, min_delta=0.05):\n",
        "  model.to(device)\n",
        "\n",
        "  best_f1 = 0\n",
        "  losses = []\n",
        "\n",
        "  for epoch in range(num_epochs):\n",
        "    pbar = tqdm(train_loader)\n",
        "    for batch in pbar:\n",
        "      model.train()\n",
        "      # Do train step\n",
        "      optim.zero_grad()\n",
        "      input_ids = batch['tokens'].to(device)\n",
        "      labels = batch['labels'].to(device)\n",
        "      outputs = model(input_ids, labels=labels)\n",
        "      loss = outputs[0]\n",
        "      loss.backward()\n",
        "      optim.step()\n",
        "\n",
        "      # Update progress bar\n",
        "      pbar_string = \"Epoch {}/{} loss {:.2f}\".format(\n",
        "          epoch + 1, num_epochs, loss.item())\n",
        "      pbar.set_description(pbar_string)\n",
        "\n",
        "      # Add loss\n",
        "      losses.append(loss.item())\n",
        "    if early_stopping:\n",
        "      f1, _, _ = eval_model(model, val_loader, device)\n",
        "      print(\"Val F1 {:.4f}\".format(f1))\n",
        "      if f1 < (best_f1 - min_delta):\n",
        "        return losses, epoch\n",
        "      elif f1 > best_f1:\n",
        "        best_f1 = f1\n",
        "  return losses, num_epochs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r4SutuJRNzed"
      },
      "source": [
        "## DistillBERT - Laptops and Restaurants"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wk1-dcoyRBiu"
      },
      "source": [
        "tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')\n",
        "laptops_train_ds, laptops_trial_ds, restaurants_train_ds, restaurants_trial_ds = create_datasets_tokenizer(tokenizer)\n",
        "\n",
        "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n",
        "\n",
        "# Laptops\n",
        "train_loader = DataLoader(laptops_train_ds, batch_size=16, shuffle=True)\n",
        "trial_loader = DataLoader(laptops_trial_ds, batch_size=16, shuffle=True)\n",
        "\n",
        "model = DistilBertForTokenClassification.from_pretrained('distilbert-base-uncased',\n",
        "                                                         num_labels=NUM_LABELS)\n",
        "model.to(device)\n",
        "optim = torch.optim.AdamW(model.parameters(), lr=5e-5)\n",
        "_, num_epochs = train_model(model, train_loader, optim, 15, device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wwio1uUpdTOP"
      },
      "source": [
        "f1, prec, recall = eval_model(model, trial_loader, device)\n",
        "print(\"Laptops f1, prec, recall trial\", f1, prec, recall)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3VnYr5k1dQzQ"
      },
      "source": [
        "# Restaurants\n",
        "train_loader = DataLoader(restaurants_train_ds, batch_size=16, shuffle=True)\n",
        "trial_loader = DataLoader(restaurants_trial_ds, batch_size=16, shuffle=True)\n",
        "\n",
        "model = DistilBertForTokenClassification.from_pretrained('distilbert-base-uncased',\n",
        "                                                         num_labels=NUM_LABELS)\n",
        "model.to(device)\n",
        "optim = torch.optim.AdamW(model.parameters(), lr=5e-5)\n",
        "_, num_epochs = train_model(model, train_loader, optim, 20, device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MmCWX7fCQUS7"
      },
      "source": [
        "f1, prec, recall = eval_model(model, trial_loader, device)\n",
        "print(\"Restaurants f1, prec, recall trial\", f1, prec, recall)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W04Kt8oPN9VS"
      },
      "source": [
        "## Dataset statistics"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kfgLP3dV_JLZ"
      },
      "source": [
        "# Produce plots detailing distributions of data points\n",
        "\n",
        "\n",
        "laptops_train, fail_ltr = parse_file(\"Laptop_Test.xml\", tokenizer)\n",
        "restaurants_train, fail_rtr = parse_file(\"Restaurants_Test.xml\", tokenizer)\n",
        "\n",
        "print(\"Total number test samples laptop\", len(laptops_train))\n",
        "print(\"Total number test samples restaurant\", len(restaurants_train))\n",
        "print(\"Total number failed to parse\", fail_ltr + fail_rtr)\n",
        "\n",
        "tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')\n",
        "laptops_train, fail_ltr = parse_file(\"Laptop_Train.xml\", tokenizer)\n",
        "restaurants_train, fail_rtr = parse_file(\"Restaurants_Train.xml\", tokenizer)\n",
        "\n",
        "print(\"Total number training samples laptop\", len(laptops_train))\n",
        "print(\"Total number training samples restaurant\", len(restaurants_train))\n",
        "print(\"Total number failed to parse\", fail_ltr + fail_rtr)\n",
        "\n",
        "aspect_distribution_laptop = {\n",
        "    \"positive\" : 0,\n",
        "    \"negative\" : 0,\n",
        "    \"conflict\" : 0,\n",
        "    \"neutral\" : 0\n",
        "}\n",
        "\n",
        "for example in laptops_train:\n",
        "  for aspect in example.aspects:\n",
        "    aspect_distribution_laptop[strings_polarities[aspect.polarity]] += 1\n",
        "\n",
        "df_aspect_dist_laptop = pd.DataFrame(aspect_distribution_laptop.items(),\n",
        "                                     columns=[\"Aspect\", \"Number of items\"])\n",
        "sns.barplot(data=df_aspect_dist_laptop, x=\"Aspect\", y=\"Number of items\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DnDmcG-xLTL7"
      },
      "source": [
        "aspect_distribution_restaurant = {\n",
        "    \"positive\" : 0,\n",
        "    \"negative\" : 0,\n",
        "    \"conflict\" : 0,\n",
        "    \"neutral\" : 0\n",
        "}\n",
        "\n",
        "for example in restaurants_train:\n",
        "  for aspect in example.aspects:\n",
        "    aspect_distribution_restaurant[strings_polarities[aspect.polarity]] += 1\n",
        "\n",
        "df_aspect_dist_restaurants = pd.DataFrame(aspect_distribution_restaurant.items(),\n",
        "                                     columns=[\"Aspect\", \"Number of items\"])\n",
        "sns.barplot(data=df_aspect_dist_restaurants, x=\"Aspect\", y=\"Number of items\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Iv4FzkbT68E3"
      },
      "source": [
        "def parse_file_baseline(filename):\n",
        "  tree = ET.parse(filename)\n",
        "  sentences = tree.getroot()\n",
        "  train_examples = []\n",
        "  for sentence in sentences:\n",
        "    aspectList = []\n",
        "    text = sentence.find(\"text\").text\n",
        "    aspects = sentence.find(\"aspectTerms\")\n",
        "    if aspects:\n",
        "      for aspect in aspects:\n",
        "          polarity = polarities[aspect.get(\"polarity\")]\n",
        "          start = int(aspect.get(\"from\"))\n",
        "          end = int(aspect.get(\"to\"))\n",
        "          aspectList.append(ABSAAspect(polarity, start, end)) \n",
        "    train_examples.append(ABSATrainExample(text, aspectList))\n",
        "  return train_examples"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fn95reRArh26"
      },
      "source": [
        "## Baseline classifier"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WDyYFhc7y8YB"
      },
      "source": [
        "def create_baseline_classifier(dataset):\n",
        "  baseline_map = {}\n",
        "\n",
        "  for example in dataset:\n",
        "    for aspect in example.aspects:\n",
        "      list_string = example.text[aspect.start:aspect.end]\n",
        "      if list_string not in baseline_map:\n",
        "        baseline_map[list_string] = np.zeros(4)\n",
        "      baseline_map[list_string][int(aspect.polarity)] += 1\n",
        "  \n",
        "  for k, v in baseline_map.items():\n",
        "    baseline_map[k] = np.argmax(v)\n",
        "\n",
        "  return baseline_map"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K0hJ2mka0nSA"
      },
      "source": [
        "def get_confusion_matrix_by_aspect(true_aspects, pred_aspects):\n",
        "  true_aspects_grouped = group_by_polarities(true_aspects)\n",
        "  pred_aspects_grouped = group_by_polarities(pred_aspects)\n",
        "\n",
        "  tps = [0, 0, 0, 0]\n",
        "  fps = [0, 0, 0, 0]\n",
        "  fns = [0, 0, 0, 0]\n",
        "\n",
        "  for key, value in true_aspects_grouped.items():\n",
        "    pred_aspects_for_item = pred_aspects_grouped[key]\n",
        "    true_aspect_set = set(value)\n",
        "    pred_aspect_set = set(pred_aspects_for_item)\n",
        "\n",
        "    tp = 0\n",
        "    fp = 0\n",
        "\n",
        "    while pred_aspect_set:\n",
        "      aspect = pred_aspect_set.pop()\n",
        "      if aspect in true_aspect_set:\n",
        "        true_aspect_set.remove(aspect)\n",
        "        tp += 1\n",
        "      else:\n",
        "        fp += 1\n",
        "    fn = len(true_aspect_set)\n",
        "\n",
        "    tps[int(key)] = tp\n",
        "    fps[int(key)] = fp\n",
        "    fns[int(key)] = fn\n",
        "\n",
        "  return tps, fps, fns\n",
        "\n",
        "def extract_aspects_baseline(dataset, baseline_classifier):\n",
        "  # Convert token sequences to strings\n",
        "  f1 = 0\n",
        "  prec = 0\n",
        "  recall = 0\n",
        "\n",
        "  fp_matrix = []\n",
        "  tp_matrix = []\n",
        "  fn_matrix = []\n",
        "\n",
        "  for datapoint in dataset:\n",
        "    aspects_processed = []\n",
        "    found_aspects = []\n",
        "    for k, v in baseline_classifier.items():\n",
        "      idx_substr = datapoint.text.find(k)\n",
        "      if idx_substr != -1:\n",
        "        polarity = Polarity(v)\n",
        "        start = idx_substr\n",
        "        end = len(k) + idx_substr\n",
        "        found_aspects.append(ABSAAspect(polarity, start, end))\n",
        "    tps, fps, fns = get_confusion_matrix_by_aspect(datapoint.aspects, found_aspects)\n",
        "\n",
        "    fp_matrix.append(fps)\n",
        "    tp_matrix.append(tps)\n",
        "    fn_matrix.append(fns)\n",
        "\n",
        "  tp_matrix = np.array(tp_matrix)\n",
        "  fp_matrix = np.array(fp_matrix)\n",
        "  fn_matrix = np.array(fn_matrix)\n",
        "\n",
        "  tp_matrix = np.sum(tp_matrix, axis=0)\n",
        "  fp_matrix = np.sum(fp_matrix, axis=0)\n",
        "  fn_matrix = np.sum(fn_matrix, axis=0)\n",
        "\n",
        "  prec_matrix = tp_matrix / (tp_matrix + fp_matrix)\n",
        "  recall_matrix = tp_matrix / (tp_matrix + fn_matrix)\n",
        "  f1_matrix = np.nan_to_num(2 * ((prec_matrix * recall_matrix) / (prec_matrix + recall_matrix)))\n",
        "\n",
        "  return np.mean(f1_matrix), np.mean(prec_matrix), np.mean(f1_matrix)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CkZo7dGQ0HLY"
      },
      "source": [
        "laptop_baseline = parse_file_baseline(\"Laptop_Train.xml\")\n",
        "laptop_test = parse_file_baseline(\"Laptop_Test.xml\")\n",
        "class_laptop = create_baseline_classifier(laptop_baseline)\n",
        "extract_aspects_baseline(laptop_test, class_laptop)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tOaXg3dWlt1f"
      },
      "source": [
        "rest_baseline = parse_file_baseline(\"Restaurants_Train.xml\")\n",
        "rest_test = parse_file_baseline(\"Restaurants_Test.xml\")\n",
        "class_rest = create_baseline_classifier(rest_baseline)\n",
        "extract_aspects_baseline(rest_test, class_rest)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}