08_TF_Developer_Certificate_Course.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-Rj7sBkxRK6G",
    "outputId": "1f3a17ef-f667-43a2-f5ba-2605f6f6b1f6",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:06.673513956Z",
     "start_time": "2024-01-02T18:13:04.220018518Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "W67y0phrRQnz",
    "outputId": "42ed52f4-ad7a-4358-be20-5d6bd952469d",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:07.176602899Z",
     "start_time": "2024-01-02T18:13:06.672889063Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] 'helper_functions.py' already exists, skipping download.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "if not os.path.exists(\"helper_functions.py\"):\n",
    "  !wget https://raw.githubusercontent.com/mrdbourke/tensorflow-deep-learning/main/extras/helper_functions.py\n",
    "else:\n",
    "  print(\"[INFO] 'helper_functions.py' already exists, skipping download.\")\n",
    "\n",
    "# Import series of helper functions for the notebook (we've created/used these in previous notebooks)\n",
    "from helper_functions import importTensorflow, unzip_data, create_tensorboard_callback, plot_loss_curves, compare_historys, calculate_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:08.132007380Z",
     "start_time": "2024-01-02T18:13:07.915793845Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.13.0\n",
      "1 Physical GPUs, 1 Logical GPUs\n",
      "INFO:tensorflow:Mixed precision compatibility check (mixed_float16): OK\n",
      "Your GPU will likely run quickly with dtype policy mixed_float16 as it has compute capability of at least 7.0. Your GPU: NVIDIA GeForce RTX 3050 Laptop GPU, compute capability 8.6\n",
      "<Policy \"mixed_float16\">\n"
     ]
    }
   ],
   "source": [
    "importTensorflow(memory=4090, precision=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "V5L-H4otpgvS"
   },
   "source": [
    "We will be using RNN to train textual information\n",
    "\n",
    "**Basic Structure of RNN**\n",
    "* Input layer\n",
    "* text_vectorizer(input)\n",
    "* embedding(x)\n",
    "* LSTM -> activation = tanh\n",
    "* output_Dense -> activation = sigmoid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "h0eqNb9Nsyzi"
   },
   "source": [
    "### Get the text dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "t2Fp9BY2pkCZ",
    "outputId": "9e45db91-9899-42d4-e2ce-e20a3b3f6ec4",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:18.613217282Z",
     "start_time": "2024-01-02T18:13:16.775034252Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2024-01-02 23:43:16--  https://storage.googleapis.com/ztm_tf_course/nlp_getting_started.zip\r\n",
      "Resolving storage.googleapis.com (storage.googleapis.com)... 2404:6800:4007:82b::201b, 2404:6800:4007:828::201b, 2404:6800:4007:829::201b, ...\r\n",
      "Connecting to storage.googleapis.com (storage.googleapis.com)|2404:6800:4007:82b::201b|:443... connected.\r\n",
      "HTTP request sent, awaiting response... 200 OK\r\n",
      "Length: 607343 (593K) [application/zip]\r\n",
      "Saving to: ‘nlp_getting_started.zip’\r\n",
      "\r\n",
      "nlp_getting_started 100%[===================>] 593.11K   554KB/s    in 1.1s    \r\n",
      "\r\n",
      "2024-01-02 23:43:18 (554 KB/s) - ‘nlp_getting_started.zip’ saved [607343/607343]\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "if not os.path.exists(\"nlp_getting_started.zip\"):\n",
    "    !wget https://storage.googleapis.com/ztm_tf_course/nlp_getting_started.zip\n",
    "    unzip_data(\"nlp_getting_started.zip\")\n",
    "else:\n",
    "  print(\"[INFO] already exists, skipping download.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_rzD1DRKtRlJ"
   },
   "source": [
    "### Visualizing a text dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "qyWmNLXTtC_x",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:20.374146949Z",
     "start_time": "2024-01-02T18:13:20.298928146Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "train_df = pd.read_csv(\"train.csv\")\n",
    "test_df = pd.read_csv(\"test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 359
    },
    "id": "BoTo82yJt4no",
    "outputId": "42f9155e-4b7b-4493-d674-529bf9b33799",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:21.231608506Z",
     "start_time": "2024-01-02T18:13:21.178022192Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "   id keyword location                                               text  \\\n0   1     NaN      NaN  Our Deeds are the Reason of this #earthquake M...   \n1   4     NaN      NaN             Forest fire near La Ronge Sask. Canada   \n2   5     NaN      NaN  All residents asked to 'shelter in place' are ...   \n3   6     NaN      NaN  13,000 people receive #wildfires evacuation or...   \n4   7     NaN      NaN  Just got sent this photo from Ruby #Alaska as ...   \n5   8     NaN      NaN  #RockyFire Update => California Hwy. 20 closed...   \n6  10     NaN      NaN  #flood #disaster Heavy rain causes flash flood...   \n7  13     NaN      NaN  I'm on top of the hill and I can see a fire in...   \n8  14     NaN      NaN  There's an emergency evacuation happening now ...   \n9  15     NaN      NaN  I'm afraid that the tornado is coming to our a...   \n\n   target  \n0       1  \n1       1  \n2       1  \n3       1  \n4       1  \n5       1  \n6       1  \n7       1  \n8       1  \n9       1  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>keyword</th>\n      <th>location</th>\n      <th>text</th>\n      <th>target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Our Deeds are the Reason of this #earthquake M...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>4</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Forest fire near La Ronge Sask. Canada</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>5</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>All residents asked to 'shelter in place' are ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>6</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>13,000 people receive #wildfires evacuation or...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>7</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Just got sent this photo from Ruby #Alaska as ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>8</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>#RockyFire Update =&gt; California Hwy. 20 closed...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>10</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>#flood #disaster Heavy rain causes flash flood...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>13</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>I'm on top of the hill and I can see a fire in...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>14</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>There's an emergency evacuation happening now ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>15</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>I'm afraid that the tornado is coming to our a...</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 359
    },
    "id": "5ImBTHVft6K2",
    "outputId": "03de4f67-1175-471c-a854-9fae6608be73",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:22.350959468Z",
     "start_time": "2024-01-02T18:13:22.278138852Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "        id      keyword               location  \\\n2644  3796  destruction                    NaN   \n2227  3185       deluge                    NaN   \n5448  7769       police                     UK   \n132    191   aftershock                    NaN   \n6845  9810       trauma  Montgomery County, MD   \n5559  7934    rainstorm                    NaN   \n1765  2538    collision                    NaN   \n1817  2611      crashed                    NaN   \n6810  9756      tragedy        Los Angeles, CA   \n4398  6254    hijacking          Athens,Greece   \n\n                                                   text  target  \n2644  So you have a new weapon that can cause un-ima...       1  \n2227  The f$&amp;@ing things I do for #GISHWHES Just...       0  \n5448  DT @georgegalloway: RT @Galloway4Mayor: ÛÏThe...       1  \n132   Aftershock back to school kick off was great. ...       0  \n6845  in response to trauma Children of Addicts deve...       0  \n5559  @Calum5SOS you look like you got caught in a r...       0  \n1765  my favorite lady came to our volunteer meeting...       1  \n1817  @brianroemmele UX fail of EMV - people want to...       1  \n6810  Can't find my ariana grande shirt  this is a f...       0  \n4398  The Murderous Story Of AmericaÛªs First Hijac...       1  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>keyword</th>\n      <th>location</th>\n      <th>text</th>\n      <th>target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>2644</th>\n      <td>3796</td>\n      <td>destruction</td>\n      <td>NaN</td>\n      <td>So you have a new weapon that can cause un-ima...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2227</th>\n      <td>3185</td>\n      <td>deluge</td>\n      <td>NaN</td>\n      <td>The f$&amp;amp;@ing things I do for #GISHWHES Just...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>5448</th>\n      <td>7769</td>\n      <td>police</td>\n      <td>UK</td>\n      <td>DT @georgegalloway: RT @Galloway4Mayor: ÛÏThe...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>132</th>\n      <td>191</td>\n      <td>aftershock</td>\n      <td>NaN</td>\n      <td>Aftershock back to school kick off was great. ...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>6845</th>\n      <td>9810</td>\n      <td>trauma</td>\n      <td>Montgomery County, MD</td>\n      <td>in response to trauma Children of Addicts deve...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>5559</th>\n      <td>7934</td>\n      <td>rainstorm</td>\n      <td>NaN</td>\n      <td>@Calum5SOS you look like you got caught in a r...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1765</th>\n      <td>2538</td>\n      <td>collision</td>\n      <td>NaN</td>\n      <td>my favorite lady came to our volunteer meeting...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1817</th>\n      <td>2611</td>\n      <td>crashed</td>\n      <td>NaN</td>\n      <td>@brianroemmele UX fail of EMV - people want to...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>6810</th>\n      <td>9756</td>\n      <td>tragedy</td>\n      <td>Los Angeles, CA</td>\n      <td>Can't find my ariana grande shirt  this is a f...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4398</th>\n      <td>6254</td>\n      <td>hijacking</td>\n      <td>Athens,Greece</td>\n      <td>The Murderous Story Of AmericaÛªs First Hijac...</td>\n      <td>1</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Shuffle to train data\n",
    "train_df_shuffled = train_df.sample(frac=1, random_state=42)\n",
    "train_df_shuffled.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 359
    },
    "id": "2wbB2XBBuUsd",
    "outputId": "0c9bade6-2760-4830-b9b1-3f4e1cdeef42",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:24.515514983Z",
     "start_time": "2024-01-02T18:13:24.465066058Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "   id keyword location                                               text\n0   0     NaN      NaN                 Just happened a terrible car crash\n1   2     NaN      NaN  Heard about #earthquake is different cities, s...\n2   3     NaN      NaN  there is a forest fire at spot pond, geese are...\n3   9     NaN      NaN           Apocalypse lighting. #Spokane #wildfires\n4  11     NaN      NaN      Typhoon Soudelor kills 28 in China and Taiwan\n5  12     NaN      NaN                 We're shaking...It's an earthquake\n6  21     NaN      NaN  They'd probably still show more life than Arse...\n7  22     NaN      NaN                                  Hey! How are you?\n8  27     NaN      NaN                                   What a nice hat?\n9  29     NaN      NaN                                          Fuck off!",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>id</th>\n      <th>keyword</th>\n      <th>location</th>\n      <th>text</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Just happened a terrible car crash</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>2</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Heard about #earthquake is different cities, s...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>there is a forest fire at spot pond, geese are...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>9</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Apocalypse lighting. #Spokane #wildfires</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>11</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Typhoon Soudelor kills 28 in China and Taiwan</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>12</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>We're shaking...It's an earthquake</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>21</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>They'd probably still show more life than Arse...</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>22</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Hey! How are you?</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>27</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>What a nice hat?</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>29</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>Fuck off!</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test_data\n",
    "test_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "VFNWm50cubJ8",
    "outputId": "aa2581ec-ec4e-447d-bcfd-d9beb8717732",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:25.102601309Z",
     "start_time": "2024-01-02T18:13:24.970725775Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "target\n0    4342\n1    3271\nName: count, dtype: int64"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#How many examples in each class\n",
    "train_df.target.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DA79OdN4umTy",
    "outputId": "507f1634-dd9b-4c98-8d57-2a00d0decde7",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:25.984230968Z",
     "start_time": "2024-01-02T18:13:25.923015393Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6886\n",
      "Target: 1, (real disaster)\n",
      "Text:\n",
      "Horrible Accident  Man Died In Wings of Airplane (29-07-2015) http://t.co/i7kZtevb2v\n",
      "\n",
      "---\n",
      "Target: 1, (real disaster)\n",
      "Text:\n",
      "Wreckage 'Conclusively Confirmed' as From MH370: Malaysia PM: Investigators and the families of those who were... http://t.co/MSsq0sVnBM\n",
      "\n",
      "---\n",
      "Target: 1, (real disaster)\n",
      "Text:\n",
      "Russian 'food crematoria' provoke outrage amid crisis famine memories - Yahoo News http://t.co/6siiRlnV6z\n",
      "\n",
      "---\n",
      "Target: 1, (real disaster)\n",
      "Text:\n",
      "The ol' meltdown victory for the Mets.\n",
      "\n",
      "---\n",
      "Target: 1, (real disaster)\n",
      "Text:\n",
      "More Natural Disaster Research Urgent http://t.co/5Cm0LfZhxn via #JakartaPost\n",
      "\n",
      "---\n"
     ]
    }
   ],
   "source": [
    "# visualise some random training examples\n",
    "import random\n",
    "random_index = random.randint(0,len(train_df)-5)\n",
    "print(random_index)\n",
    "for row in train_df_shuffled[['text','target']][random_index:random_index+5].itertuples():\n",
    "  index,text,target = row\n",
    "  print(f\"Target: {target}, {'(real disaster)' if target > 0 else '(not a real disaster)'}\")\n",
    "  print(f\"Text:\\n{text}\\n\")\n",
    "  print(\"---\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "MSRuczOy1Hze",
    "outputId": "be2ce9c4-1f4f-40a5-f885-a15dcb36971c",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:28.438872828Z",
     "start_time": "2024-01-02T18:13:28.389782730Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "7613"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_df_shuffled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "5ilKeV5jwa0E",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:29.388403204Z",
     "start_time": "2024-01-02T18:13:29.325338061Z"
    }
   },
   "outputs": [],
   "source": [
    "#Splitting the data to create validation -> (train + validation)\n",
    "from sklearn.model_selection import train_test_split\n",
    "train_sentences, val_sentences, train_labels, val_labels = train_test_split(train_df_shuffled[\"text\"].to_numpy(),\n",
    "                                                                            train_df_shuffled[\"target\"].to_numpy(),\n",
    "                                                                            test_size=0.1,\n",
    "                                                                            random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4b2LtkKK08Rd",
    "outputId": "e485da7f-324c-432d-f666-fbb8abb214a9",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:30.303716331Z",
     "start_time": "2024-01-02T18:13:30.232061684Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "(6851, 6851, 762, 762)"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Check length\n",
    "len(train_sentences),len(train_labels),len(val_sentences),len(val_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "NqZ7B-jT1EHj",
    "outputId": "7601f438-27da-4eb1-b30a-0cdfa30f5378",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:31.341626318Z",
     "start_time": "2024-01-02T18:13:31.247998317Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "(array(['@mogacola @zamtriossu i screamed after hitting tweet',\n        'Imagine getting flattened by Kurt Zouma',\n        '@Gurmeetramrahim #MSGDoing111WelfareWorks Green S welfare force ke appx 65000 members har time disaster victim ki help ke liye tyar hai....',\n        \"@shakjn @C7 @Magnums im shaking in fear he's gonna hack the planet\",\n        'Somehow find you and I collide http://t.co/Ee8RpOahPk',\n        '@EvaHanderek @MarleyKnysh great times until the bus driver held us hostage in the mall parking lot lmfao',\n        'destroy the free fandom honestly',\n        'Weapons stolen from National Guard Armory in New Albany still missing #Gunsense http://t.co/lKNU8902JE',\n        '@wfaaweather Pete when will the heat wave pass? Is it really going to be mid month? Frisco Boy Scouts have a canoe trip in Okla.',\n        'Patient-reported outcomes in long-term survivors of metastatic colorectal cancer - British Journal of Surgery http://t.co/5Yl4DC1Tqt'],\n       dtype=object),\n array([0, 0, 1, 0, 0, 1, 1, 0, 1, 1]))"
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check few sample\n",
    "train_sentences[:10],train_labels[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "oHfc38t53GjO"
   },
   "source": [
    "## Converting text into numbers\n",
    "\n",
    "1. tokenization -> can get too big as the number of text sample increases\n",
    "2. Embedding -> richer representation of relationship be between tokens, deeper the realtion between words, more the number/embedding for that token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "yru6-5TJ1U17",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:35.277700752Z",
     "start_time": "2024-01-02T18:13:35.183617644Z"
    }
   },
   "outputs": [],
   "source": [
    "# Use the default TextVectorization variables\n",
    "text_vectorizer = tf.keras.layers.experimental.preprocessing.TextVectorization(max_tokens=None, # how many words in the vocabulary (all of the different words in your text)\n",
    "                                                    standardize=\"lower_and_strip_punctuation\", # how to process text\n",
    "                                                    split=\"whitespace\", # how to split tokens\n",
    "                                                    ngrams=None, # create groups of n-words?\n",
    "                                                    output_mode=\"int\", # how to map tokens to numbers\n",
    "                                                    output_sequence_length=None) # how long should the output sequence of tokens be?\n",
    "                                                    # pad_to_max_tokens=True) # Not valid if using max_tokens=None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3SQN6iwGtjGC",
    "outputId": "ca38bcb6-72ce-483b-c1fb-5091e5248bfb",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:37.254391106Z",
     "start_time": "2024-01-02T18:13:37.197770047Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "15"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "round(sum([len(i.split()) for i in train_sentences])/len(train_sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "w51rxf5MtjGD",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:38.092272843Z",
     "start_time": "2024-01-02T18:13:38.037068546Z"
    }
   },
   "outputs": [],
   "source": [
    "max_vocab_length = 10000 # max number of words to have in our vocabulary\n",
    "max_length = round(sum([len(i.split()) for i in train_sentences])/len(train_sentences)) # max length our sequences will be (e.g. how many words from a Tweet does our model see?)\n",
    "\n",
    "text_vectorizer = tf.keras.layers.experimental.preprocessing.TextVectorization(max_tokens=max_vocab_length,\n",
    "                                                    output_mode=\"int\",\n",
    "                                                    output_sequence_length=max_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "gaJKk7dOtjGD",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:38.912674765Z",
     "start_time": "2024-01-02T18:13:38.593003678Z"
    }
   },
   "outputs": [],
   "source": [
    "# Fit the text vectorizer to the training text\n",
    "text_vectorizer.adapt(train_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "B4aF2tYAtjGD",
    "outputId": "0a16e093-c8af-4310-88dd-54cc0a863cb5",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:39.206840215Z",
     "start_time": "2024-01-02T18:13:39.103515177Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "<tf.Tensor: shape=(1, 15), dtype=int64, numpy=\narray([[264,   3, 232,   4,  13, 698,   0,   0,   0,   0,   0,   0,   0,\n          0,   0]])>"
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_sentence = \"There's a flood in my street!\"\n",
    "text_vectorizer([sample_sentence])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ZKeTD6GutjGE",
    "outputId": "4f372239-d74d-42f5-932e-a5ebe156c027",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:40.871573201Z",
     "start_time": "2024-01-02T18:13:40.821396838Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original text:\n",
      "sorry-I built a fire by my desk already. RT@irishirr\n",
      "@MChapmanWSAZ @WSAZ_Brittany @kellyannwx ..please maintain that snowstorm til I arrive.      \n",
      "\n",
      "Vectorized version:\n"
     ]
    },
    {
     "data": {
      "text/plain": "<tf.Tensor: shape=(1, 15), dtype=int64, numpy=\narray([[8282, 2532,    3,   42,   18,   13, 5829,  659, 8938,    1, 6650,\n           1,  170, 5176,   16]])>"
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Choose a random sentence from the training dataset and tokenize it\n",
    "random_sentence = random.choice(train_sentences)\n",
    "print(f\"Original text:\\n{random_sentence}\\\n",
    "      \\n\\nVectorized version:\")\n",
    "text_vectorizer([random_sentence])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GtsN3urvtjGE",
    "outputId": "97ebf691-bb6a-458a-cda1-3cc06cb44824",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:41.628349985Z",
     "start_time": "2024-01-02T18:13:41.557369077Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of words in vocab: 10000\n",
      "Top 5 most common words: ['', '[UNK]', 'the', 'a', 'in']\n",
      "Bottom 5 least common words: ['pages', 'paeds', 'pads', 'padres', 'paddytomlinson1']\n"
     ]
    }
   ],
   "source": [
    "# Get the unique words in the vocabulary\n",
    "words_in_vocab = text_vectorizer.get_vocabulary()\n",
    "top_5_words = words_in_vocab[:5] # most common tokens (notice the [UNK] token for \"unknown\" words)\n",
    "bottom_5_words = words_in_vocab[-5:] # least common tokens\n",
    "print(f\"Number of words in vocab: {len(words_in_vocab)}\")\n",
    "print(f\"Top 5 most common words: {top_5_words}\")\n",
    "print(f\"Bottom 5 least common words: {bottom_5_words}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6RGDYiyvtjGE",
    "outputId": "84f988c4-612b-4eac-94e1-64d8708c1af8",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:44.789410132Z",
     "start_time": "2024-01-02T18:13:44.709569535Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "<keras.src.layers.core.embedding.Embedding at 0x7fbd91375b20>"
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.random.set_seed(42)\n",
    "\n",
    "embedding = tf.keras.layers.Embedding(input_dim=max_vocab_length, # set input shape\n",
    "                             output_dim=128, # set size of embedding vector\n",
    "                             embeddings_initializer=\"uniform\", # default, intialize randomly\n",
    "                             input_length=max_length, # how long is each input\n",
    "                             name=\"embedding_1\")\n",
    "\n",
    "embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "uzFIn5iltjGF",
    "outputId": "aa2584c6-00c4-41eb-ebf4-4c123d8b4190",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:47.832646979Z",
     "start_time": "2024-01-02T18:13:47.749726270Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original text:\n",
      "Men escape car engulfed in flames in Parley's Canyon crews investigating cause - http://t.co/P6cyLz5lpt http://t.co/Jpu9gIps9f\n",
      "\n",
      "Embedded version:\n"
     ]
    },
    {
     "data": {
      "text/plain": "<tf.Tensor: shape=(1, 15, 128), dtype=float16, numpy=\narray([[[ 0.0132  ,  0.02333 , -0.02858 , ..., -0.000325,  0.003576,\n          0.02396 ],\n        [ 0.02167 ,  0.0347  ,  0.004395, ..., -0.01881 ,  0.00251 ,\n         -0.04385 ],\n        [ 0.03732 ,  0.004936, -0.043   , ...,  0.03043 , -0.01987 ,\n          0.00462 ],\n        ...,\n        [-0.02518 , -0.0315  ,  0.005375, ...,  0.02472 ,  0.0458  ,\n          0.03207 ],\n        [-0.02518 , -0.0315  ,  0.005375, ...,  0.02472 ,  0.0458  ,\n          0.03207 ],\n        [ 0.04428 , -0.01811 , -0.03677 , ..., -0.00434 , -0.03442 ,\n          0.03038 ]]], dtype=float16)>"
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get a random sentence from training set\n",
    "random_sentence = random.choice(train_sentences)\n",
    "print(f\"Original text:\\n{random_sentence}\\n\\nEmbedded version:\")\n",
    "\n",
    "# Embed the random sentence (turn it into numerical representation)\n",
    "sample_embed = embedding(text_vectorizer([random_sentence]))\n",
    "sample_embed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1AqUrY6BtjGF",
    "outputId": "e15679cf-4c74-43bb-f30e-6c6b504bd56a",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:49.535794081Z",
     "start_time": "2024-01-02T18:13:49.467089350Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "<tf.Tensor: shape=(128,), dtype=float16, numpy=\narray([ 0.0132   ,  0.02333  , -0.02858  ,  0.013405 , -0.00931  ,\n        0.006397 , -0.007275 ,  0.04892  ,  0.01605  , -0.04016  ,\n        0.01293  ,  0.04242  ,  0.006145 ,  0.02667  ,  0.04367  ,\n       -0.03848  ,  0.04385  ,  0.0476   ,  0.01104  , -0.03287  ,\n       -0.0472   , -0.01473  , -0.0411   , -0.016    ,  0.0264   ,\n        0.003788 , -0.004604 ,  0.0425   , -0.03943  , -0.001997 ,\n       -0.004074 ,  0.04352  ,  0.00971  ,  0.02527  , -0.001353 ,\n        0.00958  ,  0.01845  ,  0.01639  , -0.01271  ,  0.04788  ,\n        0.02959  ,  0.009926 , -0.003658 ,  0.02151  , -0.02345  ,\n       -0.03302  , -0.04047  , -0.02643  , -0.01863  , -0.008965 ,\n        0.023    ,  0.0007024, -0.04236  ,  0.0329   ,  0.01283  ,\n       -0.02919  ,  0.0095   ,  0.00873  ,  0.04532  , -0.01164  ,\n        0.01819  ,  0.02547  ,  0.004017 ,  0.001263 , -0.0001249,\n       -0.002775 ,  0.0367   , -0.0378   ,  0.04968  , -0.03796  ,\n       -0.003475 ,  0.02608  , -0.0372   ,  0.04666  , -0.0174   ,\n        0.002422 ,  0.0273   ,  0.02051  ,  0.03705  ,  0.0415   ,\n        0.03745  , -0.01863  , -0.03647  , -0.00171  ,  0.02754  ,\n        0.01578  ,  0.01534  , -0.0486   ,  0.0378   ,  0.04987  ,\n        0.007793 ,  0.04205  ,  0.02496  ,  0.01215  ,  0.02634  ,\n       -0.02583  , -0.02142  , -0.0451   ,  0.0093   , -0.007793 ,\n        0.03677  ,  0.007317 ,  0.000469 ,  0.0451   ,  0.02003  ,\n        0.04117  ,  0.0306   ,  0.012955 ,  0.000395 ,  0.02356  ,\n        0.01593  ,  0.005413 , -0.02724  , -0.00583  , -0.004383 ,\n        0.0377   , -0.01727  , -0.04056  , -0.004417 ,  0.04095  ,\n        0.02643  ,  0.0194   ,  0.00849  , -0.03647  , -0.04956  ,\n       -0.000325 ,  0.003576 ,  0.02396  ], dtype=float16)>"
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check out a single token's embedding\n",
    "sample_embed[0][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Aj6lDiuttjGG"
   },
   "source": [
    "### Modelling a text dataset\n",
    "More specifically, we'll be building the following:\n",
    "\n",
    "* Model 0: Naive Bayes (baseline)\n",
    "* Model 1: Feed-forward neural network (dense model)\n",
    "* Model 2: LSTM model\n",
    "* Model 3: GRU model\n",
    "* Model 4: Bidirectional-LSTM model\n",
    "* Model 5: 1D Convolutional Neural Network\n",
    "* Model 6: TensorFlow Hub Pretrained Feature Extractor\n",
    "* Model 7: Same as model 6 with 10% of training data\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 124
    },
    "id": "DyFjLFNHtjGG",
    "outputId": "e013b740-42f3-4168-c5ed-7b799a13d049",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:51.234612425Z",
     "start_time": "2024-01-02T18:13:51.112800506Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "Pipeline(steps=[('tfidf', TfidfVectorizer()), ('clf', MultinomialNB())])",
      "text/html": "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>Pipeline(steps=[(&#x27;tfidf&#x27;, TfidfVectorizer()), (&#x27;clf&#x27;, MultinomialNB())])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" ><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">Pipeline</label><div class=\"sk-toggleable__content\"><pre>Pipeline(steps=[(&#x27;tfidf&#x27;, TfidfVectorizer()), (&#x27;clf&#x27;, MultinomialNB())])</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" ><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">TfidfVectorizer</label><div class=\"sk-toggleable__content\"><pre>TfidfVectorizer()</pre></div></div></div><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" ><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MultinomialNB</label><div class=\"sk-toggleable__content\"><pre>MultinomialNB()</pre></div></div></div></div></div></div></div>"
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#model 0\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "# Create tokenization and modelling pipeline\n",
    "model_0 = Pipeline([\n",
    "                    (\"tfidf\", TfidfVectorizer()), # convert words to numbers using tfidf\n",
    "                    (\"clf\", MultinomialNB()) # model the text\n",
    "])\n",
    "\n",
    "# Fit the pipeline to the training data\n",
    "model_0.fit(train_sentences, train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "h9FerVWktjGG",
    "outputId": "f0932376-c838-485e-9bd3-0aba314f39e8",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:53.122185022Z",
     "start_time": "2024-01-02T18:13:53.082266623Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Our baseline model achieves an accuracy of: 79.27%\n"
     ]
    }
   ],
   "source": [
    "baseline_score = model_0.score(val_sentences, val_labels)\n",
    "print(f\"Our baseline model achieves an accuracy of: {baseline_score*100:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2kqWtkC7tjGH",
    "outputId": "3bacea1b-a6c5-46ae-b364-008f4d9d3182",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:53.714843511Z",
     "start_time": "2024-01-02T18:13:53.672150949Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "array([1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1])"
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make predictions\n",
    "baseline_preds = model_0.predict(val_sentences)\n",
    "baseline_preds[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "QdZ2KDfNtjGH",
    "outputId": "2d0fd280-e66a-459f-ce80-d26b14915ddf",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:54.719092714Z",
     "start_time": "2024-01-02T18:13:54.616348388Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "{'accuracy': 79.26509186351706,\n 'precision': 0.8111390004213173,\n 'recall': 0.7926509186351706,\n 'f1': 0.7862189758049549}"
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baseline_results = calculate_results(y_true=val_labels,\n",
    "                                     y_pred=baseline_preds)\n",
    "baseline_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6awk4LietjGH"
   },
   "source": [
    "#### Model 1: A simple dense model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "5f0zTX6btjGH",
    "outputId": "417c4318-8301-4dcf-d837-d3c66339868d",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:13:58.843856545Z",
     "start_time": "2024-01-02T18:13:58.744979418Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_1_dense\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_1 (InputLayer)        [(None, 1)]               0         \n",
      "                                                                 \n",
      " text_vectorization_1 (Text  (None, 15)                0         \n",
      " Vectorization)                                                  \n",
      "                                                                 \n",
      " embedding_1 (Embedding)     (None, 15, 128)           1280000   \n",
      "                                                                 \n",
      " global_average_pooling1d (  (None, 128)               0         \n",
      " GlobalAveragePooling1D)                                         \n",
      "                                                                 \n",
      " dense (Dense)               (None, 1)                 129       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1280129 (4.88 MB)\n",
      "Trainable params: 1280129 (4.88 MB)\n",
      "Non-trainable params: 0 (0.00 Byte)\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from helper_functions import create_tensorboard_callback\n",
    "\n",
    "# Create directory to save TensorBoard logs\n",
    "SAVE_DIR = \"model_logs\"\n",
    "\n",
    "# Build model with the Functional API\n",
    "inputs = tf.keras.layers.Input(shape=(1,), dtype=\"string\") # inputs are 1-dimensional strings\n",
    "x = text_vectorizer(inputs) # turn the input text into numbers\n",
    "x = embedding(x) # create an embedding of the numerized numbers\n",
    "x = tf.keras.layers.GlobalAveragePooling1D()(x) # lower the dimensionality of the embedding\n",
    "outputs = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x) # create the output layer, want binary outputs so use sigmoid activation\n",
    "\n",
    "model_1 = tf.keras.Model(inputs, outputs, name=\"model_1_dense\") # construct the model\n",
    "# Compile model\n",
    "model_1.compile(loss=\"binary_crossentropy\",\n",
    "                optimizer=tf.keras.optimizers.Adam(),\n",
    "                metrics=[\"accuracy\"])\n",
    "model_1.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "mv1YWc1otjGI",
    "outputId": "fed8ba4c-a19a-4be3-e2b8-668c20860eb6",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:14:17.080036127Z",
     "start_time": "2024-01-02T18:14:02.541308270Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving TensorBoard log files to: model_logs/simple_dense_model/20240102-234402\n",
      "Epoch 1/5\n",
      "215/215 [==============================] - 9s 35ms/step - loss: 0.6101 - accuracy: 0.6925 - val_loss: 0.5373 - val_accuracy: 0.7546\n",
      "Epoch 2/5\n",
      "215/215 [==============================] - 2s 7ms/step - loss: 0.4423 - accuracy: 0.8197 - val_loss: 0.4697 - val_accuracy: 0.7848\n",
      "Epoch 3/5\n",
      "215/215 [==============================] - 1s 6ms/step - loss: 0.3474 - accuracy: 0.8600 - val_loss: 0.4595 - val_accuracy: 0.7887\n",
      "Epoch 4/5\n",
      "215/215 [==============================] - 1s 5ms/step - loss: 0.2858 - accuracy: 0.8908 - val_loss: 0.4644 - val_accuracy: 0.7887\n",
      "Epoch 5/5\n",
      "215/215 [==============================] - 1s 5ms/step - loss: 0.2389 - accuracy: 0.9124 - val_loss: 0.4767 - val_accuracy: 0.7874\n"
     ]
    }
   ],
   "source": [
    "# Fit the model\n",
    "model_1_history = model_1.fit(train_sentences, # input sentences can be a list of strings due to text preprocessing layer built-in model\n",
    "                              train_labels,\n",
    "                              epochs=5,\n",
    "                              validation_data=(val_sentences, val_labels),\n",
    "                              callbacks=[create_tensorboard_callback(dir_name=SAVE_DIR,\n",
    "                                                                     experiment_name=\"simple_dense_model\")])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ivJJWB79tjGI",
    "outputId": "dcdec048-9c02-4cf7-b0da-3ead736a8224",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:14:20.055070928Z",
     "start_time": "2024-01-02T18:14:19.946131098Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 0s 2ms/step - loss: 0.4767 - accuracy: 0.7874\n"
     ]
    },
    {
     "data": {
      "text/plain": "[0.47666311264038086, 0.787401556968689]"
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the results\n",
    "model_1.evaluate(val_sentences, val_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Qfc_N0-VtjGI",
    "outputId": "75c214a4-fa18-4045-f898-4b62cbe5cdaa",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:14:20.909258434Z",
     "start_time": "2024-01-02T18:14:20.737954859Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 0s 935us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": "array([[0.408  ],\n       [0.7456 ],\n       [0.998  ],\n       [0.10876],\n       [0.1069 ],\n       [0.936  ],\n       [0.911  ],\n       [0.993  ],\n       [0.9697 ],\n       [0.262  ]], dtype=float16)"
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_1_pred_probs = model_1.predict(val_sentences)\n",
    "model_1_pred_probs[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "s29LU2c0tjGI",
    "outputId": "d1352bd9-ba40-4a1f-86f8-2efa4a5fdda8",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:14:22.948934603Z",
     "start_time": "2024-01-02T18:14:22.878411163Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "<tf.Tensor: shape=(10,), dtype=float16, numpy=array([0., 1., 1., 0., 0., 1., 1., 1., 1., 0.], dtype=float16)>"
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_1_preds=tf.squeeze(tf.round(model_1_pred_probs))\n",
    "model_1_preds[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8AgR4I2-tjGI",
    "outputId": "c4c9a85d-2909-43e8-f18b-470a68543587",
    "ExecuteTime": {
     "end_time": "2024-01-02T18:14:23.364993867Z",
     "start_time": "2024-01-02T18:14:23.301790235Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "{'accuracy': 78.74015748031496,\n 'precision': 0.7914920592553047,\n 'recall': 0.7874015748031497,\n 'f1': 0.7846966492209201}"
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_1_results=calculate_results(val_labels,model_1_preds)\n",
    "model_1_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1RJQLxUytjGJ"
   },
   "source": [
    "#### Recurrent Neural Network\n",
    "* RNN's are useful for sequence data\n",
    "* The premise of a recurrent neural network is to use the representation of previous input to aid the representation of a later input"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SWPZ8qtXtjGJ"
   },
   "source": [
    "### Model 2 : LSTM -> Long short term memory\n",
    "\n",
    "Input(text) -> Toeknize -> Embedding -> Layers(RNN/Dense) -> Output(label probability)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2t8nHBbCtjGJ",
    "outputId": "906024b7-40ce-4bb2-ce64-70489e5126bf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(None, 15, 128)\n",
      "(None, 64)\n",
      "Model: \"model\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_2 (InputLayer)        [(None, 1)]               0         \n",
      "                                                                 \n",
      " text_vectorization_1 (Text  (None, 15)                0         \n",
      " Vectorization)                                                  \n",
      "                                                                 \n",
      " embedding_2 (Embedding)     (None, 15, 128)           1280000   \n",
      "                                                                 \n",
      " lstm (LSTM)                 (None, 15, 64)            49408     \n",
      "                                                                 \n",
      " lstm_1 (LSTM)               (None, 15, 64)            33024     \n",
      "                                                                 \n",
      " lstm_2 (LSTM)               (None, 64)                33024     \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 1)                 65        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1395521 (5.32 MB)\n",
      "Trainable params: 1395521 (5.32 MB)\n",
      "Non-trainable params: 0 (0.00 Byte)\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Set random seed and create embedding layer (new embedding layer for each model)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "model_2_embedding = tf.keras.layers.Embedding(input_dim=max_vocab_length,\n",
    "                                     output_dim=128,\n",
    "                                     embeddings_initializer=\"uniform\",\n",
    "                                     input_length=max_length,\n",
    "                                     name=\"embedding_2\")\n",
    "\n",
    "\n",
    "# Create LSTM model\n",
    "inputs = tf.keras.layers.Input(shape=(1,), dtype=\"string\")\n",
    "x = text_vectorizer(inputs)\n",
    "x = model_2_embedding(x)\n",
    "\n",
    "print(x.shape)\n",
    "\n",
    "x = tf.keras.layers.LSTM(64, return_sequences=True)(x) # return vector for each word in the Tweet (you can stack RNN cells as long as return_sequences=True)\n",
    "x = tf.keras.layers.LSTM(64, return_sequences=True)(x)\n",
    "x = tf.keras.layers.LSTM(64)(x) # return vector for whole sequence\n",
    "\n",
    "print(x.shape)\n",
    "\n",
    "outputs = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n",
    "\n",
    "model_2 = tf.keras.Model(inputs, outputs)\n",
    "model_2.compile(loss=tf.keras.losses.BinaryCrossentropy(),\n",
    "                optimizer=tf.keras.optimizers.Adam(),\n",
    "                metrics=[\"accuracy\"])\n",
    "model_2.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Azw9jRxStjGK",
    "outputId": "6460d488-c6b6-4090-e84e-c6a50b79bd33"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving TensorBoard log files to: model_logs/LSTM/20231226-124100\n",
      "Epoch 1/5\n",
      "215/215 [==============================] - 13s 42ms/step - loss: 0.5084 - accuracy: 0.7470 - val_loss: 0.4583 - val_accuracy: 0.7900\n",
      "Epoch 2/5\n",
      "215/215 [==============================] - 4s 21ms/step - loss: 0.3263 - accuracy: 0.8705 - val_loss: 0.5141 - val_accuracy: 0.7835\n",
      "Epoch 3/5\n",
      "215/215 [==============================] - 4s 21ms/step - loss: 0.2300 - accuracy: 0.9140 - val_loss: 0.5503 - val_accuracy: 0.7664\n",
      "Epoch 4/5\n",
      "215/215 [==============================] - 4s 19ms/step - loss: 0.1713 - accuracy: 0.9385 - val_loss: 0.6243 - val_accuracy: 0.7822\n",
      "Epoch 5/5\n",
      "215/215 [==============================] - 4s 19ms/step - loss: 0.1210 - accuracy: 0.9512 - val_loss: 0.8839 - val_accuracy: 0.7520\n"
     ]
    }
   ],
   "source": [
    "# Fit model\n",
    "model_2_history = model_2.fit(train_sentences, train_labels,\n",
    "                              epochs=5,\n",
    "                              validation_data=(val_sentences, val_labels),\n",
    "                              callbacks=[create_tensorboard_callback(SAVE_DIR,\n",
    "                                                                     \"LSTM\")])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "r_MP0RjUtjGK",
    "outputId": "ae5f5e48-10b5-4cb8-e1ec-f7595753522d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 1s 9ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.004738],\n",
       "       [0.7188  ],\n",
       "       [1.      ],\n",
       "       [0.096   ],\n",
       "       [0.002165],\n",
       "       [1.      ],\n",
       "       [0.63    ],\n",
       "       [1.      ],\n",
       "       [1.      ],\n",
       "       [0.1917  ]], dtype=float16)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_2_pred_probs = model_2.predict(val_sentences)\n",
    "model_2_pred_probs[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "fTqfqUvwtjGK",
    "outputId": "c5d9227c-d370-4974-92ce-4baf074a2e5f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(10,), dtype=float16, numpy=array([0., 1., 1., 0., 0., 1., 1., 1., 1., 0.], dtype=float16)>"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert preds to laels\n",
    "model_2_preds = tf.squeeze(tf.round(model_2_pred_probs))\n",
    "model_2_preds[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "YIrv9t95tjGL",
    "outputId": "0a62bce6-9174-4c22-cdd4-27d13b58fec2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, 1, 1, 1, 1, 1, 1, 0])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_labels[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "nminkRFZtjGL",
    "outputId": "e5750290-bc85-46cd-e3d3-e10c0ba344a7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 75.19685039370079,\n",
       " 'precision': 0.753957198024366,\n",
       " 'recall': 0.7519685039370079,\n",
       " 'f1': 0.7492070183396126}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_2_results = calculate_results(val_labels, model_2_preds)\n",
    "model_2_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BxUSQAa6tjGL"
   },
   "source": [
    "#### Model 3: GRU\n",
    "\n",
    "1. Gated recurrent unit is effective and popular\n",
    "2. GRU cell has similar features to LSTM cell but has less paramenter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2DY0h3NStjGL",
    "outputId": "d4304ded-f0d9-48dc-9a7f-cb60d40d821e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "215/215 [==============================] - 9s 35ms/step - loss: 0.2314 - accuracy: 0.9054 - val_loss: 0.5323 - val_accuracy: 0.7730\n",
      "Epoch 2/5\n",
      "215/215 [==============================] - 3s 13ms/step - loss: 0.1601 - accuracy: 0.9419 - val_loss: 0.5948 - val_accuracy: 0.7808\n",
      "Epoch 3/5\n",
      "215/215 [==============================] - 3s 13ms/step - loss: 0.1283 - accuracy: 0.9537 - val_loss: 0.7481 - val_accuracy: 0.7848\n",
      "Epoch 4/5\n",
      "215/215 [==============================] - 3s 12ms/step - loss: 0.1096 - accuracy: 0.9600 - val_loss: 0.7634 - val_accuracy: 0.7861\n",
      "Epoch 5/5\n",
      "215/215 [==============================] - 3s 13ms/step - loss: 0.0921 - accuracy: 0.9663 - val_loss: 0.7307 - val_accuracy: 0.7887\n"
     ]
    }
   ],
   "source": [
    "# Build GRU RNN\n",
    "inputs = tf.keras.layers.Input(shape=(1,),dtype=tf.string)\n",
    "x = text_vectorizer(inputs)\n",
    "x = embedding(x)\n",
    "# x = tf.keras.layers.GRU(64, return_sequences=True)(x)\n",
    "# x = tf.keras.layers.LSTM(64, return_sequences=True)(x)\n",
    "x = tf.keras.layers.GRU(64)(x)\n",
    "# x = tf.keras.layers.Dense(64,activation=\"relu\")(x)\n",
    "outputs = tf.keras.layers.Dense(1,activation=\"sigmoid\")(x)\n",
    "\n",
    "model_3 = tf.keras.Model(inputs,outputs)\n",
    "\n",
    "model_3.compile(loss=tf.keras.losses.BinaryCrossentropy(),\n",
    "                optimizer=tf.keras.optimizers.Adam(),\n",
    "                metrics=['accuracy'])\n",
    "model_3_history = model_3.fit(train_sentences,\n",
    "                        train_labels,\n",
    "                        epochs=5,\n",
    "                        validation_data=(val_sentences,val_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "IWN8EFWPtjGL",
    "outputId": "a9de7c8d-8700-456e-cf5b-67bd403133da"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 0s 4ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[6.268e-02],\n",
       "       [8.330e-01],\n",
       "       [1.000e+00],\n",
       "       [4.922e-02],\n",
       "       [8.326e-04],\n",
       "       [9.922e-01],\n",
       "       [6.738e-01],\n",
       "       [1.000e+00],\n",
       "       [9.990e-01],\n",
       "       [6.558e-01]], dtype=float16)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_3_pred_probs = model_3.predict(val_sentences)\n",
    "model_3_pred_probs[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "zLjHzeiotjGM",
    "outputId": "58557341-5730-45a9-a00b-c2d427cea740"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(10,), dtype=float16, numpy=array([0., 1., 1., 0., 0., 1., 1., 1., 1., 1.], dtype=float16)>"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Convert model 3 pred_probs to labels\n",
    "model_3_preds = tf.squeeze(tf.round(model_3_pred_probs))\n",
    "model_3_preds[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "sDGf2ItXtjGM",
    "outputId": "a0c89d2e-4790-4e43-817c-97594b154128"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 78.87139107611549,\n",
       " 'precision': 0.7912533510367916,\n",
       " 'recall': 0.7887139107611548,\n",
       " 'f1': 0.7866156427918749}"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_3_results = calculate_results(val_labels,model_3_preds)\n",
    "model_3_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iiei4KEUtjGM"
   },
   "source": [
    "#### Model4 - Bidirectional RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "UkpwO2rMtjGM",
    "outputId": "fe6469f5-707e-4296-a1f2-9026e1c08aa8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "215/215 [==============================] - 17s 57ms/step - loss: 0.5037 - accuracy: 0.7530 - val_loss: 0.4538 - val_accuracy: 0.7822\n",
      "Epoch 2/5\n",
      "215/215 [==============================] - 6s 27ms/step - loss: 0.3089 - accuracy: 0.8755 - val_loss: 0.5242 - val_accuracy: 0.7664\n",
      "Epoch 3/5\n",
      "215/215 [==============================] - 6s 26ms/step - loss: 0.2015 - accuracy: 0.9216 - val_loss: 0.5727 - val_accuracy: 0.7546\n",
      "Epoch 4/5\n",
      "215/215 [==============================] - 5s 24ms/step - loss: 0.1363 - accuracy: 0.9566 - val_loss: 0.6746 - val_accuracy: 0.7651\n",
      "Epoch 5/5\n",
      "215/215 [==============================] - 5s 24ms/step - loss: 0.1030 - accuracy: 0.9666 - val_loss: 0.6702 - val_accuracy: 0.7559\n"
     ]
    }
   ],
   "source": [
    "# Trying out on my own\n",
    "text_vectorizer_exp = tf.keras.layers.experimental.preprocessing.TextVectorization(max_tokens=10000,\n",
    "                                      output_sequence_length=15)\n",
    "text_vectorizer_exp.adapt(train_sentences)\n",
    "\n",
    "inputs = tf.keras.layers.Input(shape=(1,), dtype=tf.string)\n",
    "x = text_vectorizer_exp(inputs)\n",
    "x = tf.keras.layers.Embedding(input_dim=10000,\n",
    "                              output_dim=128,\n",
    "                              input_length=15)(x)\n",
    "x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True))(x)\n",
    "x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64))(x)\n",
    "outputs = tf.keras.layers.Dense(1,activation=tf.keras.activations.sigmoid)(x)\n",
    "\n",
    "model_4_exp = tf.keras.Model(inputs,outputs)\n",
    "model_4_exp.compile(loss=tf.keras.losses.BinaryCrossentropy(),\n",
    "                    optimizer=tf.keras.optimizers.Adam(),\n",
    "                    metrics=['accuracy'])\n",
    "model_4_history_exp = model_4_exp.fit(train_sentences,\n",
    "                                      train_labels,\n",
    "                                      validation_data=(val_sentences,val_labels),\n",
    "                                      epochs=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vlTNW7tZtjGN"
   },
   "source": [
    "Normal RNN's go from left to right\n",
    "\n",
    "Bi-directional go from right to left as well as left to right"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "uUJANmC_tjGN",
    "outputId": "4a36d2a6-be80-42ce-8139-14fb6c34468c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "215/215 [==============================] - 11s 39ms/step - loss: 0.5115 - accuracy: 0.7463 - val_loss: 0.4585 - val_accuracy: 0.7795\n",
      "Epoch 2/5\n",
      "215/215 [==============================] - 4s 17ms/step - loss: 0.3158 - accuracy: 0.8708 - val_loss: 0.5092 - val_accuracy: 0.7717\n",
      "Epoch 3/5\n",
      "215/215 [==============================] - 4s 17ms/step - loss: 0.2146 - accuracy: 0.9169 - val_loss: 0.5608 - val_accuracy: 0.7756\n",
      "Epoch 4/5\n",
      "215/215 [==============================] - 3s 16ms/step - loss: 0.1481 - accuracy: 0.9502 - val_loss: 0.6473 - val_accuracy: 0.7769\n",
      "Epoch 5/5\n",
      "215/215 [==============================] - 3s 16ms/step - loss: 0.1048 - accuracy: 0.9653 - val_loss: 0.6983 - val_accuracy: 0.7572\n"
     ]
    }
   ],
   "source": [
    "# model 4\n",
    "text_vectorizer = tf.keras.layers.experimental.preprocessing.TextVectorization(max_tokens=10000,\n",
    "                                                    output_sequence_length=15)\n",
    "text_vectorizer.adapt(train_sentences)\n",
    "\n",
    "inputs = tf.keras.layers.Input(shape=(1,),dtype=tf.string)\n",
    "x = text_vectorizer(inputs)\n",
    "x = tf.keras.layers.Embedding(input_dim=10000,\n",
    "                              output_dim=128,\n",
    "                              input_length=15)(x)\n",
    "# x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64,return_sequences=True))(x)\n",
    "x = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64))(x)\n",
    "outputs = tf.keras.layers.Dense(1,activation=tf.keras.activations.sigmoid)(x)\n",
    "\n",
    "model_4 = tf.keras.Model(inputs,outputs)\n",
    "\n",
    "model_4.compile(loss=tf.keras.losses.BinaryCrossentropy(),\n",
    "                optimizer=tf.keras.optimizers.Adam(),\n",
    "                metrics=['accuracy'])\n",
    "model_4_history = model_4.fit(train_sentences,train_labels,\n",
    "                              epochs=5,\n",
    "                              validation_data=(val_sentences,val_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "OwhOpMaftjGN",
    "outputId": "86d6e774-6814-4414-d2e6-c6a7957a234e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 1s 7ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.05728],\n",
       "       [0.93   ],\n",
       "       [1.     ],\n",
       "       [0.1604 ],\n",
       "       [0.00715],\n",
       "       [0.999  ],\n",
       "       [0.994  ],\n",
       "       [1.     ],\n",
       "       [1.     ],\n",
       "       [0.2161 ]], dtype=float16)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_4_pred_probs = model_4.predict(val_sentences)\n",
    "model_4_pred_probs[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "v5phUteQtjGN",
    "outputId": "5a05ce56-5454-412c-bf94-5d0f39204490"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(10,), dtype=float16, numpy=array([0., 1., 1., 0., 0., 1., 1., 1., 1., 0.], dtype=float16)>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_4_preds = tf.squeeze(tf.round(model_4_pred_probs))\n",
    "model_4_preds[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Ko8xUiUjtjGN",
    "outputId": "0ce32483-0eee-49e8-ad65-22dfc5b730c4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 75.7217847769029,\n",
       " 'precision': 0.7567936195399113,\n",
       " 'recall': 0.7572178477690289,\n",
       " 'f1': 0.7567837670446225}"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_4_results = calculate_results(val_labels,model_4_preds)\n",
    "model_4_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YdMFRt5xtjGO"
   },
   "source": [
    "#### 1D CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "MSfwD4FEtjGO",
    "outputId": "6f5bc81d-7630-4960-cca2-2152ddcda99d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(TensorShape([1, 15, 128]), TensorShape([1, 11, 32]), TensorShape([1, 32]))"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_test = embedding(text_vectorizer([\"this is a test sentence\"]))\n",
    "conv_1d = tf.keras.layers.Conv1D(32,\n",
    "                                 5,\n",
    "                                 activation=\"relu\",\n",
    "                                 padding=\"valid\")\n",
    "conv_1d_output = conv_1d(embedding_test)\n",
    "\n",
    "max_pool = tf.keras.layers.GlobalMaxPool1D()\n",
    "max_pool_output = max_pool(conv_1d_output)\n",
    "\n",
    "embedding_test.shape, conv_1d_output.shape, max_pool_output.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4wNjIjfptjGO",
    "outputId": "dea458b8-91d8-4b9d-96c1-11ff2b0a5ef6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1, 15, 128), dtype=float16, numpy=\n",
       "array([[[-0.0741   ,  0.005177 , -0.000992 , ..., -0.0313   ,\n",
       "         -0.02364  , -0.003284 ],\n",
       "        [-0.01328  ,  0.0468   ,  0.03372  , ...,  0.004246 ,\n",
       "         -0.0791   ,  0.0451   ],\n",
       "        [-0.05307  ,  0.0008225, -0.0288   , ..., -0.01764  ,\n",
       "         -0.052    , -0.04272  ],\n",
       "        ...,\n",
       "        [-0.04318  , -0.01334  ,  0.01537  , ..., -0.0363   ,\n",
       "          0.003754 ,  0.03366  ],\n",
       "        [-0.04318  , -0.01334  ,  0.01537  , ..., -0.0363   ,\n",
       "          0.003754 ,  0.03366  ],\n",
       "        [-0.04318  , -0.01334  ,  0.01537  , ..., -0.0363   ,\n",
       "          0.003754 ,  0.03366  ]]], dtype=float16)>"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "v4-9Q5oltjGO",
    "outputId": "3edd2277-0503-4274-90e2-ead0fd201bc2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1, 11, 32), dtype=float16, numpy=\n",
       "array([[[0.04773  , 0.       , 0.       , 0.01585  , 0.01717  ,\n",
       "         0.       , 0.       , 0.       , 0.       , 0.04947  ,\n",
       "         0.0343   , 0.02242  , 0.0529   , 0.03488  , 0.       ,\n",
       "         0.       , 0.0937   , 0.       , 0.0464   , 0.       ,\n",
       "         0.0418   , 0.0121   , 0.       , 0.       , 0.       ,\n",
       "         0.       , 0.03014  , 0.0467   , 0.01233  , 0.05942  ,\n",
       "         0.05054  , 0.       ],\n",
       "        [0.06903  , 0.07794  , 0.       , 0.       , 0.       ,\n",
       "         0.       , 0.02995  , 0.       , 0.       , 0.       ,\n",
       "         0.02446  , 0.03668  , 0.02255  , 0.01805  , 0.       ,\n",
       "         0.       , 0.08594  , 0.005306 , 0.06604  , 0.02658  ,\n",
       "         0.04605  , 0.0326   , 0.       , 0.       , 0.       ,\n",
       "         0.       , 0.       , 0.01444  , 0.       , 0.06137  ,\n",
       "         0.       , 0.000513 ],\n",
       "        [0.       , 0.       , 0.       , 0.       , 0.       ,\n",
       "         0.01653  , 0.01924  , 0.       , 0.       , 0.0754   ,\n",
       "         0.       , 0.01558  , 0.02017  , 0.       , 0.       ,\n",
       "         0.       , 0.       , 0.0829   , 0.0536   , 0.       ,\n",
       "         0.02849  , 0.       , 0.       , 0.0852   , 0.       ,\n",
       "         0.001479 , 0.       , 0.0682   , 0.       , 0.       ,\n",
       "         0.0342   , 0.       ],\n",
       "        [0.       , 0.04916  , 0.       , 0.013054 , 0.       ,\n",
       "         0.01003  , 0.005295 , 0.       , 0.       , 0.05112  ,\n",
       "         0.02557  , 0.0431   , 0.       , 0.       , 0.02309  ,\n",
       "         0.0332   , 0.002678 , 0.07916  , 0.03488  , 0.       ,\n",
       "         0.10034  , 0.02223  , 0.       , 0.       , 0.       ,\n",
       "         0.02728  , 0.       , 0.0512   , 0.03336  , 0.0356   ,\n",
       "         0.       , 0.       ],\n",
       "        [0.       , 0.002888 , 0.00931  , 0.01616  , 0.       ,\n",
       "         0.       , 0.0003934, 0.02446  , 0.03137  , 0.02534  ,\n",
       "         0.       , 0.01682  , 0.03915  , 0.       , 0.       ,\n",
       "         0.       , 0.       , 0.01862  , 0.0676   , 0.       ,\n",
       "         0.05722  , 0.01216  , 0.       , 0.00977  , 0.       ,\n",
       "         0.       , 0.       , 0.062    , 0.       , 0.       ,\n",
       "         0.       , 0.       ],\n",
       "        [0.       , 0.002754 , 0.       , 0.01297  , 0.       ,\n",
       "         0.       , 0.0288   , 0.       , 0.00359  , 0.03244  ,\n",
       "         0.       , 0.       , 0.03534  , 0.       , 0.01302  ,\n",
       "         0.       , 0.       , 0.01262  , 0.03464  , 0.       ,\n",
       "         0.04715  , 0.       , 0.       , 0.0407   , 0.01263  ,\n",
       "         0.       , 0.01881  , 0.03992  , 0.       , 0.       ,\n",
       "         0.       , 0.       ],\n",
       "        [0.       , 0.002754 , 0.       , 0.01297  , 0.       ,\n",
       "         0.       , 0.0288   , 0.       , 0.00359  , 0.03244  ,\n",
       "         0.       , 0.       , 0.03534  , 0.       , 0.01302  ,\n",
       "         0.       , 0.       , 0.01262  , 0.03464  , 0.       ,\n",
       "         0.04715  , 0.       , 0.       , 0.0407   , 0.01263  ,\n",
       "         0.       , 0.01881  , 0.03992  , 0.       , 0.       ,\n",
       "         0.       , 0.       ],\n",
       "        [0.       , 0.002754 , 0.       , 0.01297  , 0.       ,\n",
       "         0.       , 0.0288   , 0.       , 0.00359  , 0.03244  ,\n",
       "         0.       , 0.       , 0.03534  , 0.       , 0.01302  ,\n",
       "         0.       , 0.       , 0.01262  , 0.03464  , 0.       ,\n",
       "         0.04715  , 0.       , 0.       , 0.0407   , 0.01263  ,\n",
       "         0.       , 0.01881  , 0.03992  , 0.       , 0.       ,\n",
       "         0.       , 0.       ],\n",
       "        [0.       , 0.002754 , 0.       , 0.01297  , 0.       ,\n",
       "         0.       , 0.0288   , 0.       , 0.00359  , 0.03244  ,\n",
       "         0.       , 0.       , 0.03534  , 0.       , 0.01302  ,\n",
       "         0.       , 0.       , 0.01262  , 0.03464  , 0.       ,\n",
       "         0.04715  , 0.       , 0.       , 0.0407   , 0.01263  ,\n",
       "         0.       , 0.01881  , 0.03992  , 0.       , 0.       ,\n",
       "         0.       , 0.       ],\n",
       "        [0.       , 0.002754 , 0.       , 0.01297  , 0.       ,\n",
       "         0.       , 0.0288   , 0.       , 0.00359  , 0.03244  ,\n",
       "         0.       , 0.       , 0.03534  , 0.       , 0.01302  ,\n",
       "         0.       , 0.       , 0.01262  , 0.03464  , 0.       ,\n",
       "         0.04715  , 0.       , 0.       , 0.0407   , 0.01263  ,\n",
       "         0.       , 0.01881  , 0.03992  , 0.       , 0.       ,\n",
       "         0.       , 0.       ],\n",
       "        [0.       , 0.002754 , 0.       , 0.01297  , 0.       ,\n",
       "         0.       , 0.0288   , 0.       , 0.00359  , 0.03244  ,\n",
       "         0.       , 0.       , 0.03534  , 0.       , 0.01302  ,\n",
       "         0.       , 0.       , 0.01262  , 0.03464  , 0.       ,\n",
       "         0.04715  , 0.       , 0.       , 0.0407   , 0.01263  ,\n",
       "         0.       , 0.01881  , 0.03992  , 0.       , 0.       ,\n",
       "         0.       , 0.       ]]], dtype=float16)>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conv_1d_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "V5Q9sl3LtjGO",
    "outputId": "f20dc0a6-2ae9-4213-f96a-17c2773812e8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1, 32), dtype=float16, numpy=\n",
       "array([[0.06903 , 0.07794 , 0.00931 , 0.01616 , 0.01717 , 0.01653 ,\n",
       "        0.02995 , 0.02446 , 0.03137 , 0.0754  , 0.0343  , 0.0431  ,\n",
       "        0.0529  , 0.03488 , 0.02309 , 0.0332  , 0.0937  , 0.0829  ,\n",
       "        0.0676  , 0.02658 , 0.10034 , 0.0326  , 0.      , 0.0852  ,\n",
       "        0.01263 , 0.02728 , 0.03014 , 0.0682  , 0.03336 , 0.06137 ,\n",
       "        0.05054 , 0.000513]], dtype=float16)>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_pool_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "58U70HiRtjGO",
    "outputId": "32e28f7a-700b-4813-8366-981cf4f2d7da"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "215/215 [==============================] - 10s 37ms/step - loss: 0.1587 - accuracy: 0.9461 - val_loss: 0.7339 - val_accuracy: 0.7900\n",
      "Epoch 2/5\n",
      "215/215 [==============================] - 2s 11ms/step - loss: 0.1035 - accuracy: 0.9618 - val_loss: 0.8726 - val_accuracy: 0.7782\n",
      "Epoch 3/5\n",
      "215/215 [==============================] - 3s 12ms/step - loss: 0.0810 - accuracy: 0.9691 - val_loss: 0.9544 - val_accuracy: 0.7782\n",
      "Epoch 4/5\n",
      "215/215 [==============================] - 3s 14ms/step - loss: 0.0699 - accuracy: 0.9734 - val_loss: 1.0347 - val_accuracy: 0.7730\n",
      "Epoch 5/5\n",
      "215/215 [==============================] - 2s 11ms/step - loss: 0.0608 - accuracy: 0.9752 - val_loss: 1.1027 - val_accuracy: 0.7743\n"
     ]
    }
   ],
   "source": [
    "# Create 1D CNN layer to model sequences\n",
    "inputs = tf.keras.layers.Input(shape=(1,),dtype=tf.string)\n",
    "x = text_vectorizer(inputs)\n",
    "x = embedding(x)\n",
    "x = tf.keras.layers.Conv1D(filters=64,\n",
    "                           kernel_size=5,\n",
    "                           padding=\"valid\",\n",
    "                           activation=\"relu\")(x)\n",
    "x = tf.keras.layers.GlobalMaxPool1D()(x)\n",
    "outputs = tf.keras.layers.Dense(1,activation=tf.keras.activations.sigmoid)(x)\n",
    "\n",
    "model_5 = tf.keras.Model(inputs,outputs)\n",
    "\n",
    "model_5.compile(loss=tf.keras.losses.BinaryCrossentropy(),\n",
    "                optimizer=tf.keras.optimizers.Adam(),\n",
    "                metrics=[\"accuracy\"])\n",
    "model_5_history = model_5.fit(train_sentences,train_labels,\n",
    "                              validation_data=(val_sentences,val_labels),\n",
    "                              epochs=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3M13vl1ltjGP",
    "outputId": "23b5a4d4-d03c-415a-ba13-70082ea14fcc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 0s 3ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.2205 ],\n",
       "       [0.611  ],\n",
       "       [1.     ],\n",
       "       [0.05115],\n",
       "       [0.     ],\n",
       "       [0.998  ],\n",
       "       [0.992  ],\n",
       "       [1.     ],\n",
       "       [1.     ],\n",
       "       [0.683  ]], dtype=float16)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_5_pred_probs = model_5.predict(val_sentences)\n",
    "model_5_pred_probs[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Xpfg5D_xtjGP",
    "outputId": "88ae1738-5382-4107-bcd7-4d21a4d52988"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(10,), dtype=float16, numpy=array([0., 1., 1., 0., 0., 1., 1., 1., 1., 1.], dtype=float16)>"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_5_preds = tf.squeeze(tf.round(model_5_pred_probs))\n",
    "model_5_preds[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Rbf3XnmItjGP",
    "outputId": "98521aad-97a4-4672-ea2a-1ee7ba237747"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 77.42782152230971,\n",
       " 'precision': 0.7752857985262857,\n",
       " 'recall': 0.7742782152230971,\n",
       " 'f1': 0.7725974162749719}"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_5_results = calculate_results(val_labels,model_5_preds)\n",
    "model_5_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "EIi9WX7VtjGP",
    "outputId": "2aa8205c-7211-46e4-d2a7-d968827af8e3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 79.26509186351706,\n",
       " 'precision': 0.8111390004213173,\n",
       " 'recall': 0.7926509186351706,\n",
       " 'f1': 0.7862189758049549}"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baseline_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Nw5JS3dNtjGP"
   },
   "source": [
    "#### Model 6 - Tensorflow Hub pretrained models\n",
    "USE-extractor - universal sentence encoder (USE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "RY_WYxmotjGQ",
    "outputId": "aec40bef-df08-4c36-e0f1-fe334179c1d5"
   },
   "outputs": [
    {
     "ename": "",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31mThe Kernel crashed while executing code in the the current cell or a previous cell. Please review the code in the cell(s) to identify a possible cause of the failure. Click <a href='https://aka.ms/vscodeJupyterKernelCrash'>here</a> for more info. View Jupyter <a href='command:jupyter.viewOutput'>log</a> for further details."
     ]
    }
   ],
   "source": [
    "import tensorflow_hub as hub\n",
    "embed = hub.load(\"https://tfhub.dev/google/universal-sentence-encoder/4\")\n",
    "embed_sample = embed([sample_sentence,\n",
    "                      \"When you can the unviersal sentence encider on a sentence, it turns into into numbers\"])\n",
    "print(embed_sample[0][:50])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "KFo46Hy2tjGQ",
    "outputId": "ad64f62e-5f4e-4eec-9d4b-8d9156390a20"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 512), dtype=float32, numpy=\n",
       "array([[-0.01157027,  0.0248591 ,  0.02878048, ..., -0.00186125,\n",
       "         0.02315824, -0.01485021],\n",
       "       [ 0.06832423, -0.08312771, -0.00292901, ..., -0.01492013,\n",
       "         0.01456988,  0.03475707]], dtype=float32)>"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embed_sample"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cLySGe8ctjGQ"
   },
   "source": [
    "Use the above USE\"\" embedding layer for a normal type of model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "O6FvIAsOtjGQ"
   },
   "outputs": [],
   "source": [
    "# Create a keras layer using the USE\"\" pretrained layer from tensorflow hub\n",
    "sentence_encoder_layer = hub.KerasLayer(embed,\n",
    "                                        input_shape=[],\n",
    "                                        dtype=tf.string,\n",
    "                                        trainable=False,\n",
    "                                        name=\"USE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "JAedK16ztjGQ",
    "outputId": "c103f610-6993-42d9-f460-edfe236e6873"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow_hub.keras_layer.KerasLayer at 0x7f485ad76c10>"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_encoder_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "OASRLNAatjGQ",
    "outputId": "25116497-54c1-4174-d5b3-f493a4abce65"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "215/215 [==============================] - 8s 26ms/step - loss: 0.6515 - accuracy: 0.7273 - val_loss: 0.6173 - val_accuracy: 0.7782\n",
      "Epoch 2/5\n",
      "215/215 [==============================] - 5s 23ms/step - loss: 0.5834 - accuracy: 0.7902 - val_loss: 0.5668 - val_accuracy: 0.7861\n",
      "Epoch 3/5\n",
      "215/215 [==============================] - 5s 22ms/step - loss: 0.5398 - accuracy: 0.7938 - val_loss: 0.5343 - val_accuracy: 0.7874\n",
      "Epoch 4/5\n",
      "215/215 [==============================] - 5s 22ms/step - loss: 0.5107 - accuracy: 0.7981 - val_loss: 0.5125 - val_accuracy: 0.7887\n",
      "Epoch 5/5\n",
      "215/215 [==============================] - 5s 23ms/step - loss: 0.4904 - accuracy: 0.8006 - val_loss: 0.4977 - val_accuracy: 0.7927\n"
     ]
    }
   ],
   "source": [
    "# Create  sequential layer\n",
    "model_6 = tf.keras.models.Sequential([\n",
    "    sentence_encoder_layer,\n",
    "    tf.keras.layers.Dense(1,activation=\"sigmoid\")\n",
    "])\n",
    "\n",
    "model_6.compile(loss=tf.keras.losses.BinaryCrossentropy(),\n",
    "                optimizer=tf.keras.optimizers.Adam(),\n",
    "                metrics=['accuracy'])\n",
    "model_6_history = model_6.fit(train_sentences,train_labels,\n",
    "                               validation_data=(val_sentences,val_labels),\n",
    "                               epochs=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "BjeiQMmltjGR",
    "outputId": "cf152ceb-f11a-4104-c626-c4bf5079e6b9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 1s 16ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.3652],\n",
       "       [0.678 ],\n",
       "       [0.851 ],\n",
       "       [0.3484],\n",
       "       [0.64  ],\n",
       "       [0.726 ],\n",
       "       [0.8286],\n",
       "       [0.8306],\n",
       "       [0.744 ],\n",
       "       [0.2063]], dtype=float16)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_6_pred_probs=model_6.predict(val_sentences)\n",
    "model_6_pred_probs[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "d-G9SxEMtjGR",
    "outputId": "01a34024-b608-4ecc-a0a0-6432518f9a61"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(10,), dtype=float16, numpy=array([0., 1., 1., 0., 1., 1., 1., 1., 1., 0.], dtype=float16)>"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_6_preds=tf.squeeze(tf.round(model_6_pred_probs))\n",
    "model_6_preds[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ESiAF9QWtjGR",
    "outputId": "0ee32816-b196-40e7-d513-c6d990af24b1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 79.26509186351706,\n",
       " 'precision': 0.7935978195033314,\n",
       " 'recall': 0.7926509186351706,\n",
       " 'f1': 0.791315716217291}"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_6_results = calculate_results(val_labels,model_6_preds)\n",
    "model_6_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "LV51qBK-tjGR",
    "outputId": "af79e6fc-1435-486c-9d48-5401c2fd9847"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 79.26509186351706,\n",
       " 'precision': 0.8111390004213173,\n",
       " 'recall': 0.7926509186351706,\n",
       " 'f1': 0.7862189758049549}"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baseline_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "SuxGYt3YtjGR",
    "outputId": "554e196d-669e-49a3-b35e-cef94d718d4a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "215/215 [==============================] - 7s 27ms/step - loss: 0.4870 - accuracy: 0.7894 - val_loss: 0.4344 - val_accuracy: 0.8045\n",
      "Epoch 2/5\n",
      "215/215 [==============================] - 5s 25ms/step - loss: 0.4017 - accuracy: 0.8209 - val_loss: 0.4213 - val_accuracy: 0.8176\n",
      "Epoch 3/5\n",
      "215/215 [==============================] - 5s 24ms/step - loss: 0.3812 - accuracy: 0.8335 - val_loss: 0.4217 - val_accuracy: 0.8215\n",
      "Epoch 4/5\n",
      "215/215 [==============================] - 5s 23ms/step - loss: 0.3614 - accuracy: 0.8460 - val_loss: 0.4194 - val_accuracy: 0.8176\n",
      "Epoch 5/5\n",
      "215/215 [==============================] - 5s 23ms/step - loss: 0.3374 - accuracy: 0.8551 - val_loss: 0.4238 - val_accuracy: 0.8228\n",
      "24/24 [==============================] - 1s 18ms/step\n",
      "[[0.1592 ]\n",
      " [0.7734 ]\n",
      " [0.997  ]\n",
      " [0.255  ]\n",
      " [0.7285 ]\n",
      " [0.866  ]\n",
      " [0.993  ]\n",
      " [0.993  ]\n",
      " [0.968  ]\n",
      " [0.05634]]\n",
      "tf.Tensor([0. 1. 1. 0. 1. 1. 1. 1. 1. 0.], shape=(10,), dtype=float16)\n",
      "{'accuracy': 82.28346456692913, 'precision': 0.8253135610622478, 'recall': 0.8228346456692913, 'f1': 0.8213746807704705}\n"
     ]
    }
   ],
   "source": [
    "# Challenge, beat the baseline\n",
    "model_6_exp = tf.keras.models.Sequential([\n",
    "    sentence_encoder_layer,\n",
    "    tf.keras.layers.Dense(64,activation=tf.keras.activations.relu),\n",
    "    tf.keras.layers.Dense(64,activation=tf.keras.activations.relu),\n",
    "    tf.keras.layers.Dense(1,activation=tf.keras.activations.sigmoid)\n",
    "])\n",
    "model_6_exp.compile(loss=tf.keras.losses.BinaryCrossentropy(),\n",
    "                    optimizer=tf.keras.optimizers.Adam(),\n",
    "                    metrics=['accuracy'])\n",
    "model_6_exp_history = model_6_exp.fit(train_sentences,train_labels,\n",
    "                                      epochs=5,\n",
    "                                      validation_data=(val_sentences,val_labels))\n",
    "\n",
    "model_6_exp_pred_probs = model_6_exp.predict(val_sentences)\n",
    "print(model_6_exp_pred_probs[:10])\n",
    "\n",
    "model_6_exp_preds = tf.squeeze(tf.round(model_6_exp_pred_probs))\n",
    "print(model_6_exp_preds[:10])\n",
    "\n",
    "model_6_exp_results = calculate_results(val_labels, model_6_exp_preds)\n",
    "print(model_6_exp_results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8qAovHDDtjGR"
   },
   "source": [
    "### Model 7 : TF Hub Pretrained USE but with 10% of training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "necgC53RtjGR",
    "outputId": "33c8a400-fb3c-4d29-eec7-52d2704f64c6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(761, 761)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_10_percent = train_df_shuffled[[\"text\", \"target\"]].sample(frac=0.1, random_state=42)\n",
    "train_sentences_10_percent = train_10_percent[\"text\"].to_list()\n",
    "train_labels_10_percent = train_10_percent[\"target\"].to_list()\n",
    "len(train_sentences_10_percent), len(train_labels_10_percent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "CXPVFKZ_tjGS",
    "outputId": "a730ee4e-4a31-407e-c7e9-9a4f7f097001"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "target\n",
       "0    413\n",
       "1    348\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the number of targets in the dataset\n",
    "train_10_percent[\"target\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ZuqFlmmstjGS",
    "outputId": "33f7a1f9-50ca-434d-e074-7e7191698052"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "target\n",
       "0    4342\n",
       "1    3271\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df_shuffled[\"target\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "VnvNm0cVtjGS",
    "outputId": "496c2744-282f-495a-ff86-cc5017408758"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "24/24 [==============================] - 3s 59ms/step - loss: 0.6709 - accuracy: 0.6321 - val_loss: 0.6266 - val_accuracy: 0.7795\n",
      "Epoch 2/5\n",
      "24/24 [==============================] - 1s 43ms/step - loss: 0.5695 - accuracy: 0.8095 - val_loss: 0.4916 - val_accuracy: 0.8320\n",
      "Epoch 3/5\n",
      "24/24 [==============================] - 1s 43ms/step - loss: 0.4493 - accuracy: 0.8252 - val_loss: 0.3955 - val_accuracy: 0.8386\n",
      "Epoch 4/5\n",
      "24/24 [==============================] - 1s 42ms/step - loss: 0.3795 - accuracy: 0.8436 - val_loss: 0.3376 - val_accuracy: 0.8570\n",
      "Epoch 5/5\n",
      "24/24 [==============================] - 1s 38ms/step - loss: 0.3296 - accuracy: 0.8647 - val_loss: 0.2903 - val_accuracy: 0.8832\n"
     ]
    }
   ],
   "source": [
    "# model 7\n",
    "model_7 = tf.keras.models.Sequential([\n",
    "    sentence_encoder_layer,\n",
    "    tf.keras.layers.Dense(64,activation=tf.keras.activations.relu),\n",
    "    tf.keras.layers.Dense(64,activation=tf.keras.activations.relu),\n",
    "    tf.keras.layers.Dense(1,activation=tf.keras.activations.sigmoid)\n",
    "])\n",
    "\n",
    "model_7.compile(loss=tf.keras.losses.BinaryCrossentropy(),\n",
    "                optimizer=tf.keras.optimizers.Adam(),\n",
    "                metrics=[\"accuracy\"])\n",
    "model_7_history = model_7.fit(train_sentences_10_percent, train_labels_10_percent,\n",
    "                              epochs=5,\n",
    "                              validation_data=(val_sentences,val_labels))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "PxCRfzkctjGS",
    "outputId": "83720290-d2dc-4459-c85e-37dc794e869f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 1s 16ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0.12305],\n",
       "       [0.8613 ],\n",
       "       [0.9604 ],\n",
       "       [0.2306 ],\n",
       "       [0.8906 ],\n",
       "       [0.869  ],\n",
       "       [0.957  ],\n",
       "       [0.9727 ],\n",
       "       [0.9233 ],\n",
       "       [0.00986]], dtype=float16)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_7_pred_probs = model_7.predict(val_sentences)\n",
    "model_7_pred_probs[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "N_OxLFYRtjGS",
    "outputId": "fb4a2a86-a84a-45de-bdff-57941b0be5c9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(10,), dtype=float16, numpy=array([0., 1., 1., 0., 1., 1., 1., 1., 1., 0.], dtype=float16)>"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_7_preds = tf.squeeze(tf.round(model_7_pred_probs))\n",
    "model_7_preds[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Hy-jIXjdtjGS",
    "outputId": "544b5f5f-7d58-4911-db1d-bedfc27d60c5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 88.32020997375328,\n",
       " 'precision': 0.8836591055325493,\n",
       " 'recall': 0.8832020997375328,\n",
       " 'f1': 0.8828731455017578}"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_7_results = calculate_results(val_labels,model_7_preds)\n",
    "model_7_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xhrRtbvptjGT"
   },
   "source": [
    "model 7 did very good compared to model 6, model is same, but only 10% data, why?\n",
    "what could the reason be:\n",
    "1. maybe the way we created 10% data\n",
    "\n",
    "ANS : train_sentences_10_percent has little amount of val_sentences, becuase we took 10% from whole train_df_shuffled variable\n",
    "\n",
    "So, it is already seeing what it will be validating on.\n",
    "\n",
    "This is called data leak problem, some part of validation/testing data leaks into training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "r9kD3j83tjGT",
    "outputId": "d1e7f5f8-e12e-431c-ab60-193fb68cc3e2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(685, 685)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Making correct data allocation - without data leakage\n",
    "train_10_percent_split = int(0.1 * len(train_sentences))    # train_sentences are already in random order, no need to shuffle\n",
    "train_sentence_10_percent = train_sentences[:train_10_percent_split]\n",
    "train_labels_10_percent = train_labels[:train_10_percent_split]\n",
    "len(train_sentence_10_percent), len(train_labels_10_percent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "sGZURzCxtjGT",
    "outputId": "be1bf559-735d-45bf-cd5c-756489718cc1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "22/22 [==============================] - 3s 63ms/step - loss: 0.6722 - accuracy: 0.6409 - val_loss: 0.6491 - val_accuracy: 0.6640\n",
      "Epoch 2/5\n",
      "22/22 [==============================] - 1s 45ms/step - loss: 0.5754 - accuracy: 0.7883 - val_loss: 0.5460 - val_accuracy: 0.7703\n",
      "Epoch 3/5\n",
      "22/22 [==============================] - 1s 46ms/step - loss: 0.4454 - accuracy: 0.8219 - val_loss: 0.4890 - val_accuracy: 0.7795\n",
      "Epoch 4/5\n",
      "22/22 [==============================] - 1s 44ms/step - loss: 0.3749 - accuracy: 0.8409 - val_loss: 0.4983 - val_accuracy: 0.7887\n",
      "Epoch 5/5\n",
      "22/22 [==============================] - 1s 38ms/step - loss: 0.3309 - accuracy: 0.8642 - val_loss: 0.5071 - val_accuracy: 0.7703\n"
     ]
    }
   ],
   "source": [
    "model_7 = tf.keras.models.Sequential([\n",
    "    sentence_encoder_layer,\n",
    "    tf.keras.layers.Dense(64,activation=tf.keras.activations.relu),\n",
    "    tf.keras.layers.Dense(64,activation=tf.keras.activations.relu),\n",
    "    tf.keras.layers.Dense(1,activation=tf.keras.activations.sigmoid)\n",
    "])\n",
    "model_7.compile(loss=tf.keras.losses.BinaryCrossentropy(),\n",
    "                optimizer=tf.keras.optimizers.Adam(),\n",
    "                metrics=['accuracy'])\n",
    "mdoel_7_history_split = model_7.fit(train_sentence_10_percent, train_labels_10_percent,\n",
    "                                    epochs=5,\n",
    "                                    validation_data=(val_sentences,val_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vEIWq9ZotjGT",
    "outputId": "bd3c9f81-1bc4-4e3a-d6d2-6e922e5b1b24"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 1s 15ms/step\n",
      "[[0.05908]\n",
      " [0.5107 ]\n",
      " [0.974  ]\n",
      " [0.396  ]\n",
      " [0.619  ]\n",
      " [0.602  ]\n",
      " [0.9717 ]\n",
      " [0.915  ]\n",
      " [0.955  ]\n",
      " [0.0883 ]]\n",
      "tf.Tensor([0. 1. 1. 0. 1. 1. 1. 1. 1. 0.], shape=(10,), dtype=float16)\n",
      "{'accuracy': 77.03412073490814, 'precision': 0.7711671866902318, 'recall': 0.7703412073490814, 'f1': 0.7686901866564684}\n"
     ]
    }
   ],
   "source": [
    "model_7_pred_probs = model_7.predict(val_sentences)\n",
    "print(model_7_pred_probs[:10])\n",
    "\n",
    "model_7_preds = tf.squeeze(tf.round(model_7_pred_probs))\n",
    "print(model_7_preds[:10])\n",
    "\n",
    "model_7_results = calculate_results(val_labels, model_7_preds)\n",
    "print(model_7_results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hAcSwZ5ytjGT"
   },
   "source": [
    "### Compare the performance of each of our model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 297
    },
    "id": "_ozHVAwqtjGT",
    "outputId": "41644536-f31d-4858-84c7-3b44d7461f18"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>baseline</th>\n",
       "      <td>79.265092</td>\n",
       "      <td>0.811139</td>\n",
       "      <td>0.792651</td>\n",
       "      <td>0.786219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>simple_dense</th>\n",
       "      <td>78.608924</td>\n",
       "      <td>0.790328</td>\n",
       "      <td>0.786089</td>\n",
       "      <td>0.783297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lstm</th>\n",
       "      <td>76.115486</td>\n",
       "      <td>0.764233</td>\n",
       "      <td>0.761155</td>\n",
       "      <td>0.758116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gru</th>\n",
       "      <td>79.133858</td>\n",
       "      <td>0.793954</td>\n",
       "      <td>0.791339</td>\n",
       "      <td>0.789266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bidirectional</th>\n",
       "      <td>77.952756</td>\n",
       "      <td>0.779526</td>\n",
       "      <td>0.779528</td>\n",
       "      <td>0.778606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>conv1d</th>\n",
       "      <td>77.165354</td>\n",
       "      <td>0.772820</td>\n",
       "      <td>0.771654</td>\n",
       "      <td>0.769833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tf_hub_sentence_encoder</th>\n",
       "      <td>78.608924</td>\n",
       "      <td>0.786647</td>\n",
       "      <td>0.786089</td>\n",
       "      <td>0.784865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tf_hub_10_percent_data</th>\n",
       "      <td>77.952756</td>\n",
       "      <td>0.780871</td>\n",
       "      <td>0.779528</td>\n",
       "      <td>0.777770</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          accuracy  precision    recall        f1\n",
       "baseline                 79.265092   0.811139  0.792651  0.786219\n",
       "simple_dense             78.608924   0.790328  0.786089  0.783297\n",
       "lstm                     76.115486   0.764233  0.761155  0.758116\n",
       "gru                      79.133858   0.793954  0.791339  0.789266\n",
       "bidirectional            77.952756   0.779526  0.779528  0.778606\n",
       "conv1d                   77.165354   0.772820  0.771654  0.769833\n",
       "tf_hub_sentence_encoder  78.608924   0.786647  0.786089  0.784865\n",
       "tf_hub_10_percent_data   77.952756   0.780871  0.779528  0.777770"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_model_results = pd.DataFrame({\"baseline\":baseline_results,\n",
    "                                  \"simple_dense\":model_1_results,\n",
    "                                  \"lstm\":model_2_results,\n",
    "                                  \"gru\":model_3_results,\n",
    "                                  \"bidirectional\":model_4_results,\n",
    "                                  \"conv1d\":model_5_results,\n",
    "                                  \"tf_hub_sentence_encoder\":model_6_results,\n",
    "                                  \"tf_hub_10_percent_data\":model_7_results})\n",
    "all_model_results = all_model_results.transpose()\n",
    "all_model_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fzCEvY1mtjGT"
   },
   "outputs": [],
   "source": [
    "# Reduce the accuracy to same scale as other metrics\n",
    "all_model_results[\"accuracy\"] = all_model_results[\"accuracy\"]/100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 763
    },
    "id": "n0YXHTZT2KoM",
    "outputId": "fe4c6c63-c48b-4df8-96ee-cb4fe5bddd00"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7IAAALqCAYAAAAIKmjaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABwgElEQVR4nO3deVxU9eL/8feAsimLKy4XxS0VRVFwS3NJUm99Lc1uXLVQUlsxkyy1FNPMrVwyvZJbamXaYsvNrlmkpUjuYOW+oiaImhKooMDvD39NTaA1yHA8zOv5eMzjMp/zmZk3TBd5zznncyz5+fn5AgAAAADAJFyMDgAAAAAAgD0osgAAAAAAU6HIAgAAAABMhSILAAAAADAViiwAAAAAwFQosgAAAAAAU6HIAgAAAABMhSILAAAAADCVMkYH+Dvy8vL0888/y9vbWxaLxeg4AAAAAAySn5+vX3/9VTVq1JCLC/vlnJUpiuzPP/+sgIAAo2MAAAAAuEUcP35c//jHP4yOAYOYosh6e3tLuvYfq4+Pj8FpAAAAABglIyNDAQEB1o4A52SKIvvb4cQ+Pj4UWQAAAACccujkOKgcAAAAAGAqFFkAAAAAgKlQZAEAAAAApmKKc2QBAAAA4O/Kzc3VlStXjI4BO7m6uqpMmTJ/6/xniiwAAACAUiMzM1MnTpxQfn6+0VFQBF5eXqpevbrc3NxuOI8iCwAAAKBUyM3N1YkTJ+Tl5aUqVaqwsrGJ5OfnKycnR+np6Tpy5IgaNGggF5frnwlLkQUAAABQKly5ckX5+fmqUqWKPD09jY4DO3l6eqps2bI6duyYcnJy5OHhcd25LPYEAAAAoFRhT6x53WgvrM08B+cAAAAAAKBYUWQBAAAAAKbCObIAAAAASrXAUatL9PWOTrmnRF/PGbFHFgAAAABg41a/Di9FFgAAAAAMtmbNGnXo0EF+fn6qVKmS/u///k+HDh2ybj9x4oT69u2rihUrqly5cgoLC9PmzZut2//73/+qVatW8vDwUOXKldW7d2/rNovFok8++cTm9fz8/LRkyRJJ0tGjR2WxWLRy5Up16tRJHh4eevfdd3X27Fn17dtXNWvWlJeXl4KDg/Xee+/ZPE9eXp6mTZum+vXry93dXbVq1dIrr7wiSbrzzjsVHR1tMz89PV1ubm6Kj4+/qZ8XRRYAAAAADJaVlaWYmBht27ZN8fHxcnFxUe/evZWXl6fMzEx16tRJJ0+e1Geffabk5GQ9//zzysvLkyStXr1avXv31t13362dO3cqPj5erVu3tjvDqFGjNGzYMO3Zs0fdu3fX5cuXFRoaqtWrV+vHH3/Uo48+qocfflhbtmyxPmb06NGaMmWKxo4dq927d2v58uXy9/eXJA0ePFjLly9Xdna2df4777yjmjVr6s4777ypnxfnyAIAAACAwfr06WNzf/HixapSpYp2796tTZs2KT09XVu3blXFihUlSfXr17fOfeWVV/Tvf/9b48ePt441b97c7gzPPPOM7r//fpuxESNGWL8eOnSovvzyS73//vtq3bq1fv31V73++uuaM2eOBgwYIEmqV6+eOnToIEm6//77FR0drU8//VQPPvigJGnJkiUaOHDgTV8iiT2yAAAAAGCwAwcOqG/fvqpbt658fHwUGBgoSUpJSVFSUpJatGhhLbF/lpSUpK5du950hrCwMJv7ubm5evnllxUcHKyKFSuqfPny+vLLL5WSkiJJ2rNnj7Kzs6/72h4eHnr44Ye1ePFiSdKOHTv0448/auDAgTedlT2yAAAAAGCwnj17qnbt2lqwYIFq1KihvLw8NW3aVDk5OfL09LzhY/9qu8ViUX5+vs1YYYs5lStXzub+q6++qtdff12zZs1ScHCwypUrp2eeeUY5OTl/63Wla4cXh4SE6MSJE3rrrbd05513qnbt2n/5uL/CHlkAAAAAMNDZs2e1b98+jRkzRl27dlXjxo31yy+/WLc3a9ZMSUlJOnfuXKGPb9as2Q0XT6pSpYpOnTplvX/gwAFdvHjxL3MlJCTovvvu00MPPaTmzZurbt262r9/v3V7gwYN5OnpecPXDg4OVlhYmBYsWKDly5frkUce+cvX/TsosgAAAABgoAoVKqhSpUqaP3++Dh48qG+++UYxMTHW7X379lW1atXUq1cvJSQk6PDhw/roo4+UmJgoSRo3bpzee+89jRs3Tnv27NEPP/ygqVOnWh9/5513as6cOdq5c6e2bdumxx9/XGXLlv3LXA0aNNBXX32lTZs2ac+ePXrssceUlpZm3e7h4aGRI0fq+eef17Jly3To0CF9//33WrRokc3zDB48WFOmTFF+fr7Naso3gyILAAAAAAZycXHRihUrtH37djVt2lTDhw/Xq6++at3u5uamtWvXqmrVqrr77rsVHBysKVOmyNXVVZLUuXNnffDBB/rss88UEhKiO++802Zl4enTpysgIEB33HGH+vXrpxEjRsjLy+svc40ZM0YtW7ZU9+7d1blzZ2uZ/qOxY8fq2WefVWxsrBo3bqyIiAidPn3aZk7fvn1VpkwZ9e3bVx4eHjfxk/qdJf/PB0vfgjIyMuTr66sLFy7Ix8fH6DgAAAAADHKjbnD58mUdOXJEderUKbbChJt39OhR1atXT1u3blXLli1vOPfvvocs9gQAAAAAKHZXrlzR2bNnNWbMGLVt2/YvS6w9KLJF9ZKvnfMvOCYHAAAAANyCEhIS1KVLF91222368MMPi/W5KbIAAAAAgGLXuXPnApf9KS4s9gQAAAAAMJUiFdm5c+cqMDBQHh4eatOmjc2KWIWZNWuWGjZsKE9PTwUEBGj48OG6fPlykQIDAAAAAJyb3UV25cqViomJ0bhx47Rjxw41b95c3bt3L7DE8m+WL1+uUaNGWa9ptGjRIq1cuVIvvPDCTYcHAAAAADgfu4vsjBkzNGTIEEVFRSkoKEhxcXHy8vLS4sWLC52/adMmtW/fXv369VNgYKC6deumvn37/uVeXAAAAAAACmNXkc3JydH27dsVHh7++xO4uCg8PFyJiYmFPub222/X9u3brcX18OHD+uKLL3T33Xdf93Wys7OVkZFhcwMAAAAAQLJz1eIzZ84oNzdX/v7+NuP+/v7au3dvoY/p16+fzpw5ow4dOig/P19Xr17V448/fsNDiydPnqzx48fbEw0AAAAA4CQcvmrx+vXrNWnSJP3nP//Rjh07tGrVKq1evVovv/zydR8zevRoXbhwwXo7fvy4o2MCAAAAgNNYv369LBaLzp8/X6xzS4pde2QrV64sV1dXpaWl2YynpaWpWrVqhT5m7NixevjhhzV48GBJUnBwsLKysvToo4/qxRdflItLwS7t7u4ud3d3e6IBAAAAQOFe8i3h17tQsq9XBLfffrtOnTolX9+//tnYM7ek2LVH1s3NTaGhoYqPj7eO5eXlKT4+Xu3atSv0MRcvXixQVl1dXSXJYRfHBQAAAIDSKicn56afw83NTdWqVZPFYinWuSXF7kOLY2JitGDBAi1dulR79uzRE088oaysLEVFRUmSIiMjNXr0aOv8nj17at68eVqxYoWOHDmir776SmPHjlXPnj2thRYAUEQv+dp3AwAAt5zOnTsrOjpa0dHR8vX1VeXKlTV27Fjrjr/AwEC9/PLLioyMlI+Pjx599FFJ0saNG3XHHXfI09NTAQEBevrpp5WVlWV93uzsbI0cOVIBAQFyd3dX/fr1tWjRIkkFDxc+duyYevbsqQoVKqhcuXJq0qSJvvjii0LnStJHH32kJk2ayN3dXYGBgZo+fbrN9xQYGKhJkybpkUcekbe3t2rVqqX58+cX28/MrkOLJSkiIkLp6emKjY1VamqqQkJCtGbNGusCUCkpKTZ7YMeMGSOLxaIxY8bo5MmTqlKlinr27KlXXnml2L6J4hA4arVd84962Pf8wUuD7Zr/w4Af7HsBAAAAAKa1dOlSDRo0SFu2bNG2bdv06KOPqlatWhoyZIgk6bXXXlNsbKzGjRsnSTp06JB69OihiRMnavHixUpPT7eW4bfeekvStZ2MiYmJmj17tpo3b64jR47ozJkzhb7+U089pZycHH333XcqV66cdu/erfLlyxc6d/v27XrwwQf10ksvKSIiQps2bdKTTz6pSpUqaeDAgdZ506dP18svv6wXXnhBH374oZ544gl16tRJDRs2vOmflyXfBMf3ZmRkyNfXVxcuXJCPj49DXsP+ItvPrvnBdWrZNZ8i+zfYu3fJBOcqAPwuAgDgxm7UDS5fvqwjR46oTp068vD4w56nW/wc2c6dO+v06dP66aefrIfvjho1Sp999pl2796twMBAtWjRQh9//LH1MYMHD5arq6vefPNN69jGjRvVqVMnZWVlKSUlRQ0bNtRXX31lc/nU36xfv15dunTRL7/8Ij8/PzVr1kx9+vSxFuUbze3fv7/S09O1du1a65znn39eq1ev1k8//STp2h7ZO+64Q2+//baka6eVVqtWTePHj9fjjz9+3Z/Fdd/DP7F7jywAwHnsadTYrvmN9+5xUBIAAEq3tm3b2pyD2q5dO02fPl25ubmSpLCwMJv5ycnJ2rVrl959913rWH5+vvLy8nTkyBH98MMPcnV1VadOnf7W6z/99NN64okntHbtWoWHh6tPnz5q1qxZoXP37Nmj++67z2asffv2mjVrlnJzc62nkP7x8RaLRdWqVdPp06f/Vp6/4vDL7wAAAAAAbk65cuVs7mdmZuqxxx5TUlKS9ZacnKwDBw6oXr168vT0tOv5Bw8erMOHD+vhhx/WDz/8oLCwML3xxhs3lbls2bI29y0Wi/Ly8m7qOX/DHtlblDPuBeE8ZQAAjHerneIg8W8ynMPmzZtt7n///fdq0KDBdRfIbdmypXbv3q369esXuj04OFh5eXn69ttvCz20uDABAQF6/PHH9fjjj2v06NFasGCBhg4dWmBe48aNlZCQYDOWkJCg2267rcQW9KXIAgAA4JbmjB/ww/mkpKQoJiZGjz32mHbs2KE33nijwErAfzRy5Ei1bdtW0dHRGjx4sHWBpq+++kpz5sxRYGCgBgwYoEceecS62NOxY8d0+vRpPfjggwWe75lnntE///lP3Xbbbfrll1+0bt06NW5c+P/3nn32WbVq1Uovv/yyIiIilJiYqDlz5ug///lPsf08/gpFFrgO/tEEAABASYmMjNSlS5fUunVrubq6atiwYdbL7BSmWbNm+vbbb/Xiiy/qjjvuUH5+vurVq6eIiAjrnHnz5umFF17Qk08+qbNnz6pWrVp64YUXCn2+3NxcPfXUUzpx4oR8fHzUo0cPzZw5s9C5LVu21Pvvv6/Y2Fi9/PLLql69uiZMmGCzYrGjUWQBAAAAM+NKDn/NBN9z2bJlNWvWLM2bN6/AtqNHjxb6mFatWtmsHPxnHh4emjFjhmbMmFFgW+fOnfXHC9jc6HzYP8+VpD59+qhPnz7XfUxhmZOSkq47314UWQAA/j97zw2UuAQSgOLHuiHAX6PIAgBwC+M0BwBGs/f3kMTvIjgeRRYAAAAADLR+/XqjI5gO15EFAAAAAJgKRRYAAAAAYCocWgwAAG4Z9i9yY99iWxILbgFAaUCRBQAAuAEW3AKAWw+HFgMAAAAATIUiCwAAAAAwFYosAAAAADiZl156SSEhIdb7AwcOVK9evQzLYy/OkQVQdC/52jn/gmNyAAAA3EDw0uASfT0WiXM89sgCAAAAwC0kJyfH6Ai3PPbIArCy/7IX9j2/vZ+G8mkmAABwBp07d1bTpk1VpkwZvfPOOwoODtYbb7yh5557Ths2bFC5cuXUrVs3zZw5U5UrV5Yk5eXl6bXXXtP8+fN1/Phx+fv767HHHtOLL74oSRo5cqQ+/vhjnThxQtWqVVP//v0VGxursmXLGvmtFhv2yAIAAACAwZYuXSo3NzclJCRoypQpuvPOO9WiRQtt27ZNa9asUVpamh588EHr/NGjR2vKlCkaO3asdu/ereXLl8vf39+63dvbW0uWLNHu3bv1+uuva8GCBZo5c6YR35pDsEcWwC2LazcCAABn0aBBA02bNk2SNHHiRLVo0UKTJk2ybl+8eLECAgK0f/9+Va9eXa+//rrmzJmjAQMGSJLq1aunDh06WOePGTPG+nVgYKBGjBihFStW6Pnnny+h78ixKLIAAAAAYLDQ0FDr18nJyVq3bp3Kly9fYN6hQ4d0/vx5ZWdnq2vXrtd9vpUrV2r27Nk6dOiQMjMzdfXqVfn4+DgkuxEosgAAAABgsHLlylm/zszMVM+ePTV16tQC86pXr67Dhw/f8LkSExPVv39/jR8/Xt27d5evr69WrFih6dOnF3tuo1BkAQAAAOAW0rJlS3300UcKDAxUmTIFK1uDBg3k6emp+Ph4DR48uMD2TZs2qXbt2taFnyTp2LFjDs1c0ljsCQAAAABuIU899ZTOnTunvn37auvWrTp06JC+/PJLRUVFKTc3Vx4eHho5cqSef/55LVu2TIcOHdL333+vRYsWSbpWdFNSUrRixQodOnRIs2fP1scff2zwd1W8KLIAAAAAcAupUaOGEhISlJubq27duik4OFjPPPOM/Pz85OJyrcKNHTtWzz77rGJjY9W4cWNFRETo9OnTkqR7771Xw4cPV3R0tEJCQrRp0yaNHTvWyG+p2HFoMQAAAIBS7Va/Nv369esLjDVo0ECrVq267mNcXFz04osv2hw+/EfTpk2zroL8m2eeecb69UsvvaSXXnrJen/JkiX2RDYce2QBAAAAAKZCkQUAAAAAmApFFgAAAABgKhRZAAAAAICpUGQBAAAAAKZCkQUAAAAAmApFFgAAAABgKhRZAAAAAICpUGQBAAAAAKZCkQUAAAAAA+Xn5+vRRx9VxYoVZbFYlJSUZHSkW14ZowMAAAAAgCPtadS4RF+v8d49ds1fs2aNlixZovXr16tu3brav3+/evbsqe3bt+vUqVP6+OOP1atXL8eENSn2yAIAAACAgQ4dOqTq1avr9ttvV7Vq1ZSVlaXmzZtr7ty5Rke7ZbFHFgAAAAAMMnDgQC1dulSSZLFYVLt2bR09elT//Oc/DU52a6PIAgAAAIBBXn/9ddWrV0/z58/X1q1b5erqanQkU6DIAgAAAIBBfH195e3tLVdXV1WrVs3oOKbBObIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFc6RBQAAAIBbSGZmpg4ePGi9f+TIESUlJalixYqqVauWgcluHRRZAAAAALiFbNu2TV26dLHej4mJkSQNGDBAS5YsMSjVrYUiCwAAAKBUa7x3j9ERbuiZZ57RM888Y73fuXNn5efnGxfIBDhHFgAAAABgKhRZAAAAAICpUGQBAAAAAKZCkQUAAAAAmApFFgAAAABgKkUqsnPnzlVgYKA8PDzUpk0bbdmy5bpzO3fuLIvFUuB2zz33FDk0AAAAAFwPK/6a19997+wusitXrlRMTIzGjRunHTt2qHnz5urevbtOnz5d6PxVq1bp1KlT1tuPP/4oV1dX/etf/7L3pQEAAADgulxdXSVJOTk5BidBUV28eFGSVLZs2RvOs/s6sjNmzNCQIUMUFRUlSYqLi9Pq1au1ePFijRo1qsD8ihUr2txfsWKFvLy8KLIAAAAAilWZMmXk5eWl9PR0lS1bVi4unElpFvn5+bp48aJOnz4tPz8/64cS12NXkc3JydH27ds1evRo65iLi4vCw8OVmJj4t55j0aJF+ve//61y5cpdd052drays7Ot9zMyMuyJCQAAAMAJWSwWVa9eXUeOHNGxY8eMjoMi8PPzU7Vq1f5ynl1F9syZM8rNzZW/v7/NuL+/v/bu3fuXj9+yZYt+/PFHLVq06IbzJk+erPHjx9sTDQAAAADk5uamBg0acHixCZUtW/Yv98T+xu5Di2/GokWLFBwcrNatW99w3ujRoxUTE2O9n5GRoYCAAEfHAwAAAFAKuLi4yMPDw+gYcCC7imzlypXl6uqqtLQ0m/G0tLS/3P2blZWlFStWaMKECX/5Ou7u7nJ3d7cnGgAAAADASdh19rObm5tCQ0MVHx9vHcvLy1N8fLzatWt3w8d+8MEHys7O1kMPPVS0pAAAAAAAqAiHFsfExGjAgAEKCwtT69atNWvWLGVlZVlXMY6MjFTNmjU1efJkm8ctWrRIvXr1UqVKlYonOQAAAADAKdldZCMiIpSenq7Y2FilpqYqJCREa9assS4AlZKSUmCZ63379mnjxo1au3Zt8aQGAAAAADitIi32FB0drejo6EK3rV+/vsBYw4YNlZ+fX5SXAgAAAADABlcIBgAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAAplKkIjt37lwFBgbKw8NDbdq00ZYtW244//z583rqqadUvXp1ubu767bbbtMXX3xRpMAAAAAAAOdWxt4HrFy5UjExMYqLi1ObNm00a9Ysde/eXfv27VPVqlULzM/JydFdd92lqlWr6sMPP1TNmjV17Ngx+fn5FUd+AAAAAICTsbvIzpgxQ0OGDFFUVJQkKS4uTqtXr9bixYs1atSoAvMXL16sc+fOadOmTSpbtqwkKTAw8OZSAwAAAACcll2HFufk5Gj79u0KDw///QlcXBQeHq7ExMRCH/PZZ5+pXbt2euqpp+Tv76+mTZtq0qRJys3Nve7rZGdnKyMjw+YGAAAAAIBkZ5E9c+aMcnNz5e/vbzPu7++v1NTUQh9z+PBhffjhh8rNzdUXX3yhsWPHavr06Zo4ceJ1X2fy5Mny9fW13gICAuyJCQAAAAAoxRy+anFeXp6qVq2q+fPnKzQ0VBEREXrxxRcVFxd33ceMHj1aFy5csN6OHz/u6JgAAAAAAJOw6xzZypUry9XVVWlpaTbjaWlpqlatWqGPqV69usqWLStXV1frWOPGjZWamqqcnBy5ubkVeIy7u7vc3d3tiQYAAAAAcBJ27ZF1c3NTaGio4uPjrWN5eXmKj49Xu3btCn1M+/btdfDgQeXl5VnH9u/fr+rVqxdaYgEAAAAAuBG7Dy2OiYnRggULtHTpUu3Zs0dPPPGEsrKyrKsYR0ZGavTo0db5TzzxhM6dO6dhw4Zp//79Wr16tSZNmqSnnnqq+L4LAAAAAIDTsPvyOxEREUpPT1dsbKxSU1MVEhKiNWvWWBeASklJkYvL7/04ICBAX375pYYPH65mzZqpZs2aGjZsmEaOHFl83wUAAAAAwGnYXWQlKTo6WtHR0YVuW79+fYGxdu3a6fvvvy/KSwEAAAAAYMPhqxYDAAAAAFCcKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwlSIV2blz5yowMFAeHh5q06aNtmzZct25S5YskcVisbl5eHgUOTAAAAAAwLnZXWRXrlypmJgYjRs3Tjt27FDz5s3VvXt3nT59+rqP8fHx0alTp6y3Y8eO3VRoAAAAAIDzsrvIzpgxQ0OGDFFUVJSCgoIUFxcnLy8vLV68+LqPsVgsqlatmvXm7+9/U6EBAAAAAM7LriKbk5Oj7du3Kzw8/PcncHFReHi4EhMTr/u4zMxM1a5dWwEBAbrvvvv0008/3fB1srOzlZGRYXMDAAAAAECys8ieOXNGubm5Bfao+vv7KzU1tdDHNGzYUIsXL9ann36qd955R3l5ebr99tt14sSJ677O5MmT5evra70FBATYExMAAAAAUIo5fNXidu3aKTIyUiEhIerUqZNWrVqlKlWq6M0337zuY0aPHq0LFy5Yb8ePH3d0TAAAAACASZSxZ3LlypXl6uqqtLQ0m/G0tDRVq1btbz1H2bJl1aJFCx08ePC6c9zd3eXu7m5PNAAAAACAk7Brj6ybm5tCQ0MVHx9vHcvLy1N8fLzatWv3t54jNzdXP/zwg6pXr25fUgAAAAAAZOceWUmKiYnRgAEDFBYWptatW2vWrFnKyspSVFSUJCkyMlI1a9bU5MmTJUkTJkxQ27ZtVb9+fZ0/f16vvvqqjh07psGDBxfvdwIAAAAAcAp2F9mIiAilp6crNjZWqampCgkJ0Zo1a6wLQKWkpMjF5fcdvb/88ouGDBmi1NRUVahQQaGhodq0aZOCgoKK77sAAAAAADgNu4usJEVHRys6OrrQbevXr7e5P3PmTM2cObMoLwMAAAAAQAEOX7UYAAAAAIDiRJEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqRSpyM6dO1eBgYHy8PBQmzZttGXLlr/1uBUrVshisahXr15FeVkAAAAAAOwvsitXrlRMTIzGjRunHTt2qHnz5urevbtOnz59w8cdPXpUI0aM0B133FHksAAAAAAA2F1kZ8yYoSFDhigqKkpBQUGKi4uTl5eXFi9efN3H5Obmqn///ho/frzq1q17U4EBAAAAAM7NriKbk5Oj7du3Kzw8/PcncHFReHi4EhMTr/u4CRMmqGrVqho0aFDRkwIAAAAAIKmMPZPPnDmj3Nxc+fv724z7+/tr7969hT5m48aNWrRokZKSkv7262RnZys7O9t6PyMjw56YAAAAAIBSzKGrFv/66696+OGHtWDBAlWuXPlvP27y5Mny9fW13gICAhyYEgAAAABgJnbtka1cubJcXV2VlpZmM56WlqZq1aoVmH/o0CEdPXpUPXv2tI7l5eVde+EyZbRv3z7Vq1evwONGjx6tmJgY6/2MjAzKLAAAAABAkp1F1s3NTaGhoYqPj7deQicvL0/x8fGKjo4uML9Ro0b64YcfbMbGjBmjX3/9Va+//vp1y6m7u7vc3d3tiQYAAAAAcBJ2FVlJiomJ0YABAxQWFqbWrVtr1qxZysrKUlRUlCQpMjJSNWvW1OTJk+Xh4aGmTZvaPN7Pz0+SCowDAAAAAPB32F1kIyIilJ6ertjYWKWmpiokJERr1qyxLgCVkpIiFxeHnnoLAAAAAHBidhdZSYqOji70UGJJWr9+/Q0fu2TJkqK8JAAAAAAAkhy8ajEAAAAAAMWNIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEyFIgsAAAAAMBWKLAAAAADAVCiyAAAAAABTocgCAAAAAEylSEV27ty5CgwMlIeHh9q0aaMtW7Zcd+6qVasUFhYmPz8/lStXTiEhIXr77beLHBgAAAAA4NzsLrIrV65UTEyMxo0bpx07dqh58+bq3r27Tp8+Xej8ihUr6sUXX1RiYqJ27dqlqKgoRUVF6csvv7zp8AAAAAAA52N3kZ0xY4aGDBmiqKgoBQUFKS4uTl5eXlq8eHGh8zt37qzevXurcePGqlevnoYNG6ZmzZpp48aNNx0eAAAAAOB87CqyOTk52r59u8LDw39/AhcXhYeHKzEx8S8fn5+fr/j4eO3bt08dO3a87rzs7GxlZGTY3AAAAAAAkOwssmfOnFFubq78/f1txv39/ZWamnrdx124cEHly5eXm5ub7rnnHr3xxhu66667rjt/8uTJ8vX1td4CAgLsiQkAAAAAKMVKZNVib29vJSUlaevWrXrllVcUExOj9evXX3f+6NGjdeHCBevt+PHjJRETAAAAAGACZeyZXLlyZbm6uiotLc1mPC0tTdWqVbvu41xcXFS/fn1JUkhIiPbs2aPJkyerc+fOhc53d3eXu7u7PdEAAAAAAE7Crj2ybm5uCg0NVXx8vHUsLy9P8fHxateu3d9+nry8PGVnZ9vz0gAAAAAASLJzj6wkxcTEaMCAAQoLC1Pr1q01a9YsZWVlKSoqSpIUGRmpmjVravLkyZKune8aFhamevXqKTs7W1988YXefvttzZs3r3i/EwAAAACAU7C7yEZERCg9PV2xsbFKTU1VSEiI1qxZY10AKiUlRS4uv+/ozcrK0pNPPqkTJ07I09NTjRo10jvvvKOIiIji+y4AAAAAAE7D7iIrSdHR0YqOji50258XcZo4caImTpxYlJcBAAAAAKCAElm1GAAAAACA4kKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKkUqcjOnTtXgYGB8vDwUJs2bbRly5brzl2wYIHuuOMOVahQQRUqVFB4ePgN5wMAAAAAcCN2F9mVK1cqJiZG48aN044dO9S8eXN1795dp0+fLnT++vXr1bdvX61bt06JiYkKCAhQt27ddPLkyZsODwAAAABwPnYX2RkzZmjIkCGKiopSUFCQ4uLi5OXlpcWLFxc6/91339WTTz6pkJAQNWrUSAsXLlReXp7i4+NvOjwAAAAAwPnYVWRzcnK0fft2hYeH//4ELi4KDw9XYmLi33qOixcv6sqVK6pYseJ152RnZysjI8PmBgAAAACAZGeRPXPmjHJzc+Xv728z7u/vr9TU1L/1HCNHjlSNGjVsyvCfTZ48Wb6+vtZbQECAPTEBAAAAAKVYia5aPGXKFK1YsUIff/yxPDw8rjtv9OjRunDhgvV2/PjxEkwJAAAAALiVlbFncuXKleXq6qq0tDSb8bS0NFWrVu2Gj33ttdc0ZcoUff3112rWrNkN57q7u8vd3d2eaAAAAAAAJ2HXHlk3NzeFhobaLNT028JN7dq1u+7jpk2bppdffllr1qxRWFhY0dMCAAAAAJyeXXtkJSkmJkYDBgxQWFiYWrdurVmzZikrK0tRUVGSpMjISNWsWVOTJ0+WJE2dOlWxsbFavny5AgMDrefSli9fXuXLly/GbwUAAAAA4AzsLrIRERFKT09XbGysUlNTFRISojVr1lgXgEpJSZGLy+87eufNm6ecnBw98MADNs8zbtw4vfTSSzeXHgAAAADgdOwuspIUHR2t6OjoQretX7/e5v7Ro0eL8hIAAAAAABSqRFctBgAAAADgZlFkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCoUWQAAAACAqVBkAQAAAACmQpEFAAAAAJgKRRYAAAAAYCpFKrJz585VYGCgPDw81KZNG23ZsuW6c3/66Sf16dNHgYGBslgsmjVrVlGzAgAAAABgf5FduXKlYmJiNG7cOO3YsUPNmzdX9+7ddfr06ULnX7x4UXXr1tWUKVNUrVq1mw4MAAAAAHBudhfZGTNmaMiQIYqKilJQUJDi4uLk5eWlxYsXFzq/VatWevXVV/Xvf/9b7u7uNx0YAAAAAODc7CqyOTk52r59u8LDw39/AhcXhYeHKzExsdhCZWdnKyMjw+YGAAAAAIBkZ5E9c+aMcnNz5e/vbzPu7++v1NTUYgs1efJk+fr6Wm8BAQHF9twAAAAAAHO7JVctHj16tC5cuGC9HT9+3OhIAAAAAIBbRBl7JleuXFmurq5KS0uzGU9LSyvWhZzc3d05nxYAAAAAUCi79si6ubkpNDRU8fHx1rG8vDzFx8erXbt2xR4OAAAAAIA/s2uPrCTFxMRowIABCgsLU+vWrTVr1ixlZWUpKipKkhQZGamaNWtq8uTJkq4tELV7927r1ydPnlRSUpLKly+v+vXrF+O3AgAAAABwBnYX2YiICKWnpys2NlapqakKCQnRmjVrrAtApaSkyMXl9x29P//8s1q0aGG9/9prr+m1115Tp06dtH79+pv/DgAAAAAATsXuIitJ0dHRio6OLnTbn8tpYGCg8vPzi/IyAAAAAAAUcEuuWgwAAAAAwPVQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqFFkAAAAAgKlQZAEAAAAApkKRBQAAAACYCkUWAAAAAGAqRSqyc+fOVWBgoDw8PNSmTRtt2bLlhvM/+OADNWrUSB4eHgoODtYXX3xRpLAAAAAAANhdZFeuXKmYmBiNGzdOO3bsUPPmzdW9e3edPn260PmbNm1S3759NWjQIO3cuVO9evVSr1699OOPP950eAAAAACA87G7yM6YMUNDhgxRVFSUgoKCFBcXJy8vLy1evLjQ+a+//rp69Oih5557To0bN9bLL7+sli1bas6cOTcdHgAAAADgfOwqsjk5Odq+fbvCw8N/fwIXF4WHhysxMbHQxyQmJtrMl6Tu3btfdz4AAAAAADdSxp7JZ86cUW5urvz9/W3G/f39tXfv3kIfk5qaWuj81NTU675Odna2srOzrfcvXLggScrIyLAnrl3ysi/aNT/Dkm/X/NxLuXbNz8y1b74jfzYlhffAeLwHxuM9MJa9P3+J96C4Ofr/AxLvwV+51X4PSbwHf+VW+z0kOfY9+O258/Pt//8/Sg+7imxJmTx5ssaPH19gPCAgwIA0hfO1+xF77Jrd2t6n97U/kdnxHhiP98B4vAfG4z0wVtG+W96D4uTo/w9IvAd/5Zb7PSSVyHvw66+/ytfJ3mv8zq4iW7lyZbm6uiotLc1mPC0tTdWqVSv0MdWqVbNrviSNHj1aMTEx1vt5eXk6d+6cKlWqJIvFYk/kW0JGRoYCAgJ0/Phx+fj4GB3HKfEeGI/3wHi8B8bjPTAe74Gx+PkbrzS8B/n5+fr1119Vo0YNo6PAQHYVWTc3N4WGhio+Pl69evWSdK1kxsfHKzo6utDHtGvXTvHx8XrmmWesY1999ZXatWt33ddxd3eXu7u7zZifn589UW9JPj4+pv2FUVrwHhiP98B4vAfG4z0wHu+Bsfj5G8/s7wF7YmH3ocUxMTEaMGCAwsLC1Lp1a82aNUtZWVmKioqSJEVGRqpmzZqaPHmyJGnYsGHq1KmTpk+frnvuuUcrVqzQtm3bNH/+/OL9TgAAAAAATsHuIhsREaH09HTFxsYqNTVVISEhWrNmjXVBp5SUFLm4/L4Y8u23367ly5drzJgxeuGFF9SgQQN98sknatq0afF9FwAAAAAAp1GkxZ6io6Oveyjx+vXrC4z961//0r/+9a+ivFSp4O7urnHjxhU4XBolh/fAeLwHxuM9MB7vgfF4D4zFz994vAcoLSz5rFsNAAAAADARl7+eAgAAAADArYMiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAMAhrly5okceeURHjhwxOgqAUoZVi1GqbdiwQW+++aYOHTqkDz/8UDVr1tTbb7+tOnXqqEOHDkbHAxwuJSXlhttr1apVQkkAOCtfX18lJSWpTp06RkeBpMuXLysnJ8dmzMfHx6A0QNEV6Tqy+HuuXr2q9evX69ChQ+rXr5+8vb31888/y8fHR+XLlzc6Xqn30Ucf6eGHH1b//v21c+dOZWdnS5IuXLigSZMm6YsvvjA4IeB4gYGBslgs192em5tbgmmAkhETE/O3586YMcOBSSBJvXr10ieffKLhw4cbHcVpXbx4Uc8//7zef/99nT17tsB2/i2AGVFkHeTYsWPq0aOHUlJSlJ2drbvuukve3t6aOnWqsrOzFRcXZ3TEUm/ixImKi4tTZGSkVqxYYR1v3769Jk6caGAy53H58mW98cYbWrdunU6fPq28vDyb7Tt27DAomfPYuXOnzf0rV65o586dmjFjhl555RWDUpV+FSpUuOEHCH907tw5B6dxPn/+737Hjh26evWqGjZsKEnav3+/XF1dFRoaakQ8p9OgQQNNmDBBCQkJCg0NVbly5Wy2P/300wYlcx7PPfec1q1bp3nz5unhhx/W3LlzdfLkSb355puaMmWK0fGAIqHIOsiwYcMUFham5ORkVapUyTreu3dvDRkyxMBkzmPfvn3q2LFjgXFfX1+dP3++5AM5oUGDBmnt2rV64IEH1Lp167/9hz2KT/PmzQuMhYWFqUaNGnr11Vd1//33G5Cq9Js1a5bREZzaunXrrF/PmDFD3t7eWrp0qSpUqCBJ+uWXXxQVFaU77rjDqIhOZdGiRfLz89P27du1fft2m20Wi4UiWwL++9//atmyZercubP1v/369eurdu3aevfdd9W/f3+jIwJ2o8g6yIYNG7Rp0ya5ubnZjAcGBurkyZMGpXIu1apV08GDBxUYGGgzvnHjRtWtW9eYUE7m888/1xdffKH27dsbHQV/0rBhQ23dutXoGKXWgAEDjI6A/2/69Olau3attcRK1/aYT5w4Ud26ddOzzz5rYDrnwEJPxjt37pz1bx8fHx/rkSAdOnTQE088YWQ0oMhYtdhB8vLyCj3f4MSJE/L29jYgkfMZMmSIhg0bps2bN8tisejnn3/Wu+++qxEjRvBLu4TUrFmT/94NlpGRYXO7cOGC9u7dqzFjxqhBgwZGx3M6ly9fLvCewLEyMjKUnp5eYDw9PV2//vqrAYmcV05Ojvbt26erV68aHcXp1K1b1/qBQqNGjfT+++9Luran1s/Pz8BkQNFRZB2kW7duNoeWWSwWZWZmaty4cbr77ruNC+ZERo0apX79+qlr167KzMxUx44dNXjwYD322GMaOnSo0fGcwvTp0zVy5EgdO3bM6ChOy8/PTxUqVLDeKlasqKCgICUmJmrevHlGx3MKWVlZio6OVtWqVVWuXDmb9+OPewnhGL1791ZUVJRWrVqlEydO6MSJE/roo480aNAgDq0vIRcvXtSgQYPk5eWlJk2aWFdTHzp0KOdnlpCoqCglJydLuvb30dy5c+Xh4aHhw4frueeeMzgdUDRcfsdBTpw4oe7duys/P18HDhxQWFiYDhw4oMqVK+u7775T1apVjY7oNHJycnTw4EFlZmYqKCiIFaNLUHp6uh588EF999138vLyUtmyZW22s8iN43377bc2911cXFSlShXVr19fZcpwdklJeOqpp7Ru3Tq9/PLLhS6ywrlpjnXx4kWNGDFCixcv1pUrVyRJZcqU0aBBg/Tqq68WWHgIxW/YsGFKSEjQrFmz1KNHD+3atUt169bVp59+qpdeeqnA4lxwvGPHjmn79u2qX7++mjVrZnQcoEgosg509epVrVixQrt27VJmZqZatmyp/v37y9PT0+hoTikjI0PffPONGjZsqMaNGxsdxymEh4crJSVFgwYNkr+/f4HFnjiP0LGuXLmixx57TGPHjuX6jQaqVauWdZEVHx8f7dixQ/Xr19fbb7+t9957j0uBlZCsrCwdOnRIklSvXj0KbAmqXbu2Vq5cqbZt28rb21vJycmqW7euDh48qJYtW3KIfQlYtmyZIiIi5O7ubjOek5OjFStWKDIy0qBkQNFRZFFqPfjgg+rYsaOio6N16dIlhYSE6MiRI8rPz9eKFSvUp08foyOWel5eXkpMTCx05VyUDF9fXyUlJVFkDVS+fHnt3r1btWrV0j/+8Q+tWrVKrVu31pEjRxQcHKzMzEyjIwIO5eXlpR9//FF169a1KbLJycnq2LGjLly4YHTEUs/V1VWnTp0qcETg2bNnVbVqVa4jC1PiHFkHOnDggObPn6+JEydqwoQJNjc43nfffWe9tMLHH3+svLw8nT9/XrNnz+Y6siWkUaNGunTpktExnFqvXr30ySefGB3DqbHIyq3p0KFDuvPOO42O4RTCwsK0evVq6/3fjs5ZuHCh2rVrZ1Qsp5Kfn1/oJfBOnDghX19fAxIBN48TpBxkwYIFeuKJJ1S5cmVVq1bN5peHxWJRbGysgemcw4ULF1SxYkVJ0po1a9SnTx95eXnpnnvuYWGDEjJlyhQ9++yzeuWVVxQcHFzgHFkfHx+DkjmPBg0aaMKECUpISFBoaGiBwym5fqPj/bbISqdOnTRq1Cj17NlTc+bM0ZUrVzRjxgyj4zmtzMzMAueQwzEmTZqkf/7zn9q9e7euXr2q119/Xbt379amTZt4DxysRYsWslgsslgs6tq1q83aCLm5uTpy5Ih69OhhYEKg6Di02EFq166tJ598UiNHjjQ6itO67bbbNHHiRN1zzz2qU6eOVqxYoTvvvFPJycnq2rWrzpw5Y3TEUs/F5dpBH3/+FPi3T4Y5lMnxbnRIscVi0eHDh0swDSQWWSkps2fPvuH2kydP6rXXXuP3UAk5dOiQpkyZouTkZOu6ISNHjlRwcLDR0Uq18ePHW//32WeftVnw0s3NTYGBgerTp4/c3NyMiggUGUXWQXx8fJSUlGS9+DRK3n/+8x8NGzZM5cuXV+3atbVjxw65uLjojTfe0KpVq7Ru3TqjI5Z6f/VJe6dOnUooCQBn4+LiourVq1/3D/ScnBylpqZSZOEUli5dqoiICHl4eBgdBSg2FFkHGTRokFq1aqXHH3/c6ChObdu2bTp+/Ljuuusu66eQq1evlp+fn9q3b29wutIvJSVFAQEBhe6RPX78uGrVqmVQMucRExNT6LjFYpGHh4fq16+v++67z3oYPhwjPj5e8fHxOn36tPLy8my2LV682KBUpVudOnU0depUPfjgg4VuT0pKUmhoKEXWQexZiZjTTAAUBUXWQSZPnqwZM2bonnvuKfTcQM5LgzNglUTjdenSRTt27FBubq4aNmwoSdq/f79cXV3VqFEj7du3TxaLRRs3blRQUJDBaUun8ePHa8KECQoLC1P16tULfLDz8ccfG5SsdHvggQdUr149TZ06tdDtycnJatGiRYEPFlA8XFxcCl1cqDD8W+B4ubm5mjlzpt5//32lpKQoJyfHZjvXdYcZUWQdhPPSjJebm6slS5Zcdy/IN998Y1Ay5+Hi4qK0tDRVqVLFZvzYsWMKCgpSVlaWQcmcx6xZs7Rhwwa99dZb1r0eFy5c0ODBg9WhQwcNGTJE/fr106VLl/Tll18anLZ0ql69uqZNm6aHH37Y6ChOZffu3bp48aLCwsIK3X7lyhX9/PPPql27dgkncw5/PLXk6NGjGjVqlAYOHGhdpTgxMVFLly7V5MmTuaZ4CYiNjdXChQv17LPPasyYMXrxxRd19OhRffLJJ4qNjWUHC0yJIotSKzo6WkuWLNE999xT6F6QmTNnGpSs9PvtcNbXX39dQ4YMkZeXl3Vbbm6uNm/eLFdXVyUkJBgV0WnUrFlTX331VYG9rT/99JO6deumkydPaseOHerWrRsLoDlIpUqVtGXLFtWrV8/oKIAhunbtqsGDB6tv374248uXL9f8+fO1fv16Y4I5kXr16mn27Nm655575O3traSkJOvY999/r+XLlxsdEbAbl99BqbVixQq9//77uvvuu42O4nR27twp6dq5sD/88IPNYitubm5q3ry5RowYYVQ8p3LhwgWdPn26QJFNT0+3nsPm5+dX4DAzFJ/Bgwdr+fLlGjt2rNFRnNLEiRPVv3//Gx4pBcdKTExUXFxcgfGwsDANHjzYgETOJzU11bpCdPny5XXhwgVJ0v/93//xuwmmRZEtRjExMXr55ZdVrly56y6w8huuHeh4bm5uql+/vtExnNJvK0JHRUXp9ddfZyEPA91333165JFHNH36dLVq1UqStHXrVo0YMUK9evWSJG3ZskW33XabgSlLt8uXL2v+/Pn6+uuv1axZswJrJvDvgWN98MEHGjdunNq0aaOHHnpIDz74oCpXrmx0LKcSEBCgBQsWaNq0aTbjCxcuVEBAgEGpnMs//vEPnTp1SrVq1VK9evW0du1atWzZUlu3bpW7u7vR8YAi4dDiYtSlSxd9/PHH8vPzU5cuXa47z2KxcH5mCZg+fboOHz6sOXPm/O0FJ+BYGRkZ+uabb9SoUSM1atTI6DhOITMzU8OHD9eyZct09epVSVKZMmU0YMAAzZw5U+XKlVNSUpIkKSQkxLigpRj/Hhjvp59+0rvvvqsVK1boxIkTuuuuu9S/f3/16tXL5tQHOMYXX3yhPn36qH79+mrTpo2kax+gHThwQB999BFHTpWAUaNGycfHRy+88IJWrlyphx56SIGBgUpJSdHw4cM1ZcoUoyMCdqPIotTq3bu31q1bp4oVK6pJkyYF9oKsWrXKoGTO48EHH1THjh0VHR2tS5cuqXnz5jp69Kjy8/O1YsUK9enTx+iITiMzM9O6yFzdunWtl6MCnE1CQoKWL1+uDz74QJcvX7brMjEouhMnTug///mP9u7dK0lq3LixHn/8cfbIGiQxMVGJiYlq0KCBevbsaXQcoEg4tBillp+fn3r37m10DKf23Xff6cUXX5R07RIj+fn5On/+vJYuXaqJEydSZEtQ+fLl1axZM6NjOL0TJ05IunaYH4xRrlw5eXp6ys3NTb/++qvRcZzGP/7xD02aNMnoGPj/2rVrZ11BGjAr9sgWo/vvv/9vz2VvIJyBp6en9u/fr4CAAEVGRqpGjRqaMmWKUlJSFBQUpMzMTKMjAg6Xl5eniRMnavr06db/5r29vfXss8/qxRdflIuLi8EJS78jR45o+fLlWr58ufbt26dOnTqpX79+euCBB+Tr62t0PKdw/vx5LVq0SHv27JEkNWnSRI888gg/fwf67LPP/vbce++914FJAMdgj2wx4pfxrefq1atav369Dh06pH79+snb21s///yzfHx8OLSyBAQEBCgxMVEVK1bUmjVrtGLFCknSL7/8Ig8PD4PTASXjxRdf1KJFizRlyhS1b99ekrRx40a99NJLunz5sl555RWDE5Zubdu21datW9WsWTNFRUWpb9++qlmzptGxnMq2bdvUvXt3eXp6qnXr1pKuLXL2yiuvWBcdQvH7bUG/31gsFv15/9Vva4jk5uaWVCyg2LBHFqXWsWPH1KNHD6WkpCg7O1v79+9X3bp1NWzYMGVnZxd6KQAUr//85z8aNmyYypcvr1q1amnnzp1ycXHRG2+8oVWrVllXNwZKsxo1aiguLq7AHo9PP/1UTz75pE6ePGlQMufw4osvqn///gUuQYWSc8cdd6h+/fpasGCBypS5tg/l6tWrGjx4sA4fPqzvvvvO4ISl39dff62RI0dq0qRJ1kOKExMTNWbMGE2aNEl33XWXwQkB+1FkHYi9gcbq1auXvL29tWjRIlWqVEnJycmqW7eu1q9fryFDhujAgQNGR3QK27dvV0pKirp166Zy5cpJklavXq0KFSro9ttvNzgd4HgeHh7atWtXgUsc7du3TyEhIbp06ZJByYCS4enpqZ07dxZYrX737t0KCwvTxYsXDUrmPJo2baq4uDh16NDBZnzDhg169NFHrYd8A2bCocUO8ue9gXfddZe8vb01depU9gaWkA0bNmjTpk1yc3OzGQ8MDGQPiANd7xrKGzZsKDBGkYUzaN68uebMmaPZs2fbjM+ZM0fNmzc3KJXzyM3N1ZIlSxQfH6/Tp08rLy/PZjuXP3I8Hx8fpaSkFCiyx48fl7e3t0GpnMuhQ4fk5+dXYNzX11dHjx4t8TxAcaDIOsiwYcMUFham5ORkVapUyTreu3dvDRkyxMBkziMvL6/Qcz5OnDjBP5wOtHPnzr81j2v7wllMmzZN99xzj77++mubQ/qOHz+uL774wuB0pd+wYcO0ZMkS3XPPPWratCm/ewwQERGhQYMG6bXXXrN+gJmQkKDnnntOffv2NTidc2jVqpViYmL09ttvy9/fX5KUlpam5557znreMmA2HFrsIJUqVdKmTZvUsGFDeXt7Ww9rPXr0qIKCgjiMpgRERETI19dX8+fPl7e3t3bt2qUqVarovvvuU61atfTWW28ZHRGAk/j55581d+5cm2toPvnkk6pRo4bByUq/ypUra9myZbr77ruNjuK0cnJy9NxzzykuLk5Xr16VJJUtW1ZPPPGEpkyZInd3d4MTln4HDx5U7969rVcSkK7tEW/QoIE++eQT1a9f3+CEgP0osg5SoUIFJSQkKCgoyKbIbty4UX369FFaWprREUu9EydOqHv37srPz9eBAwcUFhamAwcOqHLlyvruu+9UtWpVoyMCABysRo0aWr9+fYFzlFHyLl68qEOHDkmS6tWrJy8vL4MTOZf8/Hx99dVXNh+ohYeHc5QCTIsi6yDsDbw1XL16VStWrNCuXbuUmZmpli1bqn///vL09DQ6GoBSbNeuXWratKlcXFy0a9euG85t1qxZCaVyTtOnT9fhw4c1Z84c/mA3yIULF5Sbm6uKFSvajJ87d05lypSRj4+PQcnwZ8HBwfriiy+se22BWxlF1kHYGwgAzsvFxUWpqamqWrWqXFxcCr1+o3TtXHGu3+hYvXv31rp161SxYkU1adJEZcuWtdm+atUqg5I5j3/+85/q2bOnnnzySZvxuLg4ffbZZ5wrfgv541GEwK2OIutAV69e1cqVK5WcnMzewBLy2Wef/e25f76mIwAUl2PHjqlWrVqyWCw6duzYDefWrl27hFI5p6ioqBtu5wgpx6tYsaISEhLUuHFjm/G9e/eqffv2Onv2rEHJ8GcUWZgJRRaliouLi839wvaC/HZoGXtBAJSE7777TrfffrvKlLG9UMDVq1e1adMmdezY0aBkQMkoV66cvv/+ewUHB9uM//DDD2rTpg0LYN5CKLIwE5e/noKiWLp0qVavXm29//zzz8vPz0+33377X346j6LLy8uz3tauXauQkBD973//0/nz53X+/Hn973//U8uWLbVmzRqjowJwEl26dNG5c+cKjF+4cEFdunQxIJFzSk9P18aNG7Vx40alp6cbHceptG7dWvPnzy8wHhcXp9DQUAMSASgN2CPrIA0bNtS8efN05513KjExUV27dtWsWbP0+eefq0yZMpyTUwKaNm2quLg4dejQwWZ8w4YNevTRR7Vnzx6DkgFwJi4uLkpLS1OVKlVsxvfv36+wsDBlZGQYlMw5ZGVlaejQoVq2bJny8vIkSa6uroqMjNQbb7zByrklICEhQeHh4WrVqpW6du0qSYqPj9fWrVu1du1a3XHHHQYnxG/YIwszKfPXU1AUx48ft16T65NPPtEDDzygRx99VO3bt1fnzp2NDeckDh06JD8/vwLjvr6+Onr0aInnAeBc7r//fknXTmcYOHCgzbUyc3NztWvXLt1+++1GxXMaMTEx+vbbb/Xf//5X7du3lyRt3LhRTz/9tJ599lnNmzfP4ISlX/v27ZWYmKhXX31V77//vjw9PdWsWTMtWrRIDRo0MDoeAJOiyDpI+fLldfbsWdWqVUtr165VTEyMJMnDw0OXLl0yOJ1zaNWqlWJiYvT222/L399fkpSWlqbnnntOrVu3NjgdgNLO19dX0rVrN3p7e9ss9Ofm5qa2bdtqyJAhRsVzGh999JE+/PBDmw+R7777bnl6eurBBx+kyJaQkJAQvfvuu0bHcFrLli1TRESEzQdqkpSTk6MVK1YoMjJSkvTmm29a/2YCbnUcWuwg/fv31969e9WiRQu99957SklJUaVKlfTZZ5/phRde0I8//mh0xFLv4MGD6t27t/bv32+9Htrx48fVoEEDffLJJ9Y95gDgSOPHj9dzzz3HIawG8fLy0vbt2wusmPvTTz+pdevWysrKMiiZc8nLy9PBgwd1+vRp6yHev2HBM8dzdXXVqVOnClz+8ezZs6patSoLYMKUKLIOcv78eY0ZM0bHjx/XE088oR49ekiSxo0bJzc3N7344osGJ3QO+fn5+uqrr7R3715JUuPGjRUeHm5duRgAHO3IkSO6evVqgUMoDxw4oLJlyyowMNCYYE6ia9euqlSpkpYtWyYPDw9J0qVLlzRgwACdO3dOX3/9tcEJS7/vv/9e/fr107Fjxwq9kgAlyvGud65+cnLydRekA251FFk4veDgYH3xxRfWvbYAUJw6deqkRx55RAMGDLAZf+edd7Rw4UKtX7/emGBO4ocfflCPHj2UnZ2t5s2bS7r2x7u7u7vWrl2rJk2aGJyw9AsJCdFtt92m8ePHq3r16gU+TP7tMHwUvxYtWshisSg5OVlNmjSxuQxYbm6ujhw5oh49euj99983MCVQNBRZB7t48aJSUlKUk5NjM96sWTODEuHPWKEPgCP5+Phox44dBU5nOHjwoMLCwnT+/HljgjmRixcv6t1337U5Oqd///425y3DccqVK6fk5GRO6THA+PHjrf/77LPPqnz58tZtbm5uCgwMVJ8+feTm5mZURKDIWOzJQdLT0zVw4MDrXq+Uw2gAwDlYLBb9+uuvBcYvXLjAvwUlYPLkyfL39y+wsNbixYuVnp6ukSNHGpTMebRp00YHDx6kyBpg3LhxkqTAwEBFRERYD68HSgMXowOUVs8884wuXLigzZs3y9PTU2vWrNHSpUvVoEEDffbZZ0bHAwCUkI4dO2ry5Mk2pTU3N1eTJ08ucJ1rFL8333xTjRo1KjDepEkTxcXFGZDI+QwdOlTPPvuslixZou3bt2vXrl02NzjegAED5OHhoZycHJ04cUIpKSk2N8CMOLTYQapXr65PP/1UrVu3lo+Pj7Zt26bbbrtNn332maZNm6aNGzcaHRH/H4cWA3Ck3bt3q2PHjvLz89Mdd9whSdqwYYMyMjL0zTffqGnTpgYnLN08PDy0Z88e1alTx2b88OHDCgoK0uXLlw1K5jxcXAruN7FYLMrPz2expxJy4MABPfLII9q0aZPNOO8BzIxDix0kKyvLusR5hQoVlJ6erttuu03BwcHasWOHwekAACUlKChIu3bt0pw5c5ScnCxPT09FRkYqOjpaFStWNDpeqRcQEKCEhIQCRTYhIUE1atQwKJVzOXLkiNERnN7AgQNVpkwZff7554UuuAWYEUXWQRo2bKh9+/YpMDBQzZs315tvvqnAwEDFxcWpevXqRscDAJSgGjVqaNKkSUbHcEpDhgzRM888oytXrujOO++UJMXHx+v555/Xs88+a3A651C7dm2jIzi9pKQkbd++vdDD7AGzosg6yLBhw3Tq1ClJ106079Gjh9555x25ublp6dKlBqdzPpcvX77uAgdvvvmm/P39SzgRAGeyYcMGvfnmmzp8+LA++OAD1axZU2+//bbq1KnDebIO9txzz+ns2bN68sknrVcQ8PDw0MiRIzV69GiD0zmPt99+W3FxcTpy5IgSExNVu3ZtzZo1S3Xq1NF9991ndLxSLygoSGfOnDE6BlCsWOzJQR566CENHDhQktSyZUsdO3ZM27Zt04kTJxQREWFsOCeRl5enl19+WTVr1lT58uV1+PBhSdLYsWO1aNEi67x+/fqpXLlyRsUEUMp99NFH6t69uzw9PbVjxw5lZ2dLurZqMXtpHc9isWjq1KlKT0/X999/r+TkZJ07d06xsbFGR3Ma8+bNU0xMjO6++26dP3/eej6mn5+fZs2aZWw4JzF16lQ9//zzWr9+vc6ePauMjAybG2BGFFkHWrRokZo2bSoPDw9VqFBBkZGR+uSTT4yO5TQmTpyoJUuWaNq0aTbXR2vatKkWLlxoYDIAzmTixImKi4vTggULVLZsWet4+/btWTOhBJUvX16tWrVS06ZN5e7ubnQcp/LGG29owYIFevHFF+Xq6modDwsL0w8//GBgMucRHh6u77//Xl27dlXVqlVVoUIFVahQQX5+fqpQoYLR8YAi4dBiB4mNjdWMGTM0dOhQtWvXTpKUmJio4cOHKyUlRRMmTDA4Yem3bNkyzZ8/X127dtXjjz9uHW/evLn27t1rYDIAzmTfvn3q2LFjgXFfX1+dP3++5AMBJezIkSNq0aJFgXF3d3dlZWUZkMj5rFu3zugIQLGjyDrIvHnztGDBAvXt29c6du+996pZs2YaOnQoRbYEnDx5stCLr+fl5enKlSsGJALgjKpVq6aDBw8qMDDQZnzjxo1c9gtOoU6dOkpKSiqw6NOaNWvUuHFjg1I5l06dOhkdASh2HFrsIFeuXFFYWFiB8dDQUF29etWARM4nKChIGzZsKDD+4YcfFvrJMAA4wpAhQzRs2DBt3rxZFotFP//8s959912NGDFCTzzxhNHxAIeLiYnRU089pZUrVyo/P19btmzRK6+8otGjR+v55583Op7T2LBhgx566CHdfvvtOnnypKRri3Bt3LjR4GRA0bBH1kEefvhhzZs3TzNmzLAZnz9/vvr3729QKucSGxurAQMG6OTJk8rLy9OqVau0b98+LVu2TJ9//rnR8QA4iVGjRikvL09du3bVxYsX1bFjR7m7u2vEiBEaOnSo0fEAhxs8eLA8PT01ZswYXbx4Uf369VONGjX0+uuv69///rfR8ZzCRx99pIcfflj9+/cvdNG5L774wuCEgP0s+fn5+UaHKC1iYmKsX1+9elVLlixRrVq11LZtW0nS5s2blZKSosjISL3xxhtGxXQqGzZs0IQJE5ScnKzMzEy1bNlSsbGx6tatm9HRADiB3NxcJSQkqFmzZvLy8tLBgweVmZmpoKAglS9f3uh4QIm7ePGiMjMzVbVq1QLbEhISFBYWxmJcDtCiRQsNHz5ckZGR8vb2VnJysurWraudO3fqn//8p1JTU42OCNiNIluMunTp8rfmWSwWffPNNw5OAwC4FXh4eGjPnj2qU6eO0VGAW5qPj4+SkpI4d9wBvLy8tHv3bgUGBtoU2cOHDysoKEiXL182OiJgNw4tLkasCAcA+LOmTZvq8OHDFFngL7BvxXFYdA6lEUUWpUqFChVksVj+1txz5845OA0AXLuO7IgRI/Tyyy8rNDRU5cqVs9nu4+NjUDIAzuK3RecWL15sXXQuMTFRI0aM0NixY42OBxQJRRalyqxZs4yOAAA27r77bknXLsH2xw/a8vPzZbFYlJuba1Q0AE6CRedQGnGOLAAADvTtt9/ecDvXdwSu+eO5m3CMnJwcFp1DqUGRRamWm5urjz/+WHv27JF07dqy9913n8qU4WAEAABuJSz25DgXLlxQbm6uKlasaDN+7tw5lSlThlMcYEr8NY9S66efftK9996r1NRUNWzYUJI0depUValSRf/973/VtGlTgxMCKK127dqlpk2bysXFRbt27brh3GbNmpVQKuDWxr4Vx/n3v/+tnj176sknn7QZf//99/XZZ59xHVmYEntkUWq1a9dOVapU0dKlS1WhQgVJ0i+//KKBAwcqPT1dmzZtMjghgNLKxcVFqampqlq1qlxcXGSxWAr9I51zZOEsrl69qvXr1+vQoUPq16+fvL299fPPP8vHx4fDW0tAxYoVlZCQoMaNG9uM7927V+3bt9fZs2cNSgYUHXtkUWolJSVp27Zt1hIrXVvV+JVXXlGrVq0MTAagtDty5IiqVKli/RpwZseOHVOPHj2UkpKi7Oxs3XXXXfL29tbUqVOVnZ2tuLg4oyOWetnZ2bp69WqB8StXrujSpUsGJAJunovRAQBHue2225SWllZg/PTp06pfv74BiQA4i9q1a1tXKK5du/YNb0BpN2zYMIWFhemXX36Rp6endbx3796Kj483MJnzaN26tebPn19gPC4uTqGhoQYkAm4ee2RRak2ePFlPP/20XnrpJbVt21aS9P3332vChAmaOnWqMjIyrHNZ5ABAcfrss8/+9tx7773XgUkA423YsEGbNm2Sm5ubzXhgYKBOnjxpUCrnMnHiRIWHhys5OVldu3aVJMXHx2vr1q1au3atwemAouEcWZRaLi6/H3Dw256R3/5z/+N9zlEDUNz++PtHUoFzZP94PVl+/6C0q1ChghISEhQUFGRziZ2NGzeqT58+hR49heKXnJysadOmKSkpSZ6enmrWrJlGjx6tBg0aGB0NKBL2yKLUWrdundERADipvLw869dff/21Ro4cqUmTJqldu3aSpMTERI0ZM0aTJk0yKiJQYrp166ZZs2ZZD221WCzKzMzUuHHjdPfddxucrvS7cuWKHnvsMY0dO1bvvvuu0XGAYsMeWQAAHKhp06aKi4tThw4dbMY3bNigRx991Hqda6C0OnHihLp37678/HwdOHBAYWFhOnDggCpXrqzvvvtOVatWNTpiqefr66ukpCTVqVPH6ChAsaHIolS7fPmydu3apdOnT9vsIZE4Lw1AyfD09NTWrVsLXLt6165datOmDSuGwilcvXpVK1euVHJysjIzM9WyZUv179/fZvEnOM6AAQMUEhKi4cOHGx0FKDYUWZRaa9asUWRkpM6cOVNgG+fFAigpHTt2lIeHh95++235+/tLktLS0hQZGanLly/r22+/NTghgNJu4sSJmj59urp27arQ0FCVK1fOZvvTTz9tUDKg6CiyKLUaNGigbt26KTY21vrHIwCUtIMHD6p3797av3+/AgICJEnHjx9XgwYN9Mknn3A5MJR6kydPlr+/vx555BGb8cWLFys9PV0jR440KJnzuNEhxRaLRYcPHy7BNEDxoMii1PLx8dHOnTtVr149o6MAcHL5+fn66quvtHfvXklS48aNFR4ebrN6MVBaBQYGavny5br99tttxjdv3qx///vfOnLkiEHJAJgZqxaj1HrggQe0fv16iiwAw1ksFnXr1k3dunUzOgpQ4lJTU1W9evUC41WqVNGpU6cMSOS8cnJydOTIEdWrV09lylADYG78F4xSa86cOfrXv/6lDRs2KDg4WGXLlrXZzvkgABxl9uzZevTRR+Xh4aHZs2ffcC6/i1DaBQQEKCEhocDhrQkJCapRo4ZBqZzLxYsXNXToUC1dulSStH//ftWtW1dDhw5VzZo1NWrUKIMTAvbj0GKUWosWLdLjjz8uDw8PVapUyeYQPs4HAeBIderU0bZt21SpUiXOTYPTmzZtmqZNm6ZXX31Vd955pyQpPj5ezz//vJ599lmNHj3a4ISl37Bhw5SQkKBZs2apR48e2rVrl+rWratPP/1UL730knbu3Gl0RMBuFFmUWtWqVdPTTz+tUaNGycXFxeg4AKDf/snl3Fg4k/z8fI0aNUqzZ89WTk6OJMnDw0MjR45UbGyswemcQ+3atbVy5Uq1bdtW3t7eSk5OVt26dXXw4EG1bNlSGRkZRkcE7MZf9yi1cnJyFBERQYkFYLhFixapadOm8vDwkIeHh5o2baqFCxcaHQsoERaLRVOnTlV6erq+//57JScn69y5c5TYEpSenq6qVasWGM/KyuKDNZgWf+Gj1BowYIBWrlxpdAwATi42NlbDhg1Tz5499cEHH+iDDz5Qz549NXz4cP6Qh1MpX768WrVqpaZNm8rd3d3oOE4lLCxMq1evtt7/rbwuXLhQ7dq1MyoWcFM4tBil1tNPP61ly5apefPmatasWYHFnmbMmGFQMgDOpEqVKpo9e7b69u1rM/7ee+9p6NChOnPmjEHJgJKRlZWlKVOmKD4+XqdPn1ZeXp7Nds4Td7yNGzfqn//8px566CEtWbJEjz32mHbv3q1Nmzbp22+/VWhoqNERAbuxajFKrR9++EEtWrSQJP3444822ziMBkBJuXLlisLCwgqMh4aG6urVqwYkAkrW4MGD9e233+rhhx9W9erV+TfYAB06dFBSUpKmTJmi4OBgrV27Vi1btlRiYqKCg4ONjgcUCXtkAQBwoKFDh6ps2bIFjgIZMWKELl26pLlz5xqUDCgZfn5+Wr16tdq3b290FAClCHtkAQAoZjExMdavLRaLFi5cqLVr16pt27aSpM2bNyslJUWRkZFGRQRKTIUKFVSxYkWjYzi93Nxcffzxx9qzZ48kKSgoSPfdd5/KlKEOwJzYI4tS5f7779eSJUvk4+Oj+++//4ZzV61aVUKpADibLl26/K15FotF33zzjYPTAMZ655139Omnn2rp0qXy8vIyOo5T+umnn3TvvfcqNTVVDRs2lCTt379fVapU0X//+181bdrU4ISA/fgIBqWKr6+v9dwbX19fg9MAcFbr1q0zOgJwy5g+fboOHTokf39/BQYGFlh8cceOHQYlcx6DBw9WkyZNtG3bNlWoUEGS9Msvv2jgwIF69NFHtWnTJoMTAvZjjyxKrUuXLikvL0/lypWTJB09elSffPKJGjdurO7duxucDgAA5zB+/Pgbbh83blwJJXFenp6e2rZtm5o0aWIz/uOPP6pVq1a6dOmSQcmAomOPLEqt++67T/fff78ef/xxnT9/Xm3btlXZsmV15swZzZgxQ0888YTREQEAKPUoqsa77bbblJaWVqDInj59WvXr1zcoFXBzXIwOADjKjh07dMcdd0iSPvzwQ/n7++vYsWNatmyZZs+ebXA6AACcx/nz57Vw4UKNHj1a586dk3Tt3+mTJ08anMw5TJ48WU8//bQ+/PBDnThxQidOnNCHH36oZ555RlOnTlVGRob1BpgFhxaj1PLy8tLevXtVq1YtPfjgg2rSpInGjRun48ePq2HDhrp48aLREQEAKPV27dql8PBw+fr66ujRo9q3b5/q1q2rMWPGKCUlRcuWLTM6Yqnn4vL7vqvf1hL5rQL88b7FYlFubm7JBwSKgEOLUWrVr19fn3zyiXr37q0vv/xSw4cPl3TtMBofHx+D0wEA4BxiYmI0cOBATZs2Td7e3tbxu+++W/369TMwmfNgATqURhRZlFqxsbHq16+fhg8frq5du6pdu3aSpLVr16pFixYGpwMAwDls3bpVb775ZoHxmjVrKjU11YBEzqdTp05/a96TTz6pJk2aqHLlyg5OBNw8zpFFqfXAAw8oJSVF27Zt05o1a6zjXbt21cyZMw1MBgCA83B3dy/03MvfrmOKW8c777zDebIwDYosSrVq1aqpRYsWNueGtG7dWo0aNTIwFQAAzuPee+/VhAkTdOXKFUnXzslMSUnRyJEj1adPH4PT4Y9YOgdmQpEFAACAw0yfPl2ZmZmqWrWqLl26pE6dOql+/fry9vbWK6+8YnQ8ACbFObIAAABwGF9fX3311VdKSEhQcnKyMjMz1bJlS4WHhxsdDYCJcfkdAAAAOMyyZcsUEREhd3d3m/GcnBytWLFCkZGRBiXDn3l7eys5OVl169Y1OgrwlyiyAAAAcBhXV1edOnVKVatWtRk/e/asqlatynVLbyEUWZgJ58gCAADAYfLz82WxWAqMnzhxQr6+vgYkwvU89NBD8vHxMToG8LdwjiwAAACKXYsWLWSxWGSxWNS1a1eVKfP7n525ubk6cuSIevToYWBC57FlyxYlJiZar9tbrVo1tWvXTq1bt7aZN2/ePCPiAUVCkQUAAECx69WrlyQpKSlJ3bt3V/ny5a3b3NzcFBgYyOV3HOz06dPq06ePEhISVKtWLfn7+0uS0tLSNHz4cLVv314fffRRgcO+ATPgHFkAAAA4zNKlSxURESEPDw+jozidBx54QD///LPeeustNWzY0Gbbvn379Mgjj6hGjRr64IMPDEoIFB1FFgAAAA6Xk5Oj06dPKy8vz2a8Vq1aBiUq/by9vfXdd9+pRYsWhW7fvn27OnfurF9//bWEkwE3j0OLAQAA4DAHDhzQI488ok2bNtmM/7YIFKsWO467u7syMjKuu/3XX38tcFkkwCwosgAAAHCYgQMHqkyZMvr8889VvXr1QlcwhmNERERowIABmjlzprp27WpdkTgjI0Px8fGKiYlR3759DU4JFA2HFgMAAMBhypUrp+3bt6tRo0ZGR3E62dnZeuaZZ7R48WJdvXpVbm5ukq4d5l2mTBkNGjRIM2fOZK8sTIkiCwAAAIdp1aqVZs6cqQ4dOhgdxWllZGRo+/btNpffCQ0N5ZqxMDWKLAAAABzmm2++0ZgxYzRp0iQFBwerbNmyNtspUwCKgiILAAAAh3FxcZGkAufGstiT8dLS0vTmm28qNjbW6CiA3SiyAAAAcJhvv/32hts7depUQknwZ8nJyWrZsiUfJsCUWLUYAAAADkNRNc6uXbtuuH3fvn0llAQofuyRBQAAgENt2LBBb775pg4fPqwPPvhANWvW1Ntvv606deqwCJQDubi4yGKxqLA/938b5/BumJWL0QEAAABQen300Ufq3r27PD09tWPHDmVnZ0uSLly4oEmTJhmcrnSrWLGiFixYoCNHjhS4HT58WJ9//rnREYEi49BiAAAAOMzEiRMVFxenyMhIrVixwjrevn17TZw40cBkpV9oaKh+/vln1a5du9Dt58+fL3RvLWAGFFkAAAA4zL59+9SxY8cC476+vjp//nzJB3Iijz/+uLKysq67vVatWnrrrbdKMBFQfCiyAAAAcJhq1arp4MGDCgwMtBnfuHGj6tata0woJ9G7d+8bbq9QoYIGDBhQQmmA4sU5sgAAAHCYIUOGaNiwYdq8ebMsFot+/vlnvfvuuxoxYoSeeOIJo+PhD3x8fHT48GGjYwB/C3tkAQAA4DCjRo1SXl6eunbtqosXL6pjx45yd3fXiBEjNHToUKPj4Q84XxZmwuV3AAAA4HA5OTk6ePCgMjMzFRQUpPLlyxsdCX/i7e2t5ORkDvmGKXBoMQAAABzOzc1NQUFBatSokb7++mvt2bPH6EgATIwiCwAAAId58MEHNWfOHEnSpUuX1KpVKz344INq1qyZPvroI4PTATAriiwAAAAc5rvvvtMdd9whSfr444+Vl5en8+fPa/bs2VxH9hZjsViMjgD8bRRZAAAAOMyFCxdUsWJFSdKaNWvUp08feXl56Z577tGBAwcMToc/YukcmAlFFgAAAA4TEBCgxMREZWVlac2aNerWrZsk6ZdffpGHh4fB6ZxPfn7+dQvr//73P9WsWbOEEwFFQ5EFAACAwzzzzDPq37+//vGPf6hGjRrq3LmzpGuHHAcHBxsbzoksWrRITZs2lYeHhzw8PNS0aVMtXLjQZk6HDh3k7u5uUELAPlx+BwAAAA61fft2paSk6K677rJedmf16tXy8/NT+/btDU5X+sXGxmrGjBkaOnSo2rVrJ0lKTEzUnDlzNHz4cE2YMMHghID9KLIAAAAwnI+Pj5KSkriGqQNUqVJFs2fPVt++fW3G33vvPQ0dOlRnzpwxKBlQdBxaDAAAAMOxb8Vxrly5orCwsALjoaGhunr1qgGJgJtHkQUAAABKsYcffljz5s0rMD5//nz179/fgETAzStjdAAAAAAAxSsmJsb6tcVi0cKFC7V27Vq1bdtWkrR582alpKQoMjLSqIjATaHIAgAAAKXMzp07be6HhoZKkg4dOiRJqly5sipXrqyffvqpxLMBxYEiCwAAAMNZLBajI5Qq69atMzoC4FCcIwsAAADDsdgTAHuwRxYAAAAl4reyWtje1//973+qWbNmSUdyCl26dLnhHu9vvvmmBNMAxYM9sgAAAHCoRYsWqWnTpvLw8JCHh4eaNm2qhQsX2szp0KGD3N3dDUpYuoWEhKh58+bWW1BQkHJycrRjxw4FBwcbHQ8oEvbIAgAAwGFiY2M1Y8YMDR06VO3atZMkJSYmavjw4UpJSdGECRMMTlj6zZw5s9Dxl156SZmZmSWcBigelnxOSAAAAICDVKlSRbNnz1bfvn1txt977z0NHTpUZ86cMSgZDh48qNatW+vcuXNGRwHsxqHFAAAAcJgrV64oLCyswHhoaKiuXr1qQCL8JjExUR4eHkbHAIqEQ4sBAADgMA8//LDmzZunGTNm2IzPnz9f/fv3NyiVc7n//vtt7ufn5+vUqVPatm2bxo4da1Aq4OZQZAEAAFCsYmJirF9bLBYtXLhQa9euVdu2bSVJmzdvVkpKiiIjI42K6FR8fX1t7ru4uKhhw4aaMGGCunXrZlAq4OZwjiwAAACKVZcuXf7WPIvFwqVfABQJRRYAAABwAjk5OTp9+rTy8vJsxmvVqmVQIqDoOLQYAAAAKMX279+vQYMGadOmTTbj+fn5slgsys3NNSgZUHQUWQAAADhMly5dZLFYrrudQ4sdLyoqSmXKlNHnn3+u6tWr3/D9AMyCIgsAAACHCQkJsbl/5coVJSUl6ccff9SAAQOMCeVkkpKStH37djVq1MjoKECxocgCAADAYWbOnFno+EsvvaTMzMwSTuOcgoKCdObMGaNjAMWKxZ4AAABQ4g4ePKjWrVvr3LlzRkcplTIyMqxfb9u2TWPGjNGkSZMUHByssmXL2sz18fEp6XjATWOPLAAAAEpcYmKiPDw8jI5Ravn5+dmcC5ufn6+uXbvazGGxJ5gZRRYAAAAOc//999vcz8/P16lTp7Rt2zaNHTvWoFSl37p164yOADgUhxYDAADAYaKiomzuu7i4qEqVKrrzzjvVrVs3g1KhME8++aQmTJigypUrGx0F+EsUWQAAAADy8fFRUlKS6tata3QU4C9xaDEAAAAcLicnR6dPn1ZeXp7NeK1atQxKhD9j/xbMhCILAAAAh9m/f78GDRqkTZs22Yyz0BCAm0GRBQAAgMNERUWpTJky+vzzz1W9enWblXQBoKgosgAAAHCYpKQkbd++XY0aNTI6CoBSxMXoAAAAACi9goKCdObMGaNjAChlKLIAAAAoVhkZGdbb1KlT9fzzz2v9+vU6e/aszbaMjAyjo5Za999/v/Xnu2zZMmVnZ//lYx566CH5+Pg4OhpQLLj8DgAAAIqVi4uLzbmwvy3s9Ecs9uRYbm5uOnbsmKpXry5XV1edOnVKVatWNToWUGw4RxYAAADFat26dUZHcHqNGjXS6NGj1aVLF+Xn5+v999+/7t7WyMjIEk4H3Dz2yAIAAMBwTz75pCZMmKDKlSsbHaVU2LRpk2JiYnTo0CGdO3dO3t7eha4YbbFYdO7cOQMSAjeHIgsAAADD+fj4KCkpSXXr1jU6Sqnj4uKikydPqnr16jbj+fn5SklJUe3atQ1KBhQdiz0BAADAcOxbKXnnzp3jgwOYFkUWAAAAKOVcXV0LjGVmZsrDw8OANMDNY7EnAAAAoBSKiYmRdO082NjYWHl5eVm35ebmavPmzQoJCTEoHXBzKLIAAABAKbRz505J1w7b/uGHH+Tm5mbd5ubmpubNm2vEiBFGxQNuCkUWAAAAKIV+uwxSVFSUXn/99etefgcwI86RBQAAQLG6//77lZGRIUlatmyZsrOz//IxDz30EEXLQd566y1+tih1uPwOAAAAipWbm5uOHTum6tWry9XVVadOnVLVqlWNjgWgFOHQYgAAABSrRo0aafTo0erSpYvy8/P1/vvvX3ePYGRkZAmnA1AasEcWAAAAxWrTpk2KiYnRoUOHdO7cOXl7e8tisRSYZ7FYdO7cOQMSAjA7iiwAAAAcxsXFRSdPnlT16tVtxvPz85WSkqLatWsblAyAmbHYEwAAAErcuXPnVLduXaNjADApiiwAAAAcytXVtcBYZmamPDw8DEgDoDRgsScAAAAUu5iYGEnXzoONjY2Vl5eXdVtubq42b96skJAQg9IBMDuKLAAAAIrdzp07JV07F/aHH36Qm5ubdZubm5uaN2+uESNGGBUPgMmx2BMAAAAcJioqSq+//vp1L78DAEVBkQUAAAAAmAqLPQEAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFOhyAIAAAAATIUiCwAAAAAwFYosAAAAAMBUKLIAAAAAAFP5fx9ncY0Zd2GLAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1000x700 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot and compare all of the model results\n",
    "all_model_results.plot(kind=\"bar\", figsize=(10, 7)).legend(bbox_to_anchor=(1.0, 1.0));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 763
    },
    "id": "4toZoopG2NGo",
    "outputId": "817936c5-bc5e-4adb-aed2-5085501bfafe"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: >"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzoAAALqCAYAAAD9+CEvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABcHUlEQVR4nO3deVyVZeL+8escFBAF3FEZFJfSyAUFNbI0k9SpX4vZxKSFUdK0aCZZ5mRYZqItRJYTuY3aMtpeU45jkZYLZYJgm+aOSyBmQqCCwPn94ddTZ0AT45zH7vN5v17nNXI/z4ELzwRe57mf+7Y5HA6HAAAAAMAgdqsDAAAAAEBdo+gAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOPWsDnAmqqqqtH//fgUGBspms1kdBwAAAIBFHA6Hfv75Z7Vp00Z2+6mv2/whis7+/fsVFhZmdQwAAAAA54g9e/boT3/60ymP/yGKTmBgoKQT30xQUJDFaQAAAABYpbi4WGFhYc6OcCp/iKJzcrpaUFAQRQcAAADAb97SwmIEAAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA45xV0Zk9e7bCw8Pl7++vvn37av369ac9Py0tTZ07d1aDBg0UFham8ePH69ixY2cVGAAAAAB+S62LztKlS5WUlKQpU6YoOztbPXr00JAhQ3TgwIEaz3/ttdf00EMPacqUKfruu+80f/58LV26VH//+99/d3gAAAAAqEmti05qaqoSExOVkJCgiIgIpaenKyAgQAsWLKjx/HXr1qlfv34aMWKEwsPDNXjwYN10002/eRUIAAAAAM5WrYpOeXm5srKyFBsb+8snsNsVGxurzMzMGp9z8cUXKysry1lsduzYoWXLlunKK6885dcpKytTcXGxywMAAAAAzlS92px88OBBVVZWKiQkxGU8JCREmzdvrvE5I0aM0MGDB3XJJZfI4XCooqJCd95552mnrqWkpOixxx6rTTQAAAAAcHL7qmurVq3S9OnT9Y9//EPZ2dl6++239eGHH+rxxx8/5XMmTZqkoqIi52PPnj3ujgkAAADAILW6otO8eXP5+PiooKDAZbygoECtWrWq8TmPPPKIbrnlFo0ePVqS1K1bN5WWluqOO+7Qww8/LLu9etfy8/OTn59fbaIBAAAAgFOtruj4+voqKipKGRkZzrGqqiplZGQoJiamxuccOXKkWpnx8fGRJDkcjtrmBQAAAIDfVKsrOpKUlJSkUaNGKTo6Wn369FFaWppKS0uVkJAgSYqPj1doaKhSUlIkSVdffbVSU1PVs2dP9e3bV9u2bdMjjzyiq6++2ll4AAAAAKAu1broxMXFqbCwUMnJycrPz1dkZKSWL1/uXKAgLy/P5QrO5MmTZbPZNHnyZO3bt08tWrTQ1VdfrSeeeKLuvgsAAAAA+BWb4w8wf6y4uFjBwcEqKipSUFCQ1XEAAAAAWORMu0Gtr+iYKvyhD62O8LvsmnGV1REAAACAc4bbl5cGAAAAAE/jig7OGVxVAwAAQF2h6ABwomwCAABTUHQA4BxC2QQAoG5wjw4AAAAA41B0AAAAABiHqWsAAPyfP/rUQYnpgwBwEkUHAACcMyibAOoKU9cAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAOixEAAADAiQUhYAqu6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA47KMDAAAAnEPYy6hucEUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADDOWRWd2bNnKzw8XP7+/urbt6/Wr19/ynMvu+wy2Wy2ao+rrrrqrEMDAAAAwOnUuugsXbpUSUlJmjJlirKzs9WjRw8NGTJEBw4cqPH8t99+Wz/88IPz8fXXX8vHx0d/+ctffnd4AAAAAKhJrYtOamqqEhMTlZCQoIiICKWnpysgIEALFiyo8fymTZuqVatWzsdHH32kgIAAig4AAAAAt6lV0SkvL1dWVpZiY2N/+QR2u2JjY5WZmXlGn2P+/Pn661//qoYNG57ynLKyMhUXF7s8AAAAAOBM1aroHDx4UJWVlQoJCXEZDwkJUX5+/m8+f/369fr66681evTo056XkpKi4OBg5yMsLKw2MQEAAAB4OY+uujZ//nx169ZNffr0Oe15kyZNUlFRkfOxZ88eDyUEAAAAYIJ6tTm5efPm8vHxUUFBgct4QUGBWrVqddrnlpaWasmSJZo6depvfh0/Pz/5+fnVJhoAAAAAONXqio6vr6+ioqKUkZHhHKuqqlJGRoZiYmJO+9w33nhDZWVluvnmm88uKQAAAACcoVpd0ZGkpKQkjRo1StHR0erTp4/S0tJUWlqqhIQESVJ8fLxCQ0OVkpLi8rz58+fruuuuU7NmzeomOQAAAACcQq2LTlxcnAoLC5WcnKz8/HxFRkZq+fLlzgUK8vLyZLe7XijasmWL1qxZoxUrVtRNagAAAAA4jVoXHUkaM2aMxowZU+OxVatWVRvr3LmzHA7H2XwpAAAAAKg1j666BgAAAACeQNEBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxjmrojN79myFh4fL399fffv21fr16097/uHDh3XPPfeodevW8vPz0/nnn69ly5adVWAAAAAA+C31avuEpUuXKikpSenp6erbt6/S0tI0ZMgQbdmyRS1btqx2fnl5ua644gq1bNlSb775pkJDQ7V79241bty4LvIDAAAAQDW1LjqpqalKTExUQkKCJCk9PV0ffvihFixYoIceeqja+QsWLNChQ4e0bt061a9fX5IUHh7++1IDAAAAwGnUaupaeXm5srKyFBsb+8snsNsVGxurzMzMGp/z/vvvKyYmRvfcc49CQkLUtWtXTZ8+XZWVlaf8OmVlZSouLnZ5AAAAAMCZqlXROXjwoCorKxUSEuIyHhISovz8/Bqfs2PHDr355puqrKzUsmXL9Mgjj+iZZ57RtGnTTvl1UlJSFBwc7HyEhYXVJiYAAAAAL+f2VdeqqqrUsmVLzZkzR1FRUYqLi9PDDz+s9PT0Uz5n0qRJKioqcj727Nnj7pgAAAAADFKre3SaN28uHx8fFRQUuIwXFBSoVatWNT6ndevWql+/vnx8fJxjF1xwgfLz81VeXi5fX99qz/Hz85Ofn19togEAAACAU62u6Pj6+ioqKkoZGRnOsaqqKmVkZCgmJqbG5/Tr10/btm1TVVWVc+z7779X69atayw5AAAAAPB71XrqWlJSkubOnatFixbpu+++01133aXS0lLnKmzx8fGaNGmS8/y77rpLhw4d0rhx4/T999/rww8/1PTp03XPPffU3XcBAAAAAL9S6+Wl4+LiVFhYqOTkZOXn5ysyMlLLly93LlCQl5cnu/2X/hQWFqb//ve/Gj9+vLp3767Q0FCNGzdOEydOrLvvAgAAAAB+pdZFR5LGjBmjMWPG1Hhs1apV1cZiYmL0+eefn82XAgAAAIBac/uqawAAAADgaRQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYJyzKjqzZ89WeHi4/P391bdvX61fv/6U5y5cuFA2m83l4e/vf9aBAQAAAOC31LroLF26VElJSZoyZYqys7PVo0cPDRkyRAcOHDjlc4KCgvTDDz84H7t37/5doQEAAADgdGpddFJTU5WYmKiEhARFREQoPT1dAQEBWrBgwSmfY7PZ1KpVK+cjJCTkd4UGAAAAgNOpVdEpLy9XVlaWYmNjf/kEdrtiY2OVmZl5yueVlJSoXbt2CgsL07XXXqtvvvnmtF+nrKxMxcXFLg8AAAAAOFO1KjoHDx5UZWVltSsyISEhys/Pr/E5nTt31oIFC/Tee+/plVdeUVVVlS6++GLt3bv3lF8nJSVFwcHBzkdYWFhtYgIAAADwcm5fdS0mJkbx8fGKjIzUgAED9Pbbb6tFixZ66aWXTvmcSZMmqaioyPnYs2ePu2MCAAAAMEi92pzcvHlz+fj4qKCgwGW8oKBArVq1OqPPUb9+ffXs2VPbtm075Tl+fn7y8/OrTTQAAAAAcKrVFR1fX19FRUUpIyPDOVZVVaWMjAzFxMSc0eeorKzUV199pdatW9cuKQAAAACcoVpd0ZGkpKQkjRo1StHR0erTp4/S0tJUWlqqhIQESVJ8fLxCQ0OVkpIiSZo6daouuugiderUSYcPH9ZTTz2l3bt3a/To0XX7nQAAAADA/6l10YmLi1NhYaGSk5OVn5+vyMhILV++3LlAQV5enuz2Xy4U/fTTT0pMTFR+fr6aNGmiqKgorVu3ThEREXX3XQAAAADAr9S66EjSmDFjNGbMmBqPrVq1yuXjZ599Vs8+++zZfBkAAAAAOCtuX3UNAAAAADyNogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMc1ZFZ/bs2QoPD5e/v7/69u2r9evXn9HzlixZIpvNpuuuu+5sviwAAAAAnJFaF52lS5cqKSlJU6ZMUXZ2tnr06KEhQ4bowIEDp33erl27NGHCBF166aVnHRYAAAAAzkSti05qaqoSExOVkJCgiIgIpaenKyAgQAsWLDjlcyorKzVy5Eg99thj6tChw+8KDAAAAAC/pVZFp7y8XFlZWYqNjf3lE9jtio2NVWZm5imfN3XqVLVs2VK33377GX2dsrIyFRcXuzwAAAAA4EzVqugcPHhQlZWVCgkJcRkPCQlRfn5+jc9Zs2aN5s+fr7lz557x10lJSVFwcLDzERYWVpuYAAAAALycW1dd+/nnn3XLLbdo7ty5at68+Rk/b9KkSSoqKnI+9uzZ48aUAAAAAExTrzYnN2/eXD4+PiooKHAZLygoUKtWraqdv337du3atUtXX321c6yqqurEF65XT1u2bFHHjh2rPc/Pz09+fn61iQYAAAAATrW6ouPr66uoqChlZGQ4x6qqqpSRkaGYmJhq53fp0kVfffWVcnJynI9rrrlGAwcOVE5ODlPSAAAAALhFra7oSFJSUpJGjRql6Oho9enTR2lpaSotLVVCQoIkKT4+XqGhoUpJSZG/v7+6du3q8vzGjRtLUrVxAAAAAKgrtS46cXFxKiwsVHJysvLz8xUZGanly5c7FyjIy8uT3e7WW38AAAAA4LRqXXQkacyYMRozZkyNx1atWnXa5y5cuPBsviQAAAAAnDEuvQAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGOesis7s2bMVHh4uf39/9e3bV+vXrz/luW+//baio6PVuHFjNWzYUJGRkXr55ZfPOjAAAAAA/JZaF52lS5cqKSlJU6ZMUXZ2tnr06KEhQ4bowIEDNZ7ftGlTPfzww8rMzNSmTZuUkJCghIQE/fe///3d4QEAAACgJrUuOqmpqUpMTFRCQoIiIiKUnp6ugIAALViwoMbzL7vsMg0bNkwXXHCBOnbsqHHjxql79+5as2bN7w4PAAAAADWpVdEpLy9XVlaWYmNjf/kEdrtiY2OVmZn5m893OBzKyMjQli1b1L9//9qnBQAAAIAzUK82Jx88eFCVlZUKCQlxGQ8JCdHmzZtP+byioiKFhoaqrKxMPj4++sc//qErrrjilOeXlZWprKzM+XFxcXFtYgIAAADwcrUqOmcrMDBQOTk5KikpUUZGhpKSktShQwdddtllNZ6fkpKixx57zBPRAAAAABioVkWnefPm8vHxUUFBgct4QUGBWrVqdcrn2e12derUSZIUGRmp7777TikpKacsOpMmTVJSUpLz4+LiYoWFhdUmKgAAAAAvVqt7dHx9fRUVFaWMjAznWFVVlTIyMhQTE3PGn6eqqsplatr/8vPzU1BQkMsDAAAAAM5UraeuJSUladSoUYqOjlafPn2Ulpam0tJSJSQkSJLi4+MVGhqqlJQUSSemoUVHR6tjx44qKyvTsmXL9PLLL+vFF1+s2+8EAAAAAP5PrYtOXFycCgsLlZycrPz8fEVGRmr58uXOBQry8vJkt/9yoai0tFR333239u7dqwYNGqhLly565ZVXFBcXV3ffBQAAAAD8ylktRjBmzBiNGTOmxmOrVq1y+XjatGmaNm3a2XwZAAAAADgrtd4wFAAAAADOdRQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYJyzKjqzZ89WeHi4/P391bdvX61fv/6U586dO1eXXnqpmjRpoiZNmig2Nva05wMAAADA71XrorN06VIlJSVpypQpys7OVo8ePTRkyBAdOHCgxvNXrVqlm266SStXrlRmZqbCwsI0ePBg7du373eHBwAAAICa1LropKamKjExUQkJCYqIiFB6eroCAgK0YMGCGs9/9dVXdffddysyMlJdunTRvHnzVFVVpYyMjN8dHgAAAABqUquiU15erqysLMXGxv7yCex2xcbGKjMz84w+x5EjR3T8+HE1bdq0dkkBAAAA4AzVq83JBw8eVGVlpUJCQlzGQ0JCtHnz5jP6HBMnTlSbNm1cytL/KisrU1lZmfPj4uLi2sQEAAAA4OU8uurajBkztGTJEr3zzjvy9/c/5XkpKSkKDg52PsLCwjyYEgAAAMAfXa2KTvPmzeXj46OCggKX8YKCArVq1eq0z3366ac1Y8YMrVixQt27dz/tuZMmTVJRUZHzsWfPntrEBAAAAODlalV0fH19FRUV5bKQwMmFBWJiYk75vCeffFKPP/64li9frujo6N/8On5+fgoKCnJ5AAAAAMCZqtU9OpKUlJSkUaNGKTo6Wn369FFaWppKS0uVkJAgSYqPj1doaKhSUlIkSTNnzlRycrJee+01hYeHKz8/X5LUqFEjNWrUqA6/FQAAAAA4odZFJy4uToWFhUpOTlZ+fr4iIyO1fPly5wIFeXl5stt/uVD04osvqry8XDfccIPL55kyZYoeffTR35ceAAAAAGpQ66IjSWPGjNGYMWNqPLZq1SqXj3ft2nU2XwIAAAAAzppHV10DAAAAAE+g6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjnFXRmT17tsLDw+Xv76++fftq/fr1pzz3m2++0fDhwxUeHi6bzaa0tLSzzQoAAAAAZ6TWRWfp0qVKSkrSlClTlJ2drR49emjIkCE6cOBAjecfOXJEHTp00IwZM9SqVavfHRgAAAAAfkuti05qaqoSExOVkJCgiIgIpaenKyAgQAsWLKjx/N69e+upp57SX//6V/n5+f3uwAAAAADwW2pVdMrLy5WVlaXY2NhfPoHdrtjYWGVmZtZ5OAAAAAA4G/Vqc/LBgwdVWVmpkJAQl/GQkBBt3ry5zkKVlZWprKzM+XFxcXGdfW4AAAAA5jsnV11LSUlRcHCw8xEWFmZ1JAAAAAB/ILUqOs2bN5ePj48KCgpcxgsKCup0oYFJkyapqKjI+dizZ0+dfW4AAAAA5qtV0fH19VVUVJQyMjKcY1VVVcrIyFBMTEydhfLz81NQUJDLAwAAAADOVK3u0ZGkpKQkjRo1StHR0erTp4/S0tJUWlqqhIQESVJ8fLxCQ0OVkpIi6cQCBt9++63zz/v27VNOTo4aNWqkTp061eG3AgAAAAAn1LroxMXFqbCwUMnJycrPz1dkZKSWL1/uXKAgLy9PdvsvF4r279+vnj17Oj9++umn9fTTT2vAgAFatWrV7/8OAAAAAOB/1LroSNKYMWM0ZsyYGo/9b3kJDw+Xw+E4my8DAAAAAGflnFx1DQAAAAB+D4oOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMA5FBwAAAIBxKDoAAAAAjEPRAQAAAGAcig4AAAAA41B0AAAAABiHogMAAADAOBQdAAAAAMah6AAAAAAwDkUHAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADAORQcAAACAcSg6AAAAAIxD0QEAAABgHIoOAAAAAONQdAAAAAAYh6IDAAAAwDgUHQAAAADGoegAAAAAMM5ZFZ3Zs2crPDxc/v7+6tu3r9avX3/a89944w116dJF/v7+6tatm5YtW3ZWYQEAAADgTNS66CxdulRJSUmaMmWKsrOz1aNHDw0ZMkQHDhyo8fx169bppptu0u23366NGzfquuuu03XXXaevv/76d4cHAAAAgJrUuuikpqYqMTFRCQkJioiIUHp6ugICArRgwYIaz3/uuec0dOhQPfDAA7rgggv0+OOPq1evXnrhhRd+d3gAAAAAqEmtik55ebmysrIUGxv7yyew2xUbG6vMzMwan5OZmelyviQNGTLklOcDAAAAwO9VrzYnHzx4UJWVlQoJCXEZDwkJ0ebNm2t8Tn5+fo3n5+fnn/LrlJWVqayszPlxUVGRJKm4uLg2cWulquyI2z63J7jz78ZTeA2sx2tgPV4Da/3R//4lXoNzAa+B9XgNrOfO1+Dk53Y4HKc9r1ZFx1NSUlL02GOPVRsPCwuzIM0fQ3Ca1QnAa2A9XgPr8RpYj9fAerwG1uM1sJ4nXoOff/5ZwcHBpzxeq6LTvHlz+fj4qKCgwGW8oKBArVq1qvE5rVq1qtX5kjRp0iQlJSU5P66qqtKhQ4fUrFkz2Wy22kQ+JxQXFyssLEx79uxRUFCQ1XG8Eq+B9XgNrMdrYD1eA+vxGliLv3/rmfAaOBwO/fzzz2rTps1pz6tV0fH19VVUVJQyMjJ03XXXSTpRQjIyMjRmzJganxMTE6OMjAzdd999zrGPPvpIMTExp/w6fn5+8vPzcxlr3LhxbaKek4KCgv6w/4cyBa+B9XgNrMdrYD1eA+vxGliLv3/r/dFfg9NdyTmp1lPXkpKSNGrUKEVHR6tPnz5KS0tTaWmpEhISJEnx8fEKDQ1VSkqKJGncuHEaMGCAnnnmGV111VVasmSJNmzYoDlz5tT2SwMAAADAGal10YmLi1NhYaGSk5OVn5+vyMhILV++3LngQF5enuz2XxZzu/jii/Xaa69p8uTJ+vvf/67zzjtP7777rrp27Vp33wUAAAAA/MpZLUYwZsyYU05VW7VqVbWxv/zlL/rLX/5yNl/KCH5+fpoyZUq16XjwHF4D6/EaWI/XwHq8BtbjNbAWf//W86bXwOb4rXXZAAAAAOAPplYbhgIAAADAHwFFBwAAAIBxKDoAAAAAjEPRAQAAAGAcig6AOnf8+HHddttt2rlzp9VRAACAl2LVNTfJy8s77fG2bdt6KAlgjeDgYOXk5Kh9+/ZWR/F6q1ev1ksvvaTt27frzTffVGhoqF5++WW1b99el1xyidXxAI85duyYysvLXcb+yDvDAzi9s9pHB78tPDxcNpvtlMcrKys9mMZ7VVRUaNWqVdq+fbtGjBihwMBA7d+/X0FBQWrUqJHV8Yx23XXX6d1339X48eOtjuLV3nrrLd1yyy0aOXKkNm7cqLKyMklSUVGRpk+frmXLllmcEHCvI0eO6MEHH9Trr7+uH3/8sdpxfh+7R1JS0hmfm5qa6sYk8GYUHTfZuHGjy8fHjx/Xxo0blZqaqieeeMKiVN5l9+7dGjp0qPLy8lRWVqYrrrhCgYGBmjlzpsrKypSenm51RKOdd955mjp1qtauXauoqCg1bNjQ5fi9995rUTLvMm3aNKWnpys+Pl5Llixxjvfr10/Tpk2zMJnZmjRpcto3u37t0KFDbk7j3R544AGtXLlSL774om655RbNnj1b+/bt00svvaQZM2ZYHc9Y//vvoOzsbFVUVKhz586SpO+//14+Pj6KioqyIp5XOnbsmJ5//nmtXLlSBw4cUFVVlcvx7Oxsi5K5D0XHTXr06FFtLDo6Wm3atNFTTz2l66+/3oJU3mXcuHGKjo5Wbm6umjVr5hwfNmyYEhMTLUzmHebPn6/GjRsrKytLWVlZLsdsNhtFx0O2bNmi/v37VxsPDg7W4cOHPR/IS6SlpVkdAf/n3//+txYvXqzLLrtMCQkJuvTSS9WpUye1a9dOr776qkaOHGl1RCOtXLnS+efU1FQFBgZq0aJFatKkiSTpp59+cr4e8Izbb79dK1as0A033KA+ffqc8Zsxf2QUHQ/r3LmzvvzyS6tjeIXVq1dr3bp18vX1dRkPDw/Xvn37LErlPViI4NzQqlUrbdu2TeHh4S7ja9asUYcOHawJ5QVGjRpldQT8n0OHDjn/vx4UFOS8gnbJJZforrvusjKa13jmmWe0YsUKZ8mRTlz1nDZtmgYPHqz777/fwnTe44MPPtCyZcvUr18/q6N4DKuuuUlxcbHLo6ioSJs3b9bkyZN13nnnWR3PK1RVVdU493rv3r0KDAy0IJF3Ki8v15YtW1RRUWF1FK+UmJiocePG6YsvvpDNZtP+/fv16quvasKECfwjzwLHjh2r9vsB7tWhQwfnGy9dunTR66+/LunElZ7GjRtbmMx7FBcXq7CwsNp4YWGhfv75ZwsSeafQ0FDv+/ePA25hs9kcdrvd5WGz2Rxt27Z1rFu3zup4XuHGG290JCYmOhwOh6NRo0aOHTt2OH7++WfH5Zdf7rj11lstTme+0tJSx2233ebw8fFx+Pj4OLZv3+5wOByOMWPGOFJSUixO5z2qqqoc06ZNczRs2NBhs9kcNpvN4e/v75g8ebLV0bxGSUmJ45577nG0aNGi2u8Fu91udTzjpaamOp577jmHw+FwfPTRRw5/f3+Hn5+fw263O9LS0ixO5x1uueUWR3h4uOOtt95y7Nmzx7Fnzx7Hm2++6Wjfvr0jPj7e6nheY9myZY6hQ4c6du3aZXUUj2F5aTf59NNPXT622+1q0aKFOnXqpHr1mDHoCXv37tWQIUPkcDi0detWRUdHa+vWrWrevLk+++wztWzZ0uqIRhs3bpzWrl2rtLQ0DR06VJs2bVKHDh303nvv6dFHH612oyrcq7y8XNu2bVNJSYkiIiJYddCD7rnnHq1cuVKPP/54jTfDc4+IZ+3evVtZWVnq1KmTunfvbnUcr3DkyBFNmDBBCxYs0PHjxyVJ9erV0+23366nnnqq2mI1cI/CwkLdeOON+uyzzxQQEKD69eu7HDdxYRSKjhscP35cf/vb3/TII4+wh4jFKioqtGTJEm3atEklJSXq1auXRo4cqQYNGlgdzXjt2rXT0qVLddFFFykwMFC5ubnq0KGDtm3bpl69ejFlxyLFxcX65JNP1LlzZ11wwQVWx/EKbdu2dd4MHxQUpOzsbHXq1Ekvv/yy/vWvf7HEt5stXrxYcXFx8vPzcxkvLy/XkiVLFB8fb1Ey71NaWqrt27dLkjp27EjB8bDY2Fjl5eXp9ttvV0hISLXFCEy8t5Ci4yZslghvFxAQoK+//lodOnRwKTq5ubnq37+/ioqKrI7oFW688Ub1799fY8aM0dGjRxUZGamdO3fK4XBoyZIlGj58uNURjdeoUSN9++23atu2rf70pz/p7bffVp8+fbRz505169ZNJSUlVkc0mo+Pj3744YdqV/F//PFHtWzZkn104DUCAgKUmZlZ48rApmIxAjc5uVkirLV161bNmTNH06ZN09SpU10ecK/o6Gh9+OGHzo9PvnM0b948xcTEWBXL63z22WfO5VvfeecdVVVV6fDhw5o1axb76HgIN8Nby+Fw1LiM7t69exUcHGxBIpy0fft2XX755VbH8BpdunTR0aNHrY7hUdws4iZslmi9uXPn6q677lLz5s3VqlUrl190NptNycnJFqYz3/Tp0/XnP/9Z3377rSoqKvTcc8/p22+/1bp166rdwwb3KSoqUtOmTSVJy5cv1/DhwxUQEKCrrrpKDzzwgMXpvENCQoJyc3M1YMAAPfTQQ7r66qv1wgsv6Pjx4+wI70Y9e/aUzWaTzWbToEGDXO6Prays1M6dOzV06FALE6KkpITfBx40Y8YM3X///XriiSfUrVu3avfoBAUFWZTMfZi65ianm7Jms9m0Y8cOD6bxTu3atdPdd9+tiRMnWh3Fa23fvl0zZsxQbm6u8x6piRMnqlu3blZH8xrnn3++pk2bpquuukrt27fXkiVLdPnllys3N1eDBg3SwYMHrY7odbgZ3jMee+wx5//ef//9Lgtw+Pr6Kjw8XMOHD6+21xrqzqxZs057fN++fXr66aeZPughdvuJiVz/e4Xz5FVPE18Hig6MFRQUpJycHDZFhFf7xz/+oXHjxqlRo0Zq166dsrOzZbfb9fzzz+vtt9922b0cMNGiRYsUFxcnf39/q6N4HbvdrtatW5+yTJaXlys/P9/If2Cfi37r6tmAAQM8lMRzKDpukpSUVOO4zWaTv7+/OnXqpGuvvdY5pQR17/bbb1fv3r115513Wh3Fa9RmJTUTL5GfqzZs2KA9e/boiiuucL6r/eGHH6px48ZetUO2lTIyMpSRkaEDBw6oqqrK5diCBQssSgW4V/v27TVz5kzdeOONNR7PyclRVFQURcdD8vLyFBYWVuMVnT179qht27YWJXMfio6bDBw4UNnZ2aqsrFTnzp0lSd9//718fHzUpUsXbdmyRTabTWvWrFFERITFac2UkpKi1NRUXXXVVTXOReU+qbpnt9trvOm3Jvxig7d47LHHNHXqVEVHR6t169bV/ht55513LErmHSorK/Xss8/q9ddfV15ensrLy12Om7h3yLnihhtuUMeOHTVz5swaj+fm5qpnz57Vyj/cwxtXIKTouElaWppWr16tf/7zn853rouKijR69GhdcsklSkxM1IgRI3T06FH997//tTitmbhPyvN+fVl8165deuihh3Trrbc6V1nLzMzUokWLlJKSYuR6/eeiyspKLVy48JRXEz755BOLknmP1q1b68knn9Qtt9xidRSvlJycrHnz5un+++/X5MmT9fDDD2vXrl169913lZyczJtebvTtt9/qyJEjio6OrvH48ePHtX//frVr187DybyT3W5XQUGBWrRo4TK+e/duRUREqLS01KJk7kPRcZPQ0FB99NFH1a7WfPPNNxo8eLD27dun7OxsDR48mJuBYaRBgwZp9OjRuummm1zGX3vtNc2ZM0erVq2yJpiXGTNmjBYuXKirrrqqxqsJzz77rEXJvEezZs20fv16dezY0eooXqljx46aNWuWrrrqKgUGBionJ8c59vnnn+u1116zOiLgVidvp3juueeUmJiogIAA57HKykp98cUX8vHx0dq1a62K6DYsL+0mRUVFOnDgQLWiU1hY6LyPoXHjxtUuoQOmyMzMVHp6erXx6OhojR492oJE3mnJkiV6/fXXdeWVV1odxWuNHj1ar732mh555BGro3il/Px850qPjRo1cm5W/P/+3//jNfGQadOmaeTIkWyibpGNGzdKOnEvzldffeWyOISvr6969OihCRMmWBXPrSg6bnLttdfqtttu0zPPPKPevXtLkr788ktNmDBB1113nSRp/fr1Ov/88y1MaZ6kpCQ9/vjjatiw4SkXhDiJ/SvcKywsTHPnztWTTz7pMj5v3jyFhYVZlMr7+Pr6qlOnTlbH8GrHjh3TnDlz9PHHH6t79+7V7hfkZ5F7/elPf9IPP/ygtm3bqmPHjlqxYoV69eqlL7/8Un5+flbH8wpvvPGGpkyZor59++rmm2/WjTfeqObNm1sdy2ucXF0zISFBzz33nFctBsTUNTcpKSnR+PHjtXjxYlVUVEiS6tWrp1GjRunZZ59Vw4YNlZOTI0mKjIy0LqhhBg4cqHfeeUeNGzfWwIEDT3mezWbj3gQ3W7ZsmYYPH65OnTqpb9++kk6U+61bt+qtt97iCoOHPPPMM9qxY4deeOGFM14oAnWLn0XWeuihhxQUFKS///3vWrp0qW6++WaFh4crLy9P48eP14wZM6yO6BW++eYbvfrqq1qyZIn27t2rK664QiNHjtR1113nMpUKnlNcXKxPPvlEXbp0UZcuXayO4xYUHTcrKSlx3vTeoUMHlw3LANPt3btX//jHP7R582ZJ0gUXXKA777yTKzoeNGzYMK1cuVJNmzbVhRdeWO1qwttvv21RMsAamZmZyszM1Hnnnaerr77a6jheae3atXrttdf0xhtv6NixY7XamgBn78Ybb1T//v01ZswYHT16VD169NCuXbvkcDi0ZMkSDR8+3OqIdY6pa27WqFEjdr6G1/rTn/6k6dOnWx3DqzVu3FjDhg2zOgb+z969eyWd+G8D1oiJiXGuBAlrNGzYUA0aNJCvr69+/vlnq+N4jc8++0wPP/ywpBPL2jscDh0+fFiLFi3StGnTjCw6XNGBUa6//vozPpd3st3v8OHDmj9/vr777jtJ0oUXXqjbbrtNwcHBFicDPKeqqkrTpk3TM888o5KSEklSYGCg7r//fj388MOy2+0WJzTP+++/f8bnXnPNNW5MgpN27typ1157Ta+99pq2bNmiAQMGaMSIEbrhhhv4neAhDRo00Pfff6+wsDDFx8erTZs2mjFjhvLy8hQREeH8+WQSrujAKPywPHds2LBBQ4YMUYMGDdSnTx9JJ266fuKJJ5w3A8MzKioqtGrVKm3fvl0jRoxQYGCg9u/fr6CgIKbTesDDDz+s+fPna8aMGerXr58kac2aNXr00Ud17NgxPfHEExYnNM/JRX9Ostls+t/3dU/es2biJonnmosuukhffvmlunfvroSEBN10000KDQ21OpbXCQsLU2Zmppo2barly5dryZIlkqSffvpJ/v7+FqdzD67oAHCLSy+9VJ06ddLcuXNVr96J91QqKio0evRo7dixQ5999pnFCb3D7t27NXToUOXl5amsrEzff/+9OnTooHHjxqmsrKzGJcBRt9q0aaP09PRqVw7ee+893X333dq3b59FybzDxx9/rIkTJ2r69OkumxdPnjxZ06dP1xVXXGFxQvM9/PDDGjlyZLUtN+BZ//jHPzRu3Dg1atRIbdu21caNG2W32/X888/r7bffdq7OZhKKDozGO9nWadCggTZu3FhtJZdvv/1W0dHROnLkiEXJvMt1112nwMBAzZ8/X82aNVNubq46dOigVatWKTExUVu3brU6ovH8/f21adOmatsJbNmyRZGRkTp69KhFybxD165dlZ6erksuucRlfPXq1brjjjucU2sBb5CVlaW8vDwNHjxYDRs2lCR9+OGHatKkiS6++GKL09U9pq7BWP/7TvYVV1yhwMBAzZw5k3eyPSAoKEh5eXnVis6ePXsUGBhoUSrvs3r1aq1bt85lgzhJCg8P50qCh/To0UMvvPCCZs2a5TL+wgsvqEePHhal8h7bt29X48aNq40HBwdr165dHs/jjSorK7Vw4UJlZGTowIEDqqqqcjnOEuvuc6o9BVevXl1tjKID/IGMGzdO0dHRys3NVbNmzZzjw4YNU2JiooXJvENcXJxuv/12Pf30084fnmvXrtUDDzygm266yeJ03qOqqqrGexD27t1L4fSQJ598UldddZU+/vhjl6lTe/bs0bJlyyxOZ77evXsrKSlJL7/8skJCQiRJBQUFeuCBB5z3D8K9xo0bp4ULF+qqq65S165d2dPLgzZu3HhG55n6mjB1DcZq1qyZ1q1bp86dOyswMNA5ZWfXrl2KiIhg6pSblZeX64EHHlB6erpz09z69evrrrvu0owZM9iR3EPi4uIUHBysOXPmKDAwUJs2bVKLFi107bXXqm3btvrnP/9pdUSvsH//fs2ePdtlT6m7775bbdq0sTiZ+bZt26Zhw4Y5V5uSTlxZPu+88/Tuu++qU6dOFic0X/PmzbV48WI2iobHUXRgrCZNmmjt2rWKiIhwKTpr1qzR8OHDVVBQYHVEr3DkyBFt375dktSxY0d2wPawvXv3asiQIXI4HNq6dauio6O1detWNW/eXJ999platmxpdUTA7RwOhz766COXohkbG2vsu9jnmjZt2mjVqlXV7lMD3I2iA2PxTra1ioqKVFlZqaZNm7qMHzp0SPXq1VNQUJBFybxPRUWFlixZok2bNqmkpES9evXSyJEj1aBBA6ujGWvTpk3q2rWr7Ha7Nm3adNpz2VT63NCtWzctW7bMedUHdeeZZ57Rjh079MILL1Au4VEUHRiLd7Kt9ec//1lXX3217r77bpfx9PR0vf/++9ybAKPZ7Xbl5+erZcuWstvtNe7jIp2YF88+LueGX1/5R90aNmyYVq5cqaZNm+rCCy9U/fr1XY6zgTfchaIDo1VUVGjp0qXKzc3lnWwPa9q0qdauXasLLrjAZXzz5s3q16+ffvzxR4uSmY9d4a23e/dutW3bVjabTbt37z7tue3atfNQKpwORcd9EhISTnucGRZwF4oOALdo2LChPv/8c3Xr1s1l/KuvvlLfvn1ZDMKN7Ha7y8fsCm+tzz77TBdffLFz49yTKioqtG7dOvXv39+iZPg1ig5gHvtvnwL8MS1atEgffvih8+MHH3xQjRs31sUXX/yb77Di9+vTp4/mzJlTbTw9PV1RUVEWJPIeVVVVzseKFSsUGRmp//znPzp8+LAOHz6s//znP+rVq5eWL19udVSvMHDgQB06dKjaeFFRkQYOHGhBIsAahYWFWrNmjdasWaPCwkKr48ALcEUHxurcubNefPFFXX755crMzNSgQYOUlpamDz74QPXq1WNOsJutXbtWsbGx6t27twYNGiRJysjI0JdffqkVK1bo0ksvtTihd2BXeOvZ7XYVFBSoRYsWLuPff/+9oqOjVVxcbFEy/BpXdNyntLRUY8eO1eLFi52bhfr4+Cg+Pl7PP/88q3HCbdgwFMbas2ePc3+Ed999VzfccIPuuOMO9evXT5dddpm14bxAv379lJmZqaeeekqvv/66GjRooO7du2v+/Pk677zzrI7nNdgV3jrXX3+9pBPTBG+99VaXvaMqKyu1adMmI3ciB/5XUlKSPv30U/373/9Wv379JElr1qzRvffeq/vvv18vvviixQlhKooOjNWoUSP9+OOPatu2rVasWKGkpCRJkr+/v44ePWpxOu8QGRmpV1991eoYXo1d4a0THBws6cQeLoGBgS6LoPj6+uqiiy5SYmKiVfG8xuLFixUXF1dtk+Ly8nItWbJE8fHxkqSXXnrJ+d8I6tZbb72lN9980+VNxiuvvFINGjTQjTfeSNGB2zB1DcYaOXKkNm/erJ49e+pf//qX8vLy1KxZM73//vv6+9//rq+//trqiMarqqrStm3bdODAAed0hZO4Adsz2BXeeo899pgeeOABpudYxMfHRz/88EO1LQV+/PFHtWzZkgU5PCAgIEBZWVnVVuH85ptv1KdPH5WWllqUDKaj6MBYhw8f1uTJk7Vnzx7dddddGjp0qCRpypQp8vX11cMPP2xxQrN9/vnnGjFihHbv3l3jil/848Jz2BXeWjt37lRFRUW1KZtbt25V/fr1FR4ebk0wL3Gqe6Ryc3NPuVAE6tagQYPUrFkzLV68WP7+/pKko0ePatSoUTp06JA+/vhjixPCVBQdAG4RGRmp888/X4899phat25d7R/VJ6f14NzArvDuM2DAAN12220aNWqUy/grr7yiefPmadWqVdYEM1zPnj1ls9mUm5urCy+80GV578rKSu3cuVNDhw7V66+/bmFK7/DVV19p6NChKisrU48ePSSdKJp+fn5asWKFLrzwQosTwlQUHRjvyJEjysvLU3l5uct49+7dLUrkHRo2bKjc3FymRv1BsOKU+wQFBSk7O7vafwvbtm1TdHS0Dh8+bE0wwz322GPO/73//vvVqFEj5zFfX1+Fh4dr+PDh8vX1tSqiVzly5IheffVVlyvLbOANd2MxAhirsLBQt9566yn3CmHqlHv17dtX27Zto+jA69lsNv3888/VxouKivg55EZTpkyRJIWHhysuLs45ZQqel5KSopCQkGqLbyxYsECFhYWaOHGiRclgOjYMhbHuu+8+FRUV6YsvvlCDBg20fPlyLVq0SOedd57ef/99q+MZb+zYsbr//vu1cOFCZWVladOmTS4PwFv0799fKSkpLqWmsrJSKSkp1fY3Qt0bNWqU/P39VV5err179yovL8/lAfd76aWX1KVLl2rjF154odLT0y1IBG/B1DUYq3Xr1nrvvffUp08fBQUFacOGDTr//PP1/vvv68knn9SaNWusjmg0u736+yg2m00Oh4PFCM5BTF1zn2+//Vb9+/dX48aNnRvlrl69WsXFxfrkk0/UtWtXixOabevWrbrtttu0bt06l3F+FnmOv7+/vvvuO7Vv395lfMeOHYqIiNCxY8csSgbTMXUNxiotLXUuJ9qkSRMVFhbq/PPPV7du3ZSdnW1xOvPt3LnT6gjAOSEiIkKbNm3SCy+8oNzcXDVo0EDx8fEaM2aMmjZtanU84916662qV6+ePvjggxoXRoH7hYWFae3atdWKztq1a9WmTRuLUsEbUHRgrM6dO2vLli0KDw9Xjx499NJLLyk8PFzp6elq3bq11fGM165dO6sjAOeMNm3aaPr06VbH8Eo5OTnKysqqceoUPCMxMVH33Xefjh8/rssvv1ySlJGRoQcffFD333+/xelgMooOjDVu3Dj98MMPkk7clDp06FC98sor8vX11aJFiyxO5x1efvllpaena+fOncrMzFS7du2Ulpam9u3b69prr7U6ntc5duzYKW/IZld491q9erVeeukl7dixQ2+88YZCQ0P18ssvq3379tyn42YRERE6ePCg1TG82gMPPKAff/xRd999t3MFVH9/f02cOFGTJk2yOB1MxmIEMNbNN9+sW2+9VZLUq1cv7d69Wxs2bNDevXsVFxdnbTgv8OKLLyopKUlXXnmlDh8+7JwH37hxY6WlpVkbzotUVVXp8ccfV2hoqBo1aqQdO3ZIkh555BHNnz/fed6IESPUsGFDq2Ia7a233tKQIUPUoEEDZWdnq6ysTNKJVde4yuN+M2fO1IMPPqhVq1bpxx9/VHFxscsD7mez2TRz5kwVFhbq888/V25urg4dOqTk5GSro8FwFB0Ybf78+eratav8/f3VpEkTxcfH691337U6lld4/vnnNXfuXD388MPy8fFxjkdHR+urr76yMJl3mTZtmhYuXKgnn3zSZb+Qrl27at68eRYm8x7Tpk1Tenq65s6dq/r16zvH+/Xrx/2CHhAbG6vPP/9cgwYNUsuWLdWkSRM1adJEjRs3VpMmTayO51UaNWqk3r17q2vXrvLz87M6DrwAU9dgrOTkZKWmpmrs2LGKiYmRJGVmZmr8+PHKy8vT1KlTLU5otp07d6pnz57Vxv38/FRaWmpBIu+0ePFizZkzR4MGDdKdd97pHO/Ro4dz4z6415YtW9S/f/9q48HBwWwW6gErV660OgIAi1B0YKwXX3xRc+fO1U033eQcu+aaa9S9e3eNHTuWouNm7du3V05OTrVFCZYvX64LLrjAolTeZ9++fTVu2lpVVaXjx49bkMj7tGrVStu2bVN4eLjL+Jo1a1jO2wMGDBhgdQQAFmHqGox1/PhxRUdHVxuPiopSRUWFBYm8S1JSku655x4tXbpUDodD69ev1xNPPKFJkybpwQcftDqe14iIiNDq1aurjb/55ps1XnFD3UtMTNS4ceP0xRdfyGazaf/+/Xr11Vc1YcIE3XXXXVbH8wqrV6/WzTffrIsvvlj79u2TdGKxFPZTA8zGFR0Y65ZbbtGLL76o1NRUl/E5c+Zo5MiRFqXyHqNHj1aDBg00efJkHTlyRCNGjFCbNm303HPP6a9//avV8bxGcnKyRo0apX379qmqqkpvv/22tmzZosWLF+uDDz6wOp5XeOihh1RVVaVBgwbpyJEj6t+/v/z8/DRhwgSNHTvW6njGe+utt3TLLbdo5MiRNS4GsWzZMosTAnAXm8PhcFgdAqgrSUlJzj9XVFRo4cKFatu2rS666CJJ0hdffKG8vDzFx8fr+eeftyqm1zly5IhKSkqcG7j+2tq1axUdHc2NqW60evVqTZ06Vbm5uSopKVGvXr2UnJyswYMHWx3NeJWVlVq7dq26d++ugIAAbdu2TSUlJYqIiFCjRo2sjucVevbsqfHjxys+Pl6BgYHKzc1Vhw4dtHHjRv35z39Wfn6+1REBuAlFB0YZOHDgGZ1ns9n0ySefuDkNzkRQUJBycnK4VwHG8vf313fffVdtV3h4RkBAgL799luFh4e7FJ0dO3YoIiJCx44dszoiADdh6hqMwuo6fzy81wLTde3aVTt27KDoWITFIADvRdEBAMM0adJENpvtjM49dOiQm9Ng2rRpmjBhgh5//HFFRUVV25g1KCjIomTe4eRiEAsWLHAuBpGZmakJEybokUcesToeADei6ACAYdLS0qyOgF+58sorJZ1Y3v7XBdThcMhms6mystKqaF6BxSAA78U9OgAs9es584CJPv3009MeZ58XzygvL2cxCMDLUHQAWIrFCNyvsrJS77zzjr777jtJJ/bWufbaa1WvHhf1Yb6ioiJVVlaqadOmLuOHDh1SvXr1mDoIGIzfcgAsxXst7vXNN9/ommuuUX5+vjp37ixJmjlzplq0aKF///vf6tq1q8UJzbRp0yZ17dpVdrtdmzZtOu253bt391Aq7/TXv/5VV199te6++26X8ddff13vv/8+++gABuOKDgC3qaio0KpVq7R9+3aNGDFCgYGB2r9/v4KCgpg24iExMTFq0aKFFi1apCZNmkiSfvrpJ916660qLCzUunXrLE5oJrvdrvz8fLVs2VJ2u102m63GUs89Ou7XtGlTrV27VhdccIHL+ObNm9WvXz/9+OOPFiUD4G5c0QHgFrt379bQoUOVl5ensrIyXXHFFQoMDNTMmTNVVlam9PR0qyN6hZycHG3YsMFZcqQTq7I98cQT6t27t4XJzLZz5061aNHC+WdYp6ysTBUVFdXGjx8/rqNHj1qQCICn2K0OAMBM48aNU3R0tH766Sc1aNDAOT5s2DBlZGRYmMy7nH/++SooKKg2fuDAAXXq1MmCRN6hXbt2zhXW2rVrd9oH3KtPnz6aM2dOtfH09HRFRUVZkAiAp3BFB4BbrF69WuvWrZOvr6/LeHh4uPbt22dRKu+TkpKie++9V48++qguuugiSdLnn3+uqVOnaubMmSouLnaey03Zdef9998/43OvueYaNybBtGnTFBsbq9zcXA0aNEiSlJGRoS+//FIrVqywOB0Ad+IeHQBu0aRJE61du1YREREuS0ivWbNGw4cPr/EqA+qe3f7LhfuTVxhO/tj/9cfcK1K3fv33LqnaPTq/3k+Hv3f3y83N1ZNPPqmcnBw1aNBA3bt316RJk3TeeedZHQ2AG3FFB4BbDB48WGlpac4pIzabTSUlJZoyZYpzA0W438qVK62O4JWqqqqcf/744481ceJETZ8+XTExMZKkzMxMTZ48WdOnT7cqolc4fvy4/va3v+mRRx7Rq6++anUcAB7GFR0AbrF3714NGTJEDodDW7duVXR0tLZu3armzZvrs88+U8uWLa2OCHhE165dlZ6erksuucRlfPXq1brjjjuc+xvBPYKDg5WTk6P27dtbHQWAh1F0ALhNRUWFli5dqtzcXJWUlKhXr14aOXKky+IEcL9jx45p06ZNOnDggMuVBon7QzyhQYMG+vLLL6vtWbRp0yb17duXlb/cbNSoUYqMjNT48eOtjgLAwyg6AGCw5cuXKz4+XgcPHqx2jPtyPKN///7y9/fXyy+/rJCQEElSQUGB4uPjdezYMX366acWJzTbtGnT9Mwzz2jQoEGKiopSw4YNXY7fe++9FiUD4G4UHQBukZKSopCQEN12220u4wsWLFBhYaEmTpxoUTLvct5552nw4MFKTk52/iMbnrVt2zYNGzZM33//vcLCwiRJe/bs0Xnnnad3332XZb7d7HRT1mw2m3bs2OHBNAA8iaIDwC3Cw8P12muv6eKLL3YZ/+KLL/TXv/6VTRQ9JCgoSBs3blTHjh2tjuLVHA6HPvroI23evFmSdMEFFyg2NtZl9TUAQN1i1TUAbpGfn6/WrVtXG2/RooV++OEHCxJ5pxtuuEGrVq2i6FjMZrNp8ODBGjx4sNVRvFZ5ebl27typjh07ql49/vkDeAP+SwfgFmFhYVq7dm21aSNr165VmzZtLErlfV544QX95S9/0erVq9WtWzfVr1/f5Tj3J7jHrFmzdMcdd8jf31+zZs067bm8Bu515MgRjR07VosWLZIkff/99+rQoYPGjh2r0NBQPfTQQxYnBOAuTF0D4BZPPvmknnzyST311FO6/PLLJZ3YjfzBBx/U/fffr0mTJlmc0DvMnz9fd955p/z9/dWsWTOXqVLcn+A+7du314YNG9SsWTPuEbHYuHHjtHbtWqWlpWno0KHatGmTOnTooPfee0+PPvqoNm7caHVEAG5C0QHgFg6HQw899JBmzZql8vJySZK/v78mTpyo5ORki9N5j1atWunee+/VQw89JLvdbnUcr3fyVy735nhOu3bttHTpUl100UUKDAxUbm6uOnTooG3btqlXr14qLi62OiIAN+G3HgC3sNlsmjlzpgoLC/X5558rNzdXhw4douR4WHl5ueLi4ig5Fps/f766du0qf39/+fv7q2vXrpo3b57VsbxCYWFhjRsUl5aWUjgBw/GbD4BbNWrUSL1791bXrl3l5+dndRyvM2rUKC1dutTqGF4tOTlZ48aN09VXX6033nhDb7zxhq6++mqNHz+e4u8B0dHR+vDDD50fnyw38+bNU0xMjFWxAHgAU9cAuEVpaalmzJihjIwMHThwQFVVVS7HuS/BM+69914tXrxYPXr0UPfu3astRpCammpRMu/RokULzZo1SzfddJPL+L/+9S+NHTu2xs1cUXfWrFmjP//5z7r55pu1cOFC/e1vf9O3336rdevW6dNPP1VUVJTVEQG4CauuAXCL0aNH69NPP9Utt9yi1q1bM0XEIl999ZV69uwpSfr6669djvGaeMbx48cVHR1dbTwqKkoVFRUWJPIul1xyiXJycjRjxgx169ZNK1asUK9evZSZmalu3bpZHQ+AG3FFB4BbNG7cWB9++KH69etndRTAUmPHjlX9+vWrXT2bMGGCjh49qtmzZ1uUDADMxhUdAG7RpEkTNW3a1OoYgCWSkpKcf7bZbJo3b55WrFihiy66SJL0xRdfKC8vT/Hx8VZF9CqVlZV655139N1330mSIiIidO2117JxKGA4rugAcItXXnlF7733nhYtWqSAgACr43iV66+/XgsXLlRQUJCuv/7605779ttveyiVdxk4cOAZnWez2fTJJ5+4OY13++abb3TNNdcoPz9fnTt3lnRi09AWLVro3//+t7p27WpxQgDuwlsZANzimWee0fbt2xUSEqLw8PBqN8FnZ2dblMx8wcHBzvtvgoODLU7jnVauXGl1BPyf0aNH68ILL9SGDRvUpEkTSdJPP/2kW2+9VXfccYfWrVtncUIA7sIVHQBu8dhjj532+JQpUzyUxLsdPXpUVVVVatiwoSRp165devfdd3XBBRdoyJAhFqcD3K9BgwbasGGDLrzwQpfxr7/+Wr1799bRo0ctSgbA3biiA8AtKDLnhmuvvVbXX3+97rzzTh0+fFgXXXSR6tevr4MHDyo1NVV33XWX1REBtzr//PNVUFBQregcOHBAnTp1sigVAE9gw1AAbnP48GHNmzdPkyZN0qFDhySdmLK2b98+i5N5j+zsbF166aWSpDfffFMhISHavXu3Fi9erFmzZlmcDnC/lJQU3XvvvXrzzTe1d+9e7d27V2+++abuu+8+zZw5U8XFxc4HALMwdQ2AW2zatEmxsbEKDg7Wrl27tGXLFnXo0EGTJ09WXl6eFi9ebHVErxAQEKDNmzerbdu2uvHGG3XhhRdqypQp2rNnjzp37qwjR45YHRFwK7v9l/d0T967dvKfPr/+2GazqbKy0vMBAbgNU9cAuEVSUpJuvfVWPfnkkwoMDHSOX3nllRoxYoSFybxLp06d9O6772rYsGH673//q/Hjx0s6MW0nKCjI4nSA+7EwBOC9KDoA3OLLL7/USy+9VG08NDRU+fn5FiTyTsnJyRoxYoTGjx+vQYMGKSYmRpK0YsUK9ezZ0+J0gPsNGDDgjM67++67deGFF6p58+ZuTgTAU7hHB4Bb+Pn51Tjn/eT+FfCMG264QXl5edqwYYOWL1/uHB80aJCeffZZC5MB55ZXXnmF+3QAw1B0ALjFNddco6lTp+r48eOSTsyFz8vL08SJEzV8+HCL03mXVq1aqWfPni73KvTp00ddunSxMBVwbuGWZcA8FB0AbvHMM8+opKRELVu21NGjRzVgwAB16tRJgYGBeuKJJ6yOBwAADMc9OgDcIjg4WB999JHWrl2r3NxclZSUqFevXoqNjbU6GgAA8AIsLw3ALRYvXqy4uDj5+fm5jJeXl2vJkiWKj4+3KBkAVBcYGKjc3Fx16NDB6igA6ghFB4Bb+Pj46IcfflDLli1dxn/88Ue1bNmS/SoAnFMoOoB5uEcHgFuc3IDvf+3du1fBwcEWJAKAU7v55pvZWwowDPfoAKhTPXv2lM1mk81m06BBg1Sv3i8/ZiorK7Vz504NHTrUwoQAvMn69euVmZnp3L+rVatWiomJUZ8+fVzOe/HFF62IB8CNKDoA6tR1110nScrJydGQIUPUqFEj5zFfX1+Fh4ezvDQAtztw4ICGDx+utWvXqm3btgoJCZEkFRQUaPz48erXr5/eeuutatNrAZiDe3QAuMWiRYsUFxcnf39/q6MA8EI33HCD9u/fr3/+85/q3Lmzy7EtW7botttuU5s2bfTGG29YlBCAu1F0ALhVeXm5Dhw4oKqqKpfxtm3bWpQIgDcIDAzUZ599pp49e9Z4PCsrS5dddpl+/vlnDycD4ClMXQPgFlu3btVtt92mdevWuYyfXKSAVdcAuJOfn5+Ki4tPefznn3+utvw9ALNQdAC4xa233qp69erpgw8+UOvWrWtcgQ0A3CUuLk6jRo3Ss88+q0GDBjlXVCsuLlZGRoaSkpJ00003WZwSgDsxdQ2AWzRs2FBZWVnq0qWL1VEAeKGysjLdd999WrBggSoqKuTr6yvpxHTaevXq6fbbb9ezzz7LVR3AYBQdAG7Ru3dvPfvss7rkkkusjgLAixUXFysrK8tleemoqCj2zAG8AEUHgFt88sknmjx5sqZPn65u3bqpfv36Lsf5RwYAAHAnig4At7Db7ZJU7d4cFiMAcC4oKCjQSy+9pOTkZKujAHATig4At/j0009Pe3zAgAEeSgIA1eXm5qpXr1686QIYjFXXALgFRQaAlTZt2nTa41u2bPFQEgBW4YoOALdZvXq1XnrpJe3YsUNvvPGGQkND9fLLL6t9+/YsUgDArex2u2w2m2r6Z87JcabRAmazWx0AgJneeustDRkyRA0aNFB2drbKysokSUVFRZo+fbrF6QCYrmnTppo7d6527txZ7bFjxw598MEHVkcE4GZMXQPgFtOmTVN6erri4+O1ZMkS53i/fv00bdo0C5MB8AZRUVHav3+/2rVrV+Pxw4cP13i1B4A5KDoA3GLLli3q379/tfHg4GAdPnzY84EAeJU777xTpaWlpzzetm1b/fOf//RgIgCeRtEB4BatWrXStm3bFB4e7jK+Zs0adejQwZpQALzGsGHDTnu8SZMmGjVqlIfSALAC9+gAcIvExESNGzdOX3zxhWw2m/bv369XX31VEyZM0F133WV1PABwERQUpB07dlgdA0Ad4ooOALd46KGHVFVVpUGDBunIkSPq37+//Pz8NGHCBI0dO9bqeADggvt1APOwvDQAtyovL9e2bdtUUlKiiIgINWrUyOpIAFBNYGCgcnNzmVoLGISpawDcytfXVxEREerSpYs+/vhjfffdd1ZHAgAAXoCiA8AtbrzxRr3wwguSpKNHj6p379668cYb1b17d7311lsWpwMAAKaj6ABwi88++0yXXnqpJOmdd95RVVWVDh8+rFmzZrGPDoBzjs1mszoCgDpG0QHgFkVFRWratKkkafny5Ro+fLgCAgJ01VVXaevWrRanAwBX3LIMmIeiA8AtwsLClJmZqdLSUi1fvlyDBw+WJP3000/y9/e3OB0Ab+RwOE5ZaP7zn/8oNDTUw4kAuBNFB4Bb3HfffRo5cqT+9Kc/qU2bNrrssssknZjS1q1bN2vDAfAq8+fPV9euXeXv7y9/f3917dpV8+bNcznnkksukZ+fn0UJAbgDy0sDcJusrCzl5eXpiiuucC4r/eGHH6px48bq16+fxekAeIPk5GSlpqZq7NixiomJkSRlZmbqhRde0Pjx4zV16lSLEwJwF4oOAEsFBQUpJyeHvSsAuEWLFi00a9Ys3XTTTS7j//rXvzR27FgdPHjQomQA3I2pawAsxXstANzp+PHjio6OrjYeFRWliooKCxIB8BSKDgAAMNYtt9yiF198sdr4nDlzNHLkSAsSAfCUelYHAAAAqEtJSUnOP9tsNs2bN08rVqzQRRddJEn64osvlJeXp/j4eKsiAvAAig4AADDKxo0bXT6OioqSJG3fvl2S1Lx5czVv3lzffPONx7MB8ByKDgBLsRs5gLq2cuVKqyMAOAdwjw4AS7EYAQAAcAeu6ABwu5NlpqarN+xGDsCdBg4ceNorx5988okH0wDwJK7oAHAbdiMHYLXIyEj16NHD+YiIiFB5ebmys7PVrVs3q+MBcCOu6ABwi1PtRj5+/Hjl5eWxGzkAj3j22WdrHH/00UdVUlLi4TQAPMnmYII8ADdgN3IA57Jt27apT58+OnTokNVRALgJU9cAuAW7kQM4l2VmZsrf39/qGADciKlrANzi5G7kqampLuPsRg7Ak66//nqXjx0Oh3744Qdt2LBBjzzyiEWpAHgCRQdAnWE3cgDnmuDgYJeP7Xa7OnfurKlTp2rw4MEWpQLgCdyjA6DODBw48IzOs9lsLOkKAADciqIDAACMV15ergMHDqiqqsplvG3bthYlAuBuTF0DAADG+v7773X77bdr3bp1LuMOh0M2m02VlZUWJQPgbhQdAG7BbuQAzgUJCQmqV6+ePvjgA7Vu3fq0P5cAmIWiA8AtIiMjXT4+fvy4cnJy9PXXX2vUqFHWhALgdXJycpSVlaUuXbpYHQWAh1F0ALgFu5EDOBdERESwQTHgpViMAIBHsRs5AHcrLi52/nnDhg2aPHmypk+frm7duql+/fou5wYFBXk6HgAP4YoOAI9iN3IA7ta4cWOXe3EcDocGDRrkcg6LEQDmo+gAcAt2IwdglZUrV1odAcA5gKlrANwiISHB5WO73a4WLVro8ssvZzdyAOecu+++W1OnTlXz5s2tjgKgjlB0AACA1wsKClJOTo46dOhgdRQAdYSpawDcit3IAfwR8L4vYB6KDgC3YDdyAABgJYoOALdgN3IAAGAlig4At2A3cgAAYCW71QEAmIndyAEAgJUoOgDqTHFxsfMxc+ZMPfjgg1q1apV+/PFHl2O/3rUcAOra9ddf7/w5s3jxYpWVlf3mc26++WYFBQW5OxoAD2J5aQB1xm63V9uN/H/vzWExAgDu5uvrq927d6t169by8fHRDz/8oJYtW1odC4CHcY8OgDrDbuQAzgVdunTRpEmTNHDgQDkcDr3++uunvFoTHx/v4XQAPIUrOgAsxW7kAOraunXrlJSUpO3bt+vQoUMKDAysceVHm82mQ4cOWZAQgCdQdABYit3IAbiT3W7Xvn371Lp1a5dxh8OhvLw8tWvXzqJkANyNxQgAWIr3WgBY4dChQ7zBAhiOogMAAIzm4+NTbaykpET+/v4WpAHgKSxGAAAAjJOUlCTpxH04ycnJCggIcB6rrKzUF198ocjISIvSAfAEig4AADDOxo0bJZ2YHvvVV1/J19fXeczX11c9evTQhAkTrIoHwAMoOgAAwDgnl7tPSEjQc889x2aggBfiHh0AdYbdyAGca/75z3/yMwbwUiwvDaDOsBs5AAA4VzB1DUCdYTdyAABwruCKDoA6w27kAADgXEHRAeAW7EYOAACsxGIEADyK3cgBAIAnUHQAuA27kQMAAKuwGAGAOsVu5AAA4FxA0QFQp9iNHAAAnAtYjACAW7AbOQAAsBJFBwAAAIBxWIwAAAAAgHEoOgAAAACMQ9EBAAAAYByKDgAAAADjUHQAAAAAGIeiAwAAAMA4FB0AAAAAxqHoAAAAADDO/wcjFf4ssHOuYQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 1000x700 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Sort model results by f1-score\n",
    "all_model_results.sort_values(\"f1\", ascending=False)[\"f1\"].plot(kind=\"bar\", figsize=(10, 7))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find the most wrong examples\n",
    "\n",
    "* If our best model still isnt perfect, then what examples is it getting wrong?\n",
    "* And of these wrong examples which ones is it getting most wrong"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "K0ZIzs402ZBG"
   },
   "outputs": [],
   "source": [
    "# Create DataFrame with validation sentences and best performing model predictions\n",
    "if \"08_model_6_USE_feature_extractor.zip\" not in os.listdir(\".\"):\n",
    "    ! wget https://storage.googleapis.com/ztm_tf_course/08_model_6_USE_feature_extractor.zip\n",
    "    unzip_data(\"08_model_6_USE_feature_extractor.zip\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# IMport previously trained model\n",
    "# model_6_pretrained = tf.keras.models.load_model(\"08_model_6_USE_feature_extractor\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 1s 30ms/step - loss: 0.4952 - accuracy: 0.7861\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.49517935514450073, 0.7860892415046692]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_6.evaluate(val_sentences,val_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 1s 20ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 78.60892388451444,\n",
       " 'precision': 0.7866474744348231,\n",
       " 'recall': 0.7860892388451444,\n",
       " 'f1': 0.78486500301026}"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_6_pretrained_pred_probs = model_6.predict(val_sentences)\n",
    "model_6_pretrained_preds = tf.squeeze(tf.round(model_6_pretrained_pred_probs))\n",
    "model_6_pretrained_results = calculate_results(val_labels,model_6_pretrained_preds)\n",
    "model_6_pretrained_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test</th>\n",
       "      <th>target</th>\n",
       "      <th>prediction</th>\n",
       "      <th>pred_probs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DFR EP016 Monthly Meltdown - On Dnbheaven 2015...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.349365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FedEx no longer to transport bioterror germs i...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.669922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Gunmen kill four in El Salvador bus attack: Su...</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.854980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>@camilacabello97 Internally and externally scr...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.340332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Radiation emergency #preparedness starts with ...</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.648926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>757</th>\n",
       "      <td>That's the ultimate road to destruction</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.274658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>758</th>\n",
       "      <td>@SetZorah dad why dont you claim me that mean ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.201904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>759</th>\n",
       "      <td>FedEx will no longer transport bioterror patho...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.702148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>760</th>\n",
       "      <td>Crack in the path where I wiped out this morni...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.545898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>761</th>\n",
       "      <td>I liked a @YouTube video from @dannyonpc http:...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.375732</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>762 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  test  target  prediction  \\\n",
       "0    DFR EP016 Monthly Meltdown - On Dnbheaven 2015...       0         0.0   \n",
       "1    FedEx no longer to transport bioterror germs i...       0         1.0   \n",
       "2    Gunmen kill four in El Salvador bus attack: Su...       1         1.0   \n",
       "3    @camilacabello97 Internally and externally scr...       1         0.0   \n",
       "4    Radiation emergency #preparedness starts with ...       1         1.0   \n",
       "..                                                 ...     ...         ...   \n",
       "757            That's the ultimate road to destruction       0         0.0   \n",
       "758  @SetZorah dad why dont you claim me that mean ...       0         0.0   \n",
       "759  FedEx will no longer transport bioterror patho...       0         1.0   \n",
       "760  Crack in the path where I wiped out this morni...       0         1.0   \n",
       "761  I liked a @YouTube video from @dannyonpc http:...       0         0.0   \n",
       "\n",
       "     pred_probs  \n",
       "0      0.349365  \n",
       "1      0.669922  \n",
       "2      0.854980  \n",
       "3      0.340332  \n",
       "4      0.648926  \n",
       "..          ...  \n",
       "757    0.274658  \n",
       "758    0.201904  \n",
       "759    0.702148  \n",
       "760    0.545898  \n",
       "761    0.375732  \n",
       "\n",
       "[762 rows x 4 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df = pd.DataFrame({\"test\": val_sentences,\n",
    "                        \"target\":val_labels,\n",
    "                        \"prediction\":model_6_pretrained_preds,\n",
    "                        \"pred_probs\":tf.squeeze(model_6_pretrained_pred_probs)})\n",
    "val_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find the wrong predictions and sort by prediction probability\n",
    "most_wrong = val_df[val_df[\"target\"]!=val_df[\"prediction\"]].sort_values(by=\"pred_probs\",ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test</th>\n",
       "      <th>target</th>\n",
       "      <th>prediction</th>\n",
       "      <th>pred_probs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>@madonnamking RSPCA site multiple 7 story high...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.803711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>? High Skies - Burning Buildings ? http://t.co...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.760742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>393</th>\n",
       "      <td>@SonofLiberty357 all illuminated by the bright...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.757812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>Article by Michael Jackman at Metro Times Detr...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.707031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>759</th>\n",
       "      <td>FedEx will no longer transport bioterror patho...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.702148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>GENERAL AUDIENCE: On Wounded Families | ZENIT ...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.686523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>209</th>\n",
       "      <td>Ashes 2015: AustraliaÛªs collapse at Trent Br...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.673828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>[55436] 1950 LIONEL TRAINS SMOKE LOCOMOTIVES W...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.671875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FedEx no longer to transport bioterror germs i...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.669922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>630</th>\n",
       "      <td>Hollywood movie about trapped miners released ...</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.665039</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  test  target  prediction  \\\n",
       "49   @madonnamking RSPCA site multiple 7 story high...       0         1.0   \n",
       "31   ? High Skies - Burning Buildings ? http://t.co...       0         1.0   \n",
       "393  @SonofLiberty357 all illuminated by the bright...       0         1.0   \n",
       "181  Article by Michael Jackman at Metro Times Detr...       0         1.0   \n",
       "759  FedEx will no longer transport bioterror patho...       0         1.0   \n",
       "80   GENERAL AUDIENCE: On Wounded Families | ZENIT ...       0         1.0   \n",
       "209  Ashes 2015: AustraliaÛªs collapse at Trent Br...       0         1.0   \n",
       "109  [55436] 1950 LIONEL TRAINS SMOKE LOCOMOTIVES W...       0         1.0   \n",
       "1    FedEx no longer to transport bioterror germs i...       0         1.0   \n",
       "630  Hollywood movie about trapped miners released ...       0         1.0   \n",
       "\n",
       "     pred_probs  \n",
       "49     0.803711  \n",
       "31     0.760742  \n",
       "393    0.757812  \n",
       "181    0.707031  \n",
       "759    0.702148  \n",
       "80     0.686523  \n",
       "209    0.673828  \n",
       "109    0.671875  \n",
       "1      0.669922  \n",
       "630    0.665039  "
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "most_wrong.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test</th>\n",
       "      <th>target</th>\n",
       "      <th>prediction</th>\n",
       "      <th>pred_probs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>408</th>\n",
       "      <td>@willienelson We need help! Horses will die!Pl...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.182495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>382</th>\n",
       "      <td>I wanna set some shit on fire.</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.176025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>536</th>\n",
       "      <td>@DavidVonderhaar At least you were sincere ??</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.169434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>722</th>\n",
       "      <td>Indeed!! I am fully aware of that battle! I su...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.166138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Ron &amp;amp; Fez - Dave's High School Crush https...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.152710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352</th>\n",
       "      <td>@reriellechan HE WAS THE LICH KING'S FIRST CAS...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.151733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244</th>\n",
       "      <td>Reddit Will Now QuarantineÛ_ http://t.co/pkUA...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.150269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>681</th>\n",
       "      <td>'The way you move is like a full on rainstorm ...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.148804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>411</th>\n",
       "      <td>@SoonerMagic_ I mean I'm a fan but I don't nee...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.117004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>233</th>\n",
       "      <td>I get to smoke my shit in peace</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.115112</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  test  target  prediction  \\\n",
       "408  @willienelson We need help! Horses will die!Pl...       1         0.0   \n",
       "382                     I wanna set some shit on fire.       1         0.0   \n",
       "536      @DavidVonderhaar At least you were sincere ??       1         0.0   \n",
       "722  Indeed!! I am fully aware of that battle! I su...       1         0.0   \n",
       "23   Ron &amp; Fez - Dave's High School Crush https...       1         0.0   \n",
       "352  @reriellechan HE WAS THE LICH KING'S FIRST CAS...       1         0.0   \n",
       "244  Reddit Will Now QuarantineÛ_ http://t.co/pkUA...       1         0.0   \n",
       "681  'The way you move is like a full on rainstorm ...       1         0.0   \n",
       "411  @SoonerMagic_ I mean I'm a fan but I don't nee...       1         0.0   \n",
       "233                    I get to smoke my shit in peace       1         0.0   \n",
       "\n",
       "     pred_probs  \n",
       "408    0.182495  \n",
       "382    0.176025  \n",
       "536    0.169434  \n",
       "722    0.166138  \n",
       "23     0.152710  \n",
       "352    0.151733  \n",
       "244    0.150269  \n",
       "681    0.148804  \n",
       "411    0.117004  \n",
       "233    0.115112  "
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "most_wrong.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target: 0, Pred: 1.0, Prob: 0.8037109375\n",
      "Text :\n",
      " @madonnamking RSPCA site multiple 7 story high rise buildings next to low density character residential in an area that floods\n",
      "\n",
      "-------------------\n",
      "\n",
      "Target: 0, Pred: 1.0, Prob: 0.7607421875\n",
      "Text :\n",
      " ? High Skies - Burning Buildings ? http://t.co/uVq41i3Kx2 #nowplaying\n",
      "\n",
      "-------------------\n",
      "\n",
      "Target: 0, Pred: 1.0, Prob: 0.7578125\n",
      "Text :\n",
      " @SonofLiberty357 all illuminated by the brightly burning buildings all around the town!\n",
      "\n",
      "-------------------\n",
      "\n",
      "Target: 0, Pred: 1.0, Prob: 0.70703125\n",
      "Text :\n",
      " Article by Michael Jackman at Metro Times Detroit:\n",
      "The group later downgraded the estimate to 37 square miles of... http://t.co/h31mmuduqt\n",
      "\n",
      "-------------------\n",
      "\n",
      "Target: 0, Pred: 1.0, Prob: 0.7021484375\n",
      "Text :\n",
      " FedEx will no longer transport bioterror pathogens in wake of anthrax lab mishaps http://t.co/lHpgxc4b8J\n",
      "\n",
      "-------------------\n"
     ]
    }
   ],
   "source": [
    "# Check the false poitives (predicted true but actually it is false)\n",
    "for row in most_wrong[:5].itertuples():\n",
    "    _, text,target,pred,pred_prob = row\n",
    "    print(f\"Target: {target}, Pred: {pred}, Prob: {pred_prob}\")\n",
    "    print(f\"Text :\\n {text}\\n\")\n",
    "    print(\"-------------------\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Target: 1, Pred: 0.0, Prob: 0.1517333984375\n",
      "Text :\n",
      " @reriellechan HE WAS THE LICH KING'S FIRST CASUALTY BLOCK ME BACK I HATE YOU! http://t.co/0Gidg9U45J\n",
      "\n",
      "-------------------\n",
      "\n",
      "Target: 1, Pred: 0.0, Prob: 0.1502685546875\n",
      "Text :\n",
      " Reddit Will Now QuarantineÛ_ http://t.co/pkUAMXw6pm #onlinecommunities #reddit #amageddon #freespeech #Business http://t.co/PAWvNJ4sAP\n",
      "\n",
      "-------------------\n",
      "\n",
      "Target: 1, Pred: 0.0, Prob: 0.1488037109375\n",
      "Text :\n",
      " 'The way you move is like a full on rainstorm and I'm a house of cards'\n",
      "\n",
      "-------------------\n",
      "\n",
      "Target: 1, Pred: 0.0, Prob: 0.11700439453125\n",
      "Text :\n",
      " @SoonerMagic_ I mean I'm a fan but I don't need a girl sounding off like a damn siren\n",
      "\n",
      "-------------------\n",
      "\n",
      "Target: 1, Pred: 0.0, Prob: 0.1151123046875\n",
      "Text :\n",
      " I get to smoke my shit in peace\n",
      "\n",
      "-------------------\n"
     ]
    }
   ],
   "source": [
    "# Check the false negatives (predicted false but actually it is true)\n",
    "for row in most_wrong[-5:].itertuples():\n",
    "    _, text,target,pred,pred_prob = row\n",
    "    print(f\"Target: {target}, Pred: {pred}, Prob: {pred_prob}\")\n",
    "    print(f\"Text :\\n {text}\\n\")\n",
    "    print(\"-------------------\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>keyword</th>\n",
       "      <th>location</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Just happened a terrible car crash</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Heard about #earthquake is different cities, s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>there is a forest fire at spot pond, geese are...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Apocalypse lighting. #Spokane #wildfires</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Typhoon Soudelor kills 28 in China and Taiwan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3258</th>\n",
       "      <td>10861</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>EARTHQUAKE SAFETY LOS ANGELES ÛÒ SAFETY FASTE...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3259</th>\n",
       "      <td>10865</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Storm in RI worse than last hurricane. My city...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3260</th>\n",
       "      <td>10868</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Green Line derailment in Chicago http://t.co/U...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3261</th>\n",
       "      <td>10874</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MEG issues Hazardous Weather Outlook (HWO) htt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3262</th>\n",
       "      <td>10875</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>#CityofCalgary has activated its Municipal Eme...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3263 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         id keyword location  \\\n",
       "0         0     NaN      NaN   \n",
       "1         2     NaN      NaN   \n",
       "2         3     NaN      NaN   \n",
       "3         9     NaN      NaN   \n",
       "4        11     NaN      NaN   \n",
       "...     ...     ...      ...   \n",
       "3258  10861     NaN      NaN   \n",
       "3259  10865     NaN      NaN   \n",
       "3260  10868     NaN      NaN   \n",
       "3261  10874     NaN      NaN   \n",
       "3262  10875     NaN      NaN   \n",
       "\n",
       "                                                   text  \n",
       "0                    Just happened a terrible car crash  \n",
       "1     Heard about #earthquake is different cities, s...  \n",
       "2     there is a forest fire at spot pond, geese are...  \n",
       "3              Apocalypse lighting. #Spokane #wildfires  \n",
       "4         Typhoon Soudelor kills 28 in China and Taiwan  \n",
       "...                                                 ...  \n",
       "3258  EARTHQUAKE SAFETY LOS ANGELES ÛÒ SAFETY FASTE...  \n",
       "3259  Storm in RI worse than last hurricane. My city...  \n",
       "3260  Green Line derailment in Chicago http://t.co/U...  \n",
       "3261  MEG issues Hazardous Weather Outlook (HWO) htt...  \n",
       "3262  #CityofCalgary has activated its Municipal Eme...  \n",
       "\n",
       "[3263 rows x 4 columns]"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Making predictions of test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 228ms/step\n",
      "Pred: 0, Prob: 0.315185546875\n",
      "Text: \n",
      "Photo: bonesymcbones: Beartooth - Body Bag (x) http://t.co/kNH4HLFuMa\n",
      "\n",
      "--------------------------\n",
      "\n",
      "1/1 [==============================] - 0s 202ms/step\n",
      "Pred: 1, Prob: 0.51806640625\n",
      "Text: \n",
      "Job now are only in case of emergency.\n",
      "\n",
      "--------------------------\n",
      "\n",
      "1/1 [==============================] - 0s 211ms/step\n",
      "Pred: 0, Prob: 0.412353515625\n",
      "Text: \n",
      "#hot  Funtenna: hijacking computers to send data as sound waves [Black Hat 2015] http://t.co/a0zgzd1fYa #prebreak #best\n",
      "\n",
      "--------------------------\n",
      "\n",
      "1/1 [==============================] - 0s 244ms/step\n",
      "Pred: 0, Prob: 0.306396484375\n",
      "Text: \n",
      "@CatholicMomVA Or take away his ring &amp; throw it in a volcano...\n",
      "\n",
      "--------------------------\n",
      "\n",
      "1/1 [==============================] - 0s 248ms/step\n",
      "Pred: 0, Prob: 0.403076171875\n",
      "Text: \n",
      "Bat four reasons en route to upheaval versus internet buzz value: pVDRDLJvc\n",
      "\n",
      "--------------------------\n"
     ]
    }
   ],
   "source": [
    "test_sentences = test_df[\"text\"].to_list()\n",
    "test_samples = random.sample(test_sentences,5)\n",
    "for x in test_samples:\n",
    "    pred_probs = tf.squeeze(model_6.predict([x]))\n",
    "    pred = (tf.round(pred_probs))\n",
    "    print(f\"Pred: {int(pred)}, Prob: {pred_probs}\")\n",
    "    print(f\"Text: \\n{x}\\n\")\n",
    "    print(\"--------------------------\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The speed/score tradeoff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 79.26509186351706,\n",
       " 'precision': 0.8111390004213173,\n",
       " 'recall': 0.7926509186351706,\n",
       " 'f1': 0.7862189758049549}"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baseline_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 78.60892388451444,\n",
       " 'precision': 0.7866474744348231,\n",
       " 'recall': 0.7860892388451444,\n",
       " 'f1': 0.78486500301026}"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_6_pretrained_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a function to measure time of prediction\n",
    "import time\n",
    "def pred_time(model,samples):\n",
    "    start = time.perf_counter()\n",
    "    model.predict(samples)\n",
    "    stop = time.perf_counter()\n",
    "    total_time = stop-start\n",
    "    time_per_pred = total_time/len(samples)\n",
    "    return total_time,time_per_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 4s 160ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(4.06398299400098, 0.005333311015749318)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_6_pretrained_time = pred_time(model_6,val_sentences)\n",
    "model_6_pretrained_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.07959060900066106, 0.00010444961811110375)"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baseline_time = pred_time(model_0,val_sentences)\n",
    "baseline_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 78.60892388451444,\n",
       " 'precision': 0.7866474744348231,\n",
       " 'recall': 0.7860892388451444,\n",
       " 'f1': 0.78486500301026}"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_6_pretrained_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'F1-Score versus time per prediction')"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoAAAAGJCAYAAAAaKoJ2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABmEUlEQVR4nO3deVxUVeM/8M8wyL4pyI6D4YoCKgqBqRQoLk9GmbuBaLhiKWWKmbhm5UaR3ygf0kwLs1yoDFHEHZdUNDdU1DQCXEgIULY5vz/8cZ+uLIKCg83n/XrNK+fcc8899zDYx3PvPaMQQggQERERkdbQ0XQHiIiIiOjJYgAkIiIi0jIMgERERERahgGQiIiISMswABIRERFpGQZAIiIiIi3DAEhERESkZRgAiYiIiLQMAyARERGRlmEAJCJqJJydnTF69GhNd4MeMHfuXCgUCllZff+sRo8eDWdn53prj+hhGACJ6smaNWugUCiqfM2cOVOql5SUhLFjx6Jjx45QKpWP9Jf+b7/9hldffRUqlQoGBgZwcHBA7969ERMTU49nRA3h4MGDmDt3Lu7cuaPprtAT9ueff2Lu3LlIS0vTdFeIoKvpDhD928yfPx8tW7aUlXXs2FH68zfffIMNGzagS5cusLe3r3P7Bw8exPPPP48WLVogLCwMtra2uH79Og4dOoSPP/4YU6ZMeexzoIZz8OBBzJs3D6NHj4aFhYVsW3p6OnR0+O/yp8Gj/Kz+/PNPzJs3D87OzujUqZNs26pVq6BWq+uxh0Q1YwAkqmf9+vVD165dq93+/vvvY9WqVWjSpAn+85//4PTp03Vqf9GiRTA3N8fRo0crBYgbN248SpcfWVFREYyMjJ7oMR9FYWEhjI2NNd2Nh9LX19d0F544IQTu3bsHQ0PDem/73r170NPTa5BQXd8/qyZNmtRre0QPw39qEj1h9vb2j/WXfUZGBjp06FAp/AGAtbV1pbJ169bBy8sLRkZGaNq0KXr27ImkpCRZnf/7v/9Dhw4doK+vD3t7e0yePLnSJUo/Pz907NgRx44dQ8+ePWFkZIRZs2YBAIqLixEVFYVWrVpBX18fTk5OeOedd1BcXFzjuYSHh8PExARFRUWVtg0fPhy2trYoLy+Xyn755Rf06NEDxsbGMDU1xYABA3DmzBnZfqNHj4aJiQkyMjLQv39/mJqaYuTIkQCAixcvYtCgQbC1tYWBgQEcHR0xbNgw5OXlAQCuXr0KhUKBNWvWVOqPQqHA3Llzpfd///03pk6dCmdnZ+jr68Pa2hq9e/fG8ePHqz3fuXPnYvr06QCAli1bSrcIXL16FUDl+8oqbivYv38/3njjDTRv3hwWFhYYP348SkpKcOfOHQQHB6Np06Zo2rQp3nnnHQghZMdUq9WIjo5Ghw4dYGBgABsbG4wfPx5//fVXtf18cCwvX76MwMBAGBsbw97eHvPnz3/k4zg7O+M///kPtm/fjq5du8LQ0BCff/55tX345+fO19cXhoaGaNmyJWJjY2X1du/eDYVCgfj4eMyePRsODg4wMjJCfn4+AODw4cPo27cvzM3NYWRkhF69euHAgQOVjrd//35069YNBgYGcHFxqbZvVd0DeOfOHUybNk36TDg6OiI4OBi3bt3C7t270a1bNwBAaGio9LOv+KxVdQ9gYWEh3nrrLTg5OUFfXx9t27bF0qVLK429QqFAeHg4tmzZgo4dO0JfXx8dOnRAYmJiteNKxBlAonqWl5eHW7duycqsrKzqrX2VSoXU1FScPn1admm5KvPmzcPcuXPh6+uL+fPnQ09PD4cPH8auXbvQp08fAPdDybx58xAQEICJEyciPT0dn332GY4ePYoDBw7Iwurt27fRr18/DBs2DKNGjYKNjQ3UajUGDhyI/fv3Y9y4cWjfvj1+++03rFixAhcuXMCWLVuq7d/QoUOxcuVK/Pzzzxg8eLBUXlRUhB9//BGjR4+GUqkEAHz99dcICQlBYGAgPvzwQxQVFeGzzz7Dc889hxMnTsj+51lWVobAwEA899xzWLp0KYyMjFBSUoLAwEAUFxdjypQpsLW1RWZmJn766SfcuXMH5ubmdfo5TJgwAd9//z3Cw8Ph6uqK27dvY//+/Th37hy6dOlS5T6vvPIKLly4gG+//RYrVqyQPhfNmzev8VgV/Z03bx4OHTqEL774AhYWFjh48CBatGiB999/H9u2bcOSJUvQsWNHBAcHS/uOHz8ea9asQWhoKN544w1cuXIFn376KU6cOFHp51uV8vJy9O3bF88++yw++ugjJCYmIioqCmVlZZg/f/4jHSc9PR3Dhw/H+PHjERYWhrZt29bYh7/++gv9+/fHkCFDMHz4cHz33XeYOHEi9PT0MGbMGFndBQsWQE9PD2+//TaKi4uhp6eHXbt2oV+/fvD09ERUVBR0dHSwevVqvPDCC9i3bx+8vLwA3L+3tk+fPmjevDnmzp2LsrIyREVFwcbGpsb+AUBBQQF69OiBc+fOYcyYMejSpQtu3bqFhIQE/PHHH2jfvj3mz5+POXPmYNy4cejRowcAwNfXt8r2hBAYOHAgUlJSMHbsWHTq1Anbt2/H9OnTkZmZiRUrVsjq79+/H5s2bcKkSZNgamqKTz75BIMGDcK1a9dgaWn50P6TFhJEVC9Wr14tAFT5qs6AAQOESqWq03GSkpKEUqkUSqVS+Pj4iHfeeUds375dlJSUyOpdvHhR6OjoiJdfflmUl5fLtqnVaiGEEDdu3BB6enqiT58+sjqffvqpACC+/PJLqaxXr14CgIiNjZW19fXXXwsdHR2xb98+WXlsbKwAIA4cOFDtuajVauHg4CAGDRokK//uu+8EALF3714hhBB///23sLCwEGFhYbJ62dnZwtzcXFYeEhIiAIiZM2fK6p44cUIAEBs3bqy2P1euXBEAxOrVqyttAyCioqKk9+bm5mLy5MnVtlWdJUuWCADiypUrlbapVCoREhIiva/4TAUGBko/MyGE8PHxEQqFQkyYMEEqKysrE46OjqJXr15S2b59+wQAsX79etlxEhMTqyx/UMVYTpkyRSpTq9ViwIABQk9PT9y8ebPOx1GpVAKASExMrPHYFSo+d8uWLZPKiouLRadOnYS1tbX0uU9JSREAxDPPPCOKiopk/W3dunWlMSwqKhItW7YUvXv3lsqCgoKEgYGB+P3336Wys2fPCqVSWen3+MGf1Zw5cwQAsWnTpkrnUHHco0ePVvv5CgkJkf1dsGXLFgFALFy4UFbv1VdfFQqFQly6dEkqAyD09PRkZSdPnhQARExMTKVjEQkhBC8BE9WzlStXYseOHbJXferduzdSU1MxcOBAnDx5Eh999BECAwPh4OCAhIQEqd6WLVugVqsxZ86cSvdAVSxpsXPnTpSUlGDq1KmyOmFhYTAzM8PPP/8s209fXx+hoaGyso0bN6J9+/Zo164dbt26Jb1eeOEFAEBKSkq156JQKDB48GBs27YNBQUFUvmGDRvg4OCA5557DgCwY8cO3LlzB8OHD5cdQ6lUwtvbu8pjTJw4Ufa+YoZv+/btVV5yrisLCwscPnwYf/7552O39TBjx46VLUPi7e0NIQTGjh0rlSmVSnTt2hWXL1+WyjZu3Ahzc3P07t1bNm6enp4wMTGp8WfzT+Hh4dKfKy43lpSUYOfOnY90nJYtWyIwMLDW56+rq4vx48dL7/X09DB+/HjcuHEDx44dk9UNCQmR3U+YlpaGixcvYsSIEbh9+7bUt8LCQvj7+2Pv3r1Qq9UoLy/H9u3bERQUhBYtWkj7t2/fvlZ9/eGHH+Dh4YGXX3650rYHl5CpjW3btkGpVOKNN96Qlb/11lsQQuCXX36RlQcEBMDFxUV67+7uDjMzM9nngeifeAmYqJ55eXnV+BBIbZSXl+PmzZuysmbNmkFPTw8A0K1bN2zatAklJSU4efIkNm/ejBUrVuDVV19FWloaXF1dkZGRAR0dHbi6ulZ7nN9//x0AKl2C09PTwzPPPCNtr+Dg4CD1ocLFixdx7ty5ai9jPuzBlKFDhyI6OhoJCQkYMWIECgoKsG3bNowfP176H+fFixcBQAqVDzIzM5O919XVhaOjo6ysZcuWiIiIwPLly7F+/Xr06NEDAwcOxKhRo+p8+RcAPvroI4SEhMDJyQmenp7o378/goOD8cwzz9S5rYf5ZyAB/hdmnZycKpX/8567ixcvIi8vr8p7Q4HaPTSko6NT6ZzatGkDANK9i3U9zoNPyT+Mvb19pYd4/tmHZ599ttq2Kz47ISEh1bafl5eH4uJi3L17F61bt660vW3btti2bVuNfczIyMCgQYNqPpE6+P3332Fvbw9TU1NZefv27aXt//TgZwQAmjZtWqt7PUk7MQASNULXr1+v9D+ylJQU+Pn5ycr09PTQrVs3dOvWDW3atEFoaCg2btyIqKioBulXVU9qqtVquLm5Yfny5VXu82BIedCzzz4LZ2dnfPfddxgxYgR+/PFH3L17F0OHDpUdA7h/H6CtrW2lNnR15X+V6evrV/nk57JlyzB69Ghs3boVSUlJeOONN7B48WIcOnQIjo6O1c7U/PNBlApDhgxBjx49sHnzZiQlJWHJkiX48MMPsWnTJvTr16/Gc66rivsga1Mu/vGAgFqthrW1NdavX1/l/g+797C26nqchnjit7q2Kz47S5YsqbT0SgUTE5OHPrDU2FX3GREPPDBCVIEBkKgRsrW1rXTp2MPDo8Z9KmYds7KyAAAuLi5Qq9U4e/Zstf/jU6lUAO7flP/PWZ6SkhJcuXIFAQEBD+2ri4sLTp48CX9//0e61AXcD1Mff/wx8vPzsWHDBjg7O8tmdSoubVlbW9eqTzVxc3ODm5sbZs+ejYMHD6J79+6IjY3FwoUL0bRpUwCo9AT0g7MtFezs7DBp0iRMmjQJN27cQJcuXbBo0aIaA+CjjtGjcHFxwc6dO9G9e/dHDl1qtRqXL1+WZtwA4MKFCwAgPXhTH8epyZ9//llpKZ8H+1Cdis+OmZlZjZ+d5s2bw9DQUJox/Kf09PSH9tHFxeWhSzrV5WevUqmwc+dO/P3337JZwPPnz0vbiR4H7wEkaoQMDAwQEBAge1WEk5SUlCr/VV9xiaricm5QUBB0dHQwf/78SgvMVuwfEBAAPT09fPLJJ7I24+LikJeXhwEDBjy0r0OGDEFmZiZWrVpVadvdu3dRWFj40DaGDh2K4uJifPXVV0hMTMSQIUNk2wMDA2FmZob3338fpaWllfZ/8HJ5VfLz81FWViYrc3Nzg46OjjT7Y2ZmBisrK+zdu1dW7//+7/9k78vLy6WlYypYW1vD3t7+oTNJFSHmSXwTyJAhQ1BeXo4FCxZU2lZWVlbrPnz66afSn4UQ+PTTT9GkSRP4+/vX63GqU1ZWJluOpaSkBJ9//jmaN28OT0/PGvf19PSEi4sLli5dKrvPtELFZ0epVCIwMBBbtmzBtWvXpO3nzp3D9u3bH9rHQYMGSbdjPKjid6suP/v+/fujvLxcNvYAsGLFCigUinqfZSbtwxlAoifs1KlT0sMaly5dQl5eHhYuXAjg/izfiy++WOP+U6ZMQVFREV5++WW0a9cOJSUlOHjwoDRzVvGQRqtWrfDuu+9iwYIF6NGjB1555RXo6+vj6NGjsLe3x+LFi9G8eXNERkZi3rx56Nu3LwYOHIj09HT83//9H7p164ZRo0Y99Hxee+01fPfdd5gwYQJSUlLQvXt3lJeX4/z58/juu++k9d5q0qVLF6m/xcXFssu/wP1g9tlnn+G1115Dly5dMGzYMDRv3hzXrl3Dzz//jO7du1f6H+WDdu3ahfDwcAwePBht2rRBWVkZvv76ayiVStm9W6+//jo++OADvP766+jatSv27t0rzTZV+Pvvv+Ho6IhXX30VHh4eMDExwc6dO3H06FEsW7asxn5UBJZ3330Xw4YNQ5MmTfDiiy82yELVvXr1wvjx47F48WKkpaWhT58+aNKkCS5evIiNGzfi448/xquvvlpjGwYGBkhMTERISAi8vb3xyy+/4Oeff8asWbOkS7v1cZya2Nvb48MPP8TVq1fRpk0bbNiwAWlpafjiiy8euoyNjo4O/vvf/6Jfv37o0KEDQkND4eDggMzMTKSkpMDMzAw//vgjgPvLJiUmJqJHjx6YNGkSysrKEBMTgw4dOuDUqVM1Hmf69On4/vvvMXjwYIwZMwaenp7Izc1FQkICYmNj4eHhARcXF1hYWCA2NhampqYwNjaGt7d3lfdEvvjii3j++efx7rvv4urVq/Dw8EBSUhK2bt2KqVOnyh74IHokGnv+mOhfpmLJjqNHj9aqXlWvfy4rUZ1ffvlFjBkzRrRr106YmJgIPT090apVKzFlyhSRk5NTqf6XX34pOnfuLPT19UXTpk1Fr169xI4dO2R1Pv30U9GuXTvRpEkTYWNjIyZOnCj++usvWZ1evXqJDh06VNmnkpIS8eGHH4oOHTpIx/H09BTz5s0TeXl5Dz0nIYR49913BQDRqlWrauukpKSIwMBAYW5uLgwMDISLi4sYPXq0+PXXX6U6ISEhwtjYuNK+ly9fFmPGjBEuLi7CwMBANGvWTDz//PNi586dsnpFRUVi7NixwtzcXJiamoohQ4aIGzduyJaBKS4uFtOnTxceHh7C1NRUGBsbCw8PD/F///d/tTrXBQsWCAcHB6GjoyNbEqa6ZWAe/ExFRUUJANIyLA879y+++EJ4enoKQ0NDYWpqKtzc3MQ777wj/vzzzxr7WdFeRkaG6NOnjzAyMhI2NjYiKiqq0tJCtT2OSqUSAwYMeNgQSSo+d7/++qvw8fERBgYGQqVSiU8//VRWr2IZmOqW+Tlx4oR45ZVXhKWlpdDX1xcqlUoMGTJEJCcny+rt2bNHeHp6Cj09PfHMM8+I2NhYabz/6cGflRBC3L59W4SHhwsHBwehp6cnHB0dRUhIiLh165ZUZ+vWrcLV1VXo6urKloR5cBkYIe4vfzRt2jRhb28vmjRpIlq3bi2WLFkiW85GiPvLwFS1JFFVfSSqoBCCd4gSEVFlo0ePxvfff1/lpdMnxc/PD7du3arzVyYSUc14DyARERGRlmEAJCIiItIyDIBEREREWob3ABIRERFpGc4AEhEREWkZBkAiIiIiLcOFoBuQWq3Gn3/+CVNT0yf69U9ERESkfYQQ+Pvvv2Fvb1/l96H/EwNgA/rzzz/h5OSk6W4QERGRFrl+/TocHR1rrMMA2IAqvsD7+vXrMDMz03BviIiI6N8sPz8fTk5OUv6oCQNgA6q47GtmZsYASERERE9EbW4740MgRERERFqGAZCIiIhIyzAAEhEREWkZ3gNIREQNTgiBsrIylJeXa7orRE8tpVIJXV3dellajgGQiIgaVElJCbKyslBUVKTprhA99YyMjGBnZwc9Pb3HaocBkIiIGoxarcaVK1egVCphb28PPT09LoxP9AiEECgpKcHNmzdx5coVtG7d+qGLPdeEAfApVa4WOHIlFzf+vgdrUwN4tWwGpQ7/UiWixqWkpARqtRpOTk4wMjLSdHeInmqGhoZo0qQJfv/9d5SUlMDAwOCR22IAfAolns7CvB/PIivvnlRmZ26AqBdd0bejnQZ7RkRUtceZqSCi/6mv3yX+Rj5lEk9nYeK647LwBwDZefcwcd1xJJ7O0lDPiIiI6GnBAPgUKVcLzPvxLEQV2yrK5v14FuXqqmoQERER3ccA+BQ5ciW30szfPwkAWXn3cORK7pPrFBHRv5Sfnx+mTp2qseOPHj0aQUFBjaY/9O+i8QC4cuVKODs7w8DAAN7e3jhy5Ei1df38/KBQKCq9BgwYINUpKChAeHg4HB0dYWhoCFdXV8TGxlZqKzU1FS+88AKMjY1hZmaGnj174u7duwCAq1evYuzYsWjZsiUMDQ3h4uKCqKgolJSU1P8A1MGNv6sPf49Sj4iInh6bNm3CggULNN0N+pfQ6EMgGzZsQEREBGJjY+Ht7Y3o6GgEBgYiPT0d1tbWlepv2rRJFsJu374NDw8PDB48WCqLiIjArl27sG7dOjg7OyMpKQmTJk2Cvb09Bg4cCOB++Ovbty8iIyMRExMDXV1dnDx5Urqx8vz581Cr1fj888/RqlUrnD59GmFhYSgsLMTSpUsbeFSqZ21au6d9aluPiOhpou2rHzRr1kzTXaB/EY3OAC5fvhxhYWEIDQ2VZuqMjIzw5ZdfVlm/WbNmsLW1lV47duyAkZGRLAAePHgQISEh8PPzg7OzM8aNGwcPDw/ZzOK0adPwxhtvYObMmejQoQPatm2LIUOGQF9fHwDQt29frF69Gn369MEzzzyDgQMH4u2338amTZsadkAewqtlM9iZG6C6v+4UuP80sFdL/iVBRP8uiaez8NyHuzB81SG8GZ+G4asO4bkPdzX4g29lZWUIDw+Hubk5rKys8N5770GI+/dZf/311+jatStMTU1ha2uLESNG4MaNG9K+f/31F0aOHInmzZvD0NAQrVu3xurVq6Xt169fx5AhQ2BhYYFmzZrhpZdewtWrV6vty4OXgJ2dnfH+++9jzJgxMDU1RYsWLfDFF1/I9qnrMUh7aCwAlpSU4NixYwgICPhfZ3R0EBAQgNTU1Fq1ERcXh2HDhsHY2Fgq8/X1RUJCAjIzMyGEQEpKCi5cuIA+ffoAAG7cuIHDhw/D2toavr6+sLGxQa9evbB///4aj5WXl/fQf30VFxcjPz9f9qpPSh0Fol50BYBKIbDifdSLrlr1L2Ii+vfT5OoHX331FXR1dXHkyBF8/PHHWL58Of773/8CAEpLS7FgwQKcPHkSW7ZswdWrVzF69Ghp3/feew9nz57FL7/8gnPnzuGzzz6DlZWVtG9gYCBMTU2xb98+HDhwACYmJujbt2+dbjdatmwZunbtihMnTmDSpEmYOHEi0tPT6/UY9O+ksUvAt27dQnl5OWxsbGTlNjY2OH/+/EP3P3LkCE6fPo24uDhZeUxMDMaNGwdHR0fo6upCR0cHq1atQs+ePQEAly9fBgDMnTsXS5cuRadOnbB27Vr4+/vj9OnTaN26daVjXbp0CTExMQ+9/Lt48WLMmzfvoX1/HH072uGzUV0qrQNoy3UAiehf6GGrHyhwf/WD3q62DfKPXycnJ6xYsQIKhQJt27bFb7/9hhUrViAsLAxjxoyR6j3zzDP45JNP0K1bNxQUFMDExATXrl1D586d0bVrVwD3Z+wqbNiwAWq1Gv/973+lb0ZZvXo1LCwssHv3bmnS4mH69++PSZMmAQBmzJiBFStWICUlBW3btq23Y9C/01O7EHRcXBzc3Nzg5eUlK4+JicGhQ4eQkJAAlUqFvXv3YvLkybC3t0dAQADUajUAYPz48QgNDQUAdO7cGcnJyfjyyy+xePFiWXuZmZno27cvBg8ejLCwsBr7FBkZiYiICOl9fn4+nJyc6uN0Zfp2tENvV1utvheGiLRDXVY/8HGxrPfjP/vss7KvrvPx8cGyZctQXl6OtLQ0zJ07FydPnsRff/0l/f/l2rVrcHV1xcSJEzFo0CAcP34cffr0QVBQEHx9fQEAJ0+exKVLl2Bqaio73r1795CRkVHr/rm7u0t/VigUsLW1lS5D19cx6N9JYwHQysoKSqUSOTk5svKcnBzY2trWuG9hYSHi4+Mxf/58Wfndu3cxa9YsbN68WXoy2N3dHWlpaVi6dCkCAgJgZ3d/hszV1VW2b/v27XHt2jVZ2Z9//onnn38evr6+le6rqIq+vr50H2FDU+ooGuQvOyKixqSxrn5w7949BAYGIjAwEOvXr0fz5s1x7do1BAYGSpdX+/Xrh99//x3btm3Djh074O/vj8mTJ2Pp0qUoKCiAp6cn1q9fX6nt5s2b17ofTZo0kb1XKBRSEK2vY9C/k8buAdTT04OnpyeSk5OlMrVajeTkZPj4+NS478aNG1FcXIxRo0bJyktLS1FaWlrpa1KUSqX0C+Hs7Ax7e3vpHokKFy5cgEqlkt5nZmbCz88Pnp6eWL16Nb/GiIhIAzS9+sHhw4dl7w8dOoTWrVvj/PnzuH37Nj744AP06NED7dq1kz0AUqF58+YICQnBunXrEB0dLU0mdOnSBRcvXoS1tTVatWole5mbm9dL35/EMejppdFUExERgVWrVuGrr77CuXPnMHHiRBQWFkqXZoODgxEZGVlpv7i4OAQFBcHSUj4DZmZmhl69emH69OnYvXs3rly5gjVr1mDt2rV4+eWXAdz/19H06dPxySef4Pvvv8elS5fw3nvv4fz58xg7diyA/4W/Fi1aYOnSpbh58yays7ORnZ3dwCNCRET/pOnVD65du4aIiAikp6fj22+/RUxMDN588020aNECenp6iImJweXLl5GQkFBpjb45c+Zg69atuHTpEs6cOYOffvoJ7du3BwCMHDkSVlZWeOmll7Bv3z5cuXIFu3fvxhtvvIE//vijXvr+JI5BTy+N3gM4dOhQ3Lx5E3PmzEF2djY6deqExMRE6cGQa9euVZp5S09Px/79+5GUlFRlm/Hx8YiMjMTIkSORm5sLlUqFRYsWYcKECVKdqVOn4t69e5g2bRpyc3Ph4eGBHTt2wMXFBQCwY8cOXLp0CZcuXYKjo6Os/YrH/4mIqOFVrH4wcd1xKADZwyBPYvWD4OBg3L17F15eXlAqlXjzzTcxbtw4KBQKrFmzBrNmzcInn3yCLl26YOnSpdJ6s8D9K12RkZG4evUqDA0N0aNHD8THxwMAjIyMsHfvXsyYMQOvvPIK/v77bzg4OMDf3x9mZmb10vcncQx6eikEE02Dyc/Ph7m5OfLy8vjLRkRa6d69e7hy5QpatmwJA4NHv0ybeDqr0uoHdlz9gLRQTb9TdckdT+1TwEREpD24+gFR/WIAJCKipwJXPyCqP3y0lYiIiEjLMAASERERaRkGQCIiIiItwwBIREREpGUYAImIiIi0DAMgERERkZZhACQiIiLSMgyAREREj+jAgQNwc3NDkyZNEBQUVK9tX716FQqFAmlpafXaLj05fn5+mDp1qqa7USUGQCIiogcoFIoaX3PnzgUAREREoFOnTrhy5QrWrFlTY2hrqDCwefNmPPvsszA3N4epqSk6dOhQ78fZvXs3FAoF7ty5U6/t1oeKvlX1ys7O1nT3Gi1+EwgRET0d1OXA7weBghzAxAZQ+QI6ygY5VFZWlvTnDRs2YM6cOUhPT5fKTExMAAAZGRmYMGECHB0dAeCJB6Tk5GQMHToUixYtwsCBA6FQKHD27Fns2LHjifajMUhPT6/0/bfW1tYa6k39KC8vh0KhgI5O/c/XcQaQiIgav7MJQHRH4Kv/AD+Mvf/f6I73yxuAra2t9DI3N4dCoZCV3bp1CwqFArdv38aYMWOgUCiwZs2aOh1DoVBgy5YtsjILC4tK7Zw/fx6+vr4wMDBAx44dsWfPHmnbjz/+iO7du2P69Olo27Yt2rRpg6CgIKxcuVLWxtatW9GlSxcYGBjgmWeewbx581BWVibry3//+1+8/PLLMDIyQuvWrZGQcH9sr169iueffx4A0LRpUygUCowePRoAoFarsXjxYrRs2RKGhobw8PDA999/L7VbMTuXnJyMrl27wsjICL6+vrIwXXEe3bp1g4GBAaysrPDyyy9L24qLi/H222/DwcEBxsbG8Pb2xu7duyuNp7W1texnZGtrKwWn0aNHIygoCEuXLoWdnR0sLS0xefJklJaWyo4zY8YMODk5QV9fH61atUJcXJy0fc+ePfDy8oK+vj7s7Owwc+ZM2RgWFhYiODgYJiYmsLOzw7Jlyyr18WHnsmbNGlhYWCAhIQGurq7Q19fHtWvXKrVTHxgAiYiocTubAHwXDOT/KS/Pz7pf3kAhsCZOTk7IysqCmZkZoqOjkZWVhaFDhzbIsaZPn4633noLJ06cgI+PD1588UXcvn0bwP2geubMGZw+fbra/fft24fg4GC8+eabOHv2LD7//HOsWbMGixYtktWbN28ehgwZglOnTqF///4YOXIkcnNz4eTkhB9++AHA/Vm2rKwsfPzxxwCAxYsXY+3atYiNjcWZM2cwbdo0jBo1ShZSAeDdd9/FsmXL8Ouvv0JXVxdjxoyRtv388894+eWX0b9/f5w4cQLJycnw8vKStoeHhyM1NRXx8fE4deoUBg8ejL59++LixYt1GseUlBRkZGQgJSUFX331FdasWSML28HBwfj222/xySef4Ny5c/j888+lmd7MzEz0798f3bp1w8mTJ/HZZ58hLi4OCxculPafPn069uzZg61btyIpKQm7d+/G8ePHZX2ozbkUFRXhww8/xH//+1+cOXOm4WYxBTWYvLw8AUDk5eVpuitERBpx9+5dcfbsWXH37t1Ha6C8TIhl7YSIMqvmZS7Esvb36zWQ1atXC3Nz8yq3mZubi9WrV0vvr1y5IgAIQ0NDYWxsLHvp6OiIN998U6oLQGzevLna9ira+uCDD6TtpaWlwtHRUXz44YdCCCEKCgpE//79BQChUqnE0KFDRVxcnLh37560j7+/v3j//fdlx/n666+FnZ2drC+zZ8+W3hcUFAgA4pdffhFCCJGSkiIAiL/++kuqc+/ePWFkZCQOHjwoa3vs2LFi+PDhsv127twpbf/5558FAOkz4ePjI0aOHFnl+P7+++9CqVSKzMxMWbm/v7+IjIyUHePB8XZ1dZXqh4SECJVKJcrK/vc5GTx4sBg6dKgQQoj09HQBQOzYsaPKfsyaNUu0bdtWqNVqqWzlypXCxMRElJeXi7///lvo6emJ7777Ttp++/ZtYWhoKP3Ma3Muq1evFgBEWlpalf0QoubfqbrkDt4DSEREjdfvByvP/MkIID/zfr2WPZ5Ytx5mw4YNaN++vaxs5MiRj9SWj4+P9GddXV107doV586dAwAYGxvj559/lma2Dh06hLfeegsff/wxUlNTYWRkhJMnT+LAgQOyGb/y8nLcu3cPRUVFMDIyAgC4u7tL242NjWFmZoYbN25U269Lly6hqKgIvXv3lpWXlJSgc+fOsrJ/tm1nZwcAuHHjBlq0aIG0tDSEhYVVeYzffvsN5eXlaNOmjay8uLgYlpaWsrJ9+/bB1NRUet+kSRPZ9g4dOkCp/N89o3Z2dvjtt98AAGlpaVAqlejVq1eV/Th37hx8fHygUCiksu7du6OgoAB//PEH/vrrL5SUlMDb21va3qxZM7Rt27bO56Knpycbr4bCAEhERI1XQU791ntCnJyc0KpVK1mZoaGh7L1CoYAQQlb2z3vS6sLFxQUuLi54/fXX8e6776JNmzbYsGEDQkNDUVBQgHnz5uGVV16ptJ+BgYH05wcDk0KhgFqtrvaYBQUFAO5fwnVwcJBt09fXl73/Z9sVIaqi7QfH5cFjKJVKHDt2TBbegP89iFOhZcuWsLCwqLatms6vpj7Ul9qei6GhoSxoNhQGQCIiarxMbOq3XiPSvHlz2dPGFy9eRFFRUaV6hw4dQs+ePQEAZWVlOHbsGMLDw6tt19nZGUZGRigsLAQAdOnSBenp6ZUCaV3o6ekBuD9zWOGfDylUN3NWG+7u7khOTkZoaGilbZ07d0Z5eTlu3LiBHj0abobXzc0NarUae/bsQUBAQKXt7du3xw8//AAhhBTODhw4AFNTUzg6OqJZs2Zo0qQJDh8+jBYtWgAA/vrrL1y4cEEamyd1LrXFAEhERI2Xyhcws7//wAdEFRUU97erfJ90zx7bCy+8gE8//RQ+Pj4oLy/HjBkzKs1SAcDKlSvRunVrtG/fHitWrMBff/0lPUQxd+5cFBUVoX///lCpVLhz5w4++eQTlJaWSpdm58yZg//85z9o0aIFXn31Vejo6ODkyZM4ffq07CGGmqhUKigUCvz000/o378/DA0NYWpqirfffhvTpk2DWq3Gc889h7y8PBw4cABmZmYICQmpVdtRUVHw9/eHi4sLhg0bhrKyMmzbtg0zZsxAmzZtMHLkSAQHB2PZsmXo3Lkzbt68ieTkZLi7u2PAgAFSOzdu3MC9e/dkbVtaWlY5pg9ydnZGSEgIxowZg08++QQeHh74/fffcePGDQwZMgSTJk1CdHQ0pkyZgvDwcKSnpyMqKgoRERHQ0dGBiYkJxo4di+nTp8PS0hLW1tZ49913Zcu31OVcngQ+BUxERI2XjhLo++H/f/PgZbH//77vBw22HmBDWrZsGZycnNCjRw+MGDECb7/9tnQ/3j998MEH+OCDD+Dh4YH9+/cjISEBVlZWAIBevXrh8uXLCA4ORrt27dCvXz9kZ2cjKSlJuv8sMDAQP/30E5KSktCtWzc8++yzWLFiBVQqVa376uDggHnz5mHmzJmwsbGRZiAXLFiA9957D4sXL0b79u3Rt29f/Pzzz2jZsmWt2/bz88PGjRuRkJCATp064YUXXsCRI0ek7atXr0ZwcDDeeusttG3bFkFBQTh69Kg001ahbdu2sLOzk72OHTtW63589tlnePXVVzFp0iS0a9cOYWFh0iyqg4MDtm3bhiNHjsDDwwMTJkzA2LFjMXv2bGn/JUuWoEePHnjxxRcREBCA5557Dp6enrJj1PZcngSFePAGBKo3+fn5MDc3R15eXqXFKYmItMG9e/dw5coVtGzZUna/WZ2dTQASZ8gfCDFzuB/+XAc+fkeJnhI1/U7VJXfwEjARETV+rgOBdgOe2DeBEP3bMQASEdHTQUfZqJZ6IXqa8R5AIiIiIi3DAEhERESkZRgAiYiowfF5Q6L6UV+/SwyARETUYCrWYKtqgWMiqruK36XarG9YEz4EQkREDUapVMLCwkL6TlkjI6Mn8jVXRP82QggUFRXhxo0bsLCwqPR1cnXFAEhERA3K1tYWAKQQSESPzsLCQvqdehwMgERE1KAUCgXs7OxgbW2N0tJSTXeH6KnVpEmTx575q6DxALhy5UosWbIE2dnZ8PDwQExMDLy8vKqs6+fnhz179lQq79+/P37++WcAQEFBAWbOnIktW7bg9u3baNmyJd544w1MmDBBtk9qaireffddHD58GEqlEp06dcL27dthaGgIAMjNzcWUKVPw448/QkdHB4MGDcLHH38MExOTeh4BIiLtoFQq6+1/XkT0eDT6EMiGDRsQERGBqKgoHD9+HB4eHggMDKz2MsGmTZuQlZUlvU6fPg2lUonBgwdLdSIiIpCYmIh169bh3LlzmDp1KsLDw5GQkCDVSU1NRd++fdGnTx8cOXIER48eRXh4uOxLm0eOHIkzZ85gx44d+Omnn7B3716MGzeu4QaDiIiI6AnR6HcBe3t7o1u3bvj0008BAGq1Gk5OTpgyZQpmzpz50P2jo6MxZ84cZGVlwdjYGADQsWNHDB06FO+9955Uz9PTE/369cPChQsBAM8++yx69+6NBQsWVNnuuXPn4OrqiqNHj6Jr164AgMTERPTv3x9//PEH7O3ta3V+/C5gIiIielLqkjs0NgNYUlKCY8eOISAg4H+d0dFBQEAAUlNTa9VGXFwchg0bJoU/APD19UVCQgIyMzMhhEBKSgouXLiAPn36ALh/E/Lhw4dhbW0NX19f2NjYoFevXti/f7/URmpqKiwsLKTwBwABAQHQ0dHB4cOHq+1PcXEx8vPzZS8iIiKixkZjAfDWrVsoLy+HjY2NrNzGxgbZ2dkP3f/IkSM4ffo0Xn/9dVl5TEwMXF1d4ejoCD09PfTt2xcrV65Ez549AQCXL18GAMydOxdhYWFITExEly5d4O/vj4sXLwIAsrOzYW1tLWtXV1cXzZo1q7Fvixcvhrm5ufRycnJ6+EAQERERPWFP7ULQcXFxcHNzq/TASExMDA4dOoSEhAQcO3YMy5Ytw+TJk7Fz504A9y8zA8D48eMRGhqKzp07Y8WKFWjbti2+/PLLx+pTZGQk8vLypNf169cfqz0iIiKihqCxp4CtrKygVCqRk5MjK8/JyXno+jaFhYWIj4/H/PnzZeV3797FrFmzsHnzZgwYMAAA4O7ujrS0NCxduhQBAQGws7MDALi6usr2bd++Pa5duwbg/ppVDz6IUlZWhtzc3Br7pq+vD319/Rr7TkRERKRpGpsB1NPTg6enJ5KTk6UytVqN5ORk+Pj41Ljvxo0bUVxcjFGjRsnKS0tLUVpaKnuaF7i/9EDFzJ+zszPs7e2Rnp4uq3PhwgWoVCoAgI+PD+7cuYNjx45J23ft2gW1Wg1vb++6nywRERFRI6LRdQAjIiIQEhKCrl27wsvLC9HR0SgsLERoaCgAIDg4GA4ODli8eLFsv7i4OAQFBcHS0lJWbmZmhl69emH69OkwNDSESqXCnj17sHbtWixfvhzA/QVJp0+fjqioKHh4eKBTp0746quvcP78eXz//fcA7s8G9u3bF2FhYYiNjUVpaSnCw8MxbNiwWj8BTERERNRYaTQADh06FDdv3sScOXOQnZ2NTp06ITExUXow5Nq1a5Vm89LT07F//34kJSVV2WZ8fDwiIyMxcuRI5ObmQqVSYdGiRbKFoKdOnYp79+5h2rRpyM3NhYeHB3bs2AEXFxepzvr16xEeHg5/f39pIehPPvmkAUaBiIiI6MnS6DqA/3ZcB5CIiIielKdiHUAiIiIi0gwGQCIiIiItwwBIREREpGUYAImIiIi0DAMgERERkZZhACQiIiLSMgyARERERFqGAZCIiIhIyzAAEhEREWkZBkAiIiIiLcMASERERKRlGACJiIiItAwDIBEREZGWYQAkIiIi0jIMgERERERahgGQiIiISMswABIRERFpGQZAIiIiIi3DAEhERESkZRgAiYiIiLQMAyARERGRlmEAJCIiItIyDIBEREREWoYBkIiIiEjLMAASERERaRkGQCIiIiItwwBIREREpGUYAImIiIi0DAMgERERkZZhACQiIiLSMhoPgCtXroSzszMMDAzg7e2NI0eOVFvXz88PCoWi0mvAgAFSnYKCAoSHh8PR0RGGhoZwdXVFbGzsQ9uZMGGCrM7Ro0fh7+8PCwsLNG3aFIGBgTh58mT9njwRERGRBmg0AG7YsAERERGIiorC8ePH4eHhgcDAQNy4caPK+ps2bUJWVpb0On36NJRKJQYPHizViYiIQGJiItatW4dz585h6tSpCA8PR0JCgqytsLAwWVsfffSRtK2goAB9+/ZFixYtcPjwYezfvx+mpqYIDAxEaWlpwwwGERER0ROi0QC4fPlyhIWFITQ0VJqpMzIywpdfflll/WbNmsHW1lZ67dixA0ZGRrIAePDgQYSEhMDPzw/Ozs4YN24cPDw8Ks0sGhkZydoyMzOTtp0/fx65ubmYP38+2rZtiw4dOiAqKgo5OTn4/fffG2YwiIiIiJ4QjQXAkpISHDt2DAEBAf/rjI4OAgICkJqaWqs24uLiMGzYMBgbG0tlvr6+SEhIQGZmJoQQSElJwYULF9CnTx/ZvuvXr4eVlRU6duyIyMhIFBUVSdvatm0LS0tLxMXFoaSkBHfv3kVcXBzat28PZ2fnavtTXFyM/Px82YuIiIiosdHV1IFv3bqF8vJy2NjYyMptbGxw/vz5h+5/5MgRnD59GnFxcbLymJgYjBs3Do6OjtDV1YWOjg5WrVqFnj17SnVGjBgBlUoFe3t7nDp1CjNmzEB6ejo2bdoEADA1NcXu3bsRFBSEBQsWAABat26N7du3Q1e3+iFbvHgx5s2bV+sxICIiItIEjQXAxxUXFwc3Nzd4eXnJymNiYnDo0CEkJCRApVJh7969mDx5Muzt7aXZxnHjxkn13dzcYGdnB39/f2RkZMDFxQV3797F2LFj0b17d3z77bcoLy/H0qVLMWDAABw9ehSGhoZV9ikyMhIRERHS+/z8fDg5OTXA2RMRERE9Oo0FQCsrKyiVSuTk5MjKc3JyYGtrW+O+hYWFiI+Px/z582Xld+/exaxZs7B582bpyWB3d3ekpaVh6dKlssvN/+Tt7Q0AuHTpElxcXPDNN9/g6tWrSE1NhY7O/avk33zzDZo2bYqtW7di2LBhVbajr68PfX39h588ERERkQZp7B5APT09eHp6Ijk5WSpTq9VITk6Gj49Pjftu3LgRxcXFGDVqlKy8tLQUpaWlUmiroFQqoVarq20vLS0NAGBnZwcAKCoqgo6ODhQKhVSn4n1N7RARERE9DTT6FHBERARWrVqFr776CufOncPEiRNRWFiI0NBQAEBwcDAiIyMr7RcXF4egoCBYWlrKys3MzNCrVy9Mnz4du3fvxpUrV7BmzRqsXbsWL7/8MgAgIyMDCxYswLFjx3D16lUkJCQgODgYPXv2hLu7OwCgd+/e+OuvvzB58mScO3cOZ86cQWhoKHR1dfH888838KgQERERNSyN3gM4dOhQ3Lx5E3PmzEF2djY6deqExMRE6cGQa9euVZrNS09Px/79+5GUlFRlm/Hx8YiMjMTIkSORm5sLlUqFRYsWSQs96+npYefOnYiOjkZhYSGcnJwwaNAgzJ49W2qjXbt2+PHHHzFv3jz4+PhAR0cHnTt3RmJiojRLSERERPS0UgghhKY78W+Vn58Pc3Nz5OXlydYZJCIiIqpvdckdGv8qOCIiIiJ6shgAiYiIiLQMAyARERGRlmEAJCIiItIyDIBEREREWoYBkIiIiEjLMAASERERaRkGQCIiIiItwwBIREREpGUYAImIiIi0DAMgERERkZZhACQiIiLSMgyARERERFqGAZCIiIhIyzAAEhEREWkZBkAiIiIiLcMASERERKRlGACJiIiItAwDIBEREZGWYQAkIiIi0jKPFAD37duHUaNGwcfHB5mZmQCAr7/+Gvv376/XzhERERFR/atzAPzhhx8QGBgIQ0NDnDhxAsXFxQCAvLw8vP/++/XeQSIiIiKqX3UOgAsXLkRsbCxWrVqFJk2aSOXdu3fH8ePH67VzRERERFT/6hwA09PT0bNnz0rl5ubmuHPnTn30iYiIiIgaUJ0DoK2tLS5dulSpfP/+/XjmmWfqpVNERERE1HDqHADDwsLw5ptv4vDhw1AoFPjzzz+xfv16vP3225g4cWJD9JGIiIiI6pFuXXeYOXMm1Go1/P39UVRUhJ49e0JfXx9vv/02pkyZ0hB9JCIiIqJ6pBBCiNpWLi8vx4EDB+Du7g4jIyNcunQJBQUFcHV1hYmJSUP286mUn58Pc3Nz5OXlwczMTNPdISIion+xuuSOOs0AKpVK9OnTB+fOnYOFhQVcXV0fq6NERERE9OTV+R7Ajh074vLlyw3RFyIiIiJ6Ah5pHcC3334bP/30E7KyspCfny97EREREVHjVucA2L9/f5w8eRIDBw6Eo6MjmjZtiqZNm8LCwgJNmzatcwdWrlwJZ2dnGBgYwNvbG0eOHKm2rp+fHxQKRaXXgAEDpDoFBQUIDw+Ho6MjDA0N4erqitjY2Ie2M2HChErHW7NmDdzd3WFgYABra2tMnjy5zudHRERE1NjU+SnglJSUejv4hg0bEBERgdjYWHh7eyM6OhqBgYFIT0+HtbV1pfqbNm1CSUmJ9P727dvw8PDA4MGDpbKIiAjs2rUL69atg7OzM5KSkjBp0iTY29tj4MCBUr2wsDDMnz9fem9kZCQ71vLly7Fs2TIsWbIE3t7eKCwsxNWrV+vt3ImIiIg0pU5PAdc3b29vdOvWDZ9++ikAQK1Ww8nJCVOmTMHMmTMfun90dDTmzJmDrKwsGBsbA7h/j+LQoUPx3nvvSfU8PT3Rr18/LFy4EMD9GcBOnTohOjq6ynb/+usvODg44Mcff4S/v/8jnx+fAiYiIqInpS65o86XgAHgzp07WLZsGV5//XW8/vrrWLFiBfLy8urURklJCY4dO4aAgID/dUZHBwEBAUhNTa1VG3FxcRg2bJgU/gDA19cXCQkJyMzMhBACKSkpuHDhAvr06SPbd/369bCyskLHjh0RGRmJoqIiaduOHTugVquRmZmJ9u3bw9HREUOGDMH169dr7E9xcTHviSQiIqJGr84B8Ndff4WLiwtWrFiB3Nxc5ObmYvny5XBxccHx48dr3c6tW7dQXl4OGxsbWbmNjQ2ys7Mfuv+RI0dw+vRpvP7667LymJgYuLq6wtHREXp6eujbty9Wrlwp+/7iESNGYN26dUhJSUFkZCS+/vprjBo1Stp++fJlqNVqvP/++4iOjsb333+P3Nxc9O7dW3YJ+kGLFy+Gubm59HJycqrtcBARERE9MXW+B3DatGkYOHAgVq1aBV3d+7uXlZXh9ddfx9SpU7F3795672RV4uLi4ObmBi8vL1l5TEwMDh06hISEBKhUKuzduxeTJ0+Gvb29NNs4btw4qb6bmxvs7Ozg7++PjIwMuLi4QK1Wo7S0FJ988ok0c/jtt9/C1tYWKSkpCAwMrLJPkZGRiIiIkN7n5+czBBIREVGjU+cA+Ouvv8rCHwDo6urinXfeQdeuXWvdjpWVFZRKJXJycmTlOTk5sLW1rXHfwsJCxMfHyx7iAIC7d+9i1qxZ2Lx5s/RksLu7O9LS0rB06VLZ5eZ/8vb2BgBcunQJLi4usLOzAwDZQtfNmzeHlZUVrl27Vm2/9PX1oa+vX2PfiYiIiDStzpeAzczMqgxB169fh6mpaa3b0dPTg6enJ5KTk6UytVqN5ORk+Pj41Ljvxo0bUVxcLLtsCwClpaUoLS2Fjo78tJRKJdRqdbXtpaWlAYAU/Lp37w4ASE9Pl+rk5ubi1q1bUKlUDz85IiIiokaszjOAQ4cOxdixY7F06VL4+voCAA4cOIDp06dj+PDhdWorIiICISEh6Nq1K7y8vBAdHY3CwkKEhoYCAIKDg+Hg4IDFixfL9ouLi0NQUBAsLS1l5WZmZujVqxemT58OQ0NDqFQq7NmzB2vXrsXy5csBABkZGfjmm2/Qv39/WFpa4tSpU5g2bRp69uwJd3d3AECbNm3w0ksv4c0338QXX3wBMzMzREZGol27dnj++efrOmREREREjUqdA+DSpUuhUCgQHByMsrIyAECTJk0wceJEfPDBB3Vqa+jQobh58ybmzJmD7OxsdOrUCYmJidKDIdeuXas0m5eeno79+/cjKSmpyjbj4+MRGRmJkSNHIjc3FyqVCosWLZIWetbT08POnTulsOnk5IRBgwZh9uzZsnbWrl2LadOmYcCAAdDR0UGvXr2QmJiIJk2a1OkciYiIiBqbR14HsKioCBkZGQAAFxeXSgspE9cBJCIioienLrmjzjOAeXl5KC8vR7NmzeDm5iaV5+bmQldXl0GHiIiIqJGr80Mgw4YNQ3x8fKXy7777DsOGDauXThERERFRw6lzADx8+HCVD0L4+fnh8OHD9dIpIiIiImo4dQ6AxcXF0sMf/1RaWoq7d+/WS6eIiIiIqOHUOQB6eXnhiy++qFQeGxsLT0/PeukUERERETWcOj8EsnDhQgQEBODkyZPw9/cHACQnJ+Po0aPVLs1CRERERI1HnWcAu3fvjtTUVDg5OeG7777Djz/+iFatWuHUqVPo0aNHQ/SRiIiIiOrRI68DSA/HdQCJiIjoSalL7qjzDODx48fx22+/Se+3bt2KoKAgzJo1CyUlJXXvLRERERE9UXUOgOPHj8eFCxcAAJcvX8bQoUNhZGSEjRs34p133qn3DhIRERFR/apzALxw4QI6deoEANi4cSN69eqFb775BmvWrMEPP/xQ3/0jIiIionpW5wAohIBarQYA7Ny5E/379wcAODk54datW/XbOyIiIiKqd3UOgF27dsXChQvx9ddfY8+ePRgwYAAA4MqVK7Cxsan3DhIRERFR/apzAIyOjsbx48cRHh6Od999F61atQIAfP/99/D19a33DhIRERFR/aq3ZWDu3bsHpVKJJk2a1Edz/wpcBoaIiIielLrkjjp/E0h1DAwM6qspIiIiImpAdb4ETERERERPNwZAIiIiIi3DAEhERESkZRgAiYiIiLRMvQXA69evY8yYMfXVHBERERE1kHoLgLm5ufjqq6/qqzkiIiIiaiC1XgYmISGhxu2XL19+7M4QERERUcOrdQAMCgqCQqFATetGKxSKeukUERERETWcWl8CtrOzw6ZNm6BWq6t8HT9+vCH7SURERET1pNYB0NPTE8eOHat2+8NmB4mIiIiocaj1JeDp06ejsLCw2u2tWrVCSkpKvXSKiIiIiBqOQnDarsHU5UuZiYiIiB5HXXJHrS8BX758mZd4iYiIiP4Fah0AW7dujZs3b0rvhw4dipycnAbpFBERERE1nFoHwAdn/7Zt21bjPYF1sXLlSjg7O8PAwADe3t44cuRItXX9/PygUCgqvQYMGCDVKSgoQHh4OBwdHWFoaAhXV1fExsY+tJ0JEyZUeczbt2/D0dERCoUCd+7cqZdzJiIiItKUWj8E0lA2bNiAiIgIxMbGwtvbG9HR0QgMDER6ejqsra0r1d+0aRNKSkqk97dv34aHhwcGDx4slUVERGDXrl1Yt24dnJ2dkZSUhEmTJsHe3h4DBw6U6oWFhWH+/PnSeyMjoyr7OHbsWLi7uyMzM7M+TpmIiIhIo2o9A1gxS/Zg2eNavnw5wsLCEBoaKs3UGRkZ4csvv6yyfrNmzWBrayu9duzYASMjI1kAPHjwIEJCQuDn5wdnZ2eMGzcOHh4elWYWjYyMZG1VdcPkZ599hjt37uDtt99+7HMlIiIiagxqPQMohMDo0aOhr68PALh37x4mTJgAY2NjWb1NmzbV+uAlJSU4duwYIiMjpTIdHR0EBAQgNTW1Vm3ExcVh2LBhsn74+voiISEBY8aMgb29PXbv3o0LFy5gxYoVsn3Xr1+PdevWwdbWFi+++CLee+892Szg2bNnMX/+fBw+fLhWX3VXXFyM4uJi6X1+fn6tzoGIiIjoSap1AAwJCZG9HzVq1GMf/NatWygvL4eNjY2s3MbGBufPn3/o/keOHMHp06cRFxcnK4+JicG4cePg6OgIXV1d6OjoYNWqVejZs6dUZ8SIEVCpVLC3t8epU6cwY8YMpKenSwG2uLgYw4cPx5IlS9CiRYtaBcDFixdj3rx5tTl1IiIiIo2pdQBcvXp1Q/bjkcTFxcHNzQ1eXl6y8piYGBw6dAgJCQlQqVTYu3cvJk+eDHt7ewQEBAAAxo0bJ9V3c3ODnZ0d/P39kZGRARcXF0RGRqJ9+/Z1CrqRkZGIiIiQ3ufn58PJyekxz5KIiIiofmn0IRArKysolcpKy8nk5OTA1ta2xn0LCwsRHx8ve4gDAO7evYtZs2Zh8+bN0pPB7u7uSEtLw9KlS6UA+CBvb28AwKVLl+Di4oJdu3bht99+w/fffw/gf09BW1lZ4d13361ypk9fX1+6RE5ERETUWGk0AOrp6cHT0xPJyckICgoCAKjVaiQnJyM8PLzGfTdu3Iji4uJKM3SlpaUoLS2Fjo78+RalUgm1Wl1te2lpaQAAOzs7AMAPP/yAu3fvStuPHj2KMWPGYN++fXBxcantKRIRERE1OhpfBiYiIgIhISHo2rUrvLy8EB0djcLCQoSGhgIAgoOD4eDggMWLF8v2i4uLQ1BQECwtLWXlZmZm6NWrF6ZPnw5DQ0OoVCrs2bMHa9euxfLlywEAGRkZ+Oabb9C/f39YWlri1KlTmDZtGnr27Al3d3cAqBTybt26BQBo3749LCwsGmIoiIiIiJ4IjQfAoUOH4ubNm5gzZw6ys7PRqVMnJCYmSg+GXLt2rdJsXnp6Ovbv34+kpKQq24yPj0dkZCRGjhyJ3NxcqFQqLFq0SFroWU9PDzt37pTCppOTEwYNGoTZs2c37MkSERERNQIKwS/4bTB1+VJmIiIiosdRl9xR64WgiYiIiOjfgQGQiIiISMswABIRERFpGQZAIiIiIi3DAEhERESkZRgAiYiIiLQMAyARERGRlmEAJCIiItIyDIBEREREWoYBkIiIiEjLMAASERERaRkGQCIiIiItwwBIREREpGUYAImIiIi0DAMgERERkZZhACQiIiLSMgyARERERFqGAZCIiIhIyzAAEhEREWkZBkAiIiIiLcMASERERKRlGACJiIiItAwDIBEREZGWYQAkIiIi0jIMgERERERahgGQiIiISMswABIRERFpGQZAIiIiIi3DAEhERESkZRgAiYiIiLQMAyARERGRlmkUAXDlypVwdnaGgYEBvL29ceTIkWrr+vn5QaFQVHoNGDBAqlNQUIDw8HA4OjrC0NAQrq6uiI2NfWg7EyZMkLafPHkSw4cPh5OTEwwNDdG+fXt8/PHH9X/yRERERE+YrqY7sGHDBkRERCA2Nhbe3t6Ijo5GYGAg0tPTYW1tXan+pk2bUFJSIr2/ffs2PDw8MHjwYKksIiICu3btwrp16+Ds7IykpCRMmjQJ9vb2GDhwoFQvLCwM8+fPl94bGRlJfz527Bisra2xbt06ODk54eDBgxg3bhyUSiXCw8PrexiIiIiInhiFEEJosgPe3t7o1q0bPv30UwCAWq2Gk5MTpkyZgpkzZz50/+joaMyZMwdZWVkwNjYGAHTs2BFDhw7Fe++9J9Xz9PREv379sHDhQgD3ZwA7deqE6OjoWvd18uTJOHfuHHbt2lXl9uLiYhQXF0vv8/Pz4eTkhLy8PJiZmdX6OERERER1lZ+fD3Nz81rlDo1eAi4pKcGxY8cQEBAgleno6CAgIACpqam1aiMuLg7Dhg2Twh8A+Pr6IiEhAZmZmRBCICUlBRcuXECfPn1k+65fvx5WVlbo2LEjIiMjUVRUVOOx8vLy0KxZs2q3L168GObm5tLLycmpVudARERE9CRp9BLwrVu3UF5eDhsbG1m5jY0Nzp8//9D9jxw5gtOnTyMuLk5WHhMTg3HjxsHR0RG6urrQ0dHBqlWr0LNnT6nOiBEjoFKpYG9vj1OnTmHGjBlIT0/Hpk2bqjzWwYMHsWHDBvz888/V9icyMhIRERHS+4oZQCIiIqLGROP3AD6OuLg4uLm5wcvLS1YeExODQ4cOISEhASqVCnv37sXkyZNhb28vzTaOGzdOqu/m5gY7Ozv4+/sjIyMDLi4usvZOnz6Nl156CVFRUZVmEf9JX18f+vr69XiGRERERPVPowHQysoKSqUSOTk5svKcnBzY2trWuG9hYSHi4+NlD3EAwN27dzFr1ixs3rxZejLY3d0daWlpWLp0qexy8z95e3sDAC5duiQLgGfPnoW/vz/GjRuH2bNn1/kciYiIiBobjd4DqKenB09PTyQnJ0tlarUaycnJ8PHxqXHfjRs3ori4GKNGjZKVl5aWorS0FDo68lNTKpVQq9XVtpeWlgYAsLOzk8rOnDmD559/HiEhIVi0aFFtT4uIiIioUdP4JeCIiAiEhISga9eu8PLyQnR0NAoLCxEaGgoACA4OhoODAxYvXizbLy4uDkFBQbC0tJSVm5mZoVevXpg+fToMDQ2hUqmwZ88erF27FsuXLwcAZGRk4JtvvkH//v1haWmJU6dOYdq0aejZsyfc3d0B3L/s+8ILLyAwMBARERHIzs4GcD9INm/evKGHhYiIiKjBaDwADh06FDdv3sScOXOQnZ2NTp06ITExUXow5Nq1a5Vm89LT07F//34kJSVV2WZ8fDwiIyMxcuRI5ObmQqVSYdGiRdJCz3p6eti5c6cUNp2cnDBo0CDZJd7vv/8eN2/exLp167Bu3TqpXKVS4erVq/U8CkRERERPjsbXAfw3q8t6PERERESP46lZB5CIiIiInjwGQCIiIiItwwBIREREpGUYAImIiIi0DAMgERERkZZhACQiIiLSMgyARERERFqGAZCIiIhIyzAAEhEREWkZBkAiIiIiLcMASERERKRlGACJiIiItAwDIBEREZGWYQAkIiIi0jIMgERERERahgGQiIiISMswABIRERFpGQZAIiIiIi3DAEhERESkZRgAiYiIiLQMAyARERGRlmEAJCIiItIyDIBEREREWoYBkIiIiEjLMAASERERaRkGQCIiIiItwwBIREREpGUYAImIiIi0DAMgERERkZZhACQiIiLSMo0iAK5cuRLOzs4wMDCAt7c3jhw5Um1dPz8/KBSKSq8BAwZIdQoKChAeHg5HR0cYGhrC1dUVsbGxD21nwoQJsjrXrl3DgAEDYGRkBGtra0yfPh1lZWX1e/JERERET5iupjuwYcMGREREIDY2Ft7e3oiOjkZgYCDS09NhbW1dqf6mTZtQUlIivb99+zY8PDwwePBgqSwiIgK7du3CunXr4OzsjKSkJEyaNAn29vYYOHCgVC8sLAzz58+X3hsZGUl/Li8vx4ABA2Bra4uDBw8iKysLwcHBaNKkCd5///36HgYiIiKiJ0bjM4DLly9HWFgYQkNDpZk6IyMjfPnll1XWb9asGWxtbaXXjh07YGRkJAuABw8eREhICPz8/ODs7Ixx48bBw8Oj0syikZGRrC0zMzNpW1JSEs6ePYt169ahU6dO6NevHxYsWICVK1fKAigRERHR00ajAbCkpATHjh1DQECAVKajo4OAgACkpqbWqo24uDgMGzYMxsbGUpmvry8SEhKQmZkJIQRSUlJw4cIF9OnTR7bv+vXrYWVlhY4dOyIyMhJFRUXSttTUVLi5ucHGxkYqCwwMRH5+Ps6cOVNlX4qLi5Gfny97ERERETU2Gr0EfOvWLZSXl8tCFgDY2Njg/PnzD93/yJEjOH36NOLi4mTlMTExGDduHBwdHaGrqwsdHR2sWrUKPXv2lOqMGDECKpUK9vb2OHXqFGbMmIH09HRs2rQJAJCdnV1lvyq2VWXx4sWYN2/ew0+ciIiISIM0fg/g44iLi4Obmxu8vLxk5TExMTh06BASEhKgUqmwd+9eTJ48Gfb29tJs47hx46T6bm5usLOzg7+/PzIyMuDi4vJI/YmMjERERIT0Pj8/H05OTo/UFhEREVFD0WgAtLKyglKpRE5Ojqw8JycHtra2Ne5bWFiI+Ph42UMcAHD37l3MmjULmzdvlp4Mdnd3R1paGpYuXSq73PxP3t7eAIBLly7BxcUFtra2le4ZrOhndX3T19eHvr5+jf0mIiIi0jSN3gOop6cHT09PJCcnS2VqtRrJycnw8fGpcd+NGzeiuLgYo0aNkpWXlpaitLQUOjryU1MqlVCr1dW2l5aWBgCws7MDAPj4+OC3337DjRs3pDo7duyAmZkZXF1da3V+RERERI2Rxi8BR0REICQkBF27doWXlxeio6NRWFiI0NBQAEBwcDAcHBywePFi2X5xcXEICgqCpaWlrNzMzAy9evXC9OnTYWhoCJVKhT179mDt2rVYvnw5ACAjIwPffPMN+vfvD0tLS5w6dQrTpk1Dz5494e7uDgDo06cPXF1d8dprr+Gjjz5CdnY2Zs+ejcmTJ3OWj4iIiJ5qGg+AQ4cOxc2bNzFnzhxkZ2ejU6dOSExMlB64uHbtWqXZvPT0dOzfvx9JSUlVthkfH4/IyEiMHDkSubm5UKlUWLRokbTQs56eHnbu3CmFTScnJwwaNAizZ8+W2lAqlfjpp58wceJE+Pj4wNjYGCEhIZUuORMRERE9bRRCCKHpTvxb5efnw9zcHHl5ebI1BomIiIjqW11yh8YXgiYiIiKiJ4sBkIiIiEjLMAASERERaRkGQCIiIiItwwBIREREpGUYAImIiIi0DAMgERERkZZhACQiIiLSMgyARERERFqGAZCIiIhIyzAAEhEREWkZBkAiIiIiLcMASERERKRlGACJiIiItAwDIBEREZGWYQAkIiIi0jK6mu4AERER0b+auhz4/SBQkAOY2AAqX0BHqdEuMQASERERNZSzCUDiDCD/z/+VmdkDfT8EXAdqrFu8BExERETUEM4mAN8Fy8MfAORn3S8/m6CZfoEBkIiIiKj+qcvvz/xBVLHx/5clzrxfTwMYAImIiIjq2+8HK8/8yQggP/N+PQ1gACQiIiKqbwU59VuvnjEAEhEREdU3E5v6rVfPGACJiIiI6pvK9/7TvlBUU0EBmDncr6cBDIBERERE9U1HeX+pFwCVQ+D/f9/3A42tB8gASERERNQQXAcCQ9YCZnbycjP7++UaXAeQC0ETERERNRTXgUC7AfwmECIiIiKtoqMEWvbQdC9keAmYiIiISMswABIRERFpGY0HwJUrV8LZ2RkGBgbw9vbGkSNHqq3r5+cHhUJR6TVgwACpTkFBAcLDw+Ho6AhDQ0O4uroiNja2yvaEEOjXrx8UCgW2bNki23b06FH4+/vDwsICTZs2RWBgIE6ePFkv50xERESkSRoNgBs2bEBERASioqJw/PhxeHh4IDAwEDdu3Kiy/qZNm5CVlSW9Tp8+DaVSicGDB0t1IiIikJiYiHXr1uHcuXOYOnUqwsPDkZBQ+QuXo6OjoVBUXp+noKAAffv2RYsWLXD48GHs378fpqamCAwMRGlpaf0NABEREZEGaDQALl++HGFhYQgNDZVm6oyMjPDll19WWb9Zs2awtbWVXjt27ICRkZEsAB48eBAhISHw8/ODs7Mzxo0bBw8Pj0ozi2lpaVi2bFmVxzp//jxyc3Mxf/58tG3bFh06dEBUVBRycnLw+++/1+8gEBERET1hGguAJSUlOHbsGAICAv7XGR0dBAQEIDU1tVZtxMXFYdiwYTA2NpbKfH19kZCQgMzMTAghkJKSggsXLqBPnz5SnaKiIowYMQIrV66Era1tpXbbtm0LS0tLxMXFoaSkBHfv3kVcXBzat28PZ2fnavtTXFyM/Px82YuIiIiosdHYMjC3bt1CeXk5bGzk34FnY2OD8+fPP3T/I0eO4PTp04iLi5OVx8TEYNy4cXB0dISuri50dHSwatUq9OzZU6ozbdo0+Pr64qWXXqqybVNTU+zevRtBQUFYsGABAKB169bYvn07dHWrH7LFixdj3rx5lcoZBImIiKihVeQNIcRD6z616wDGxcXBzc0NXl5esvKYmBgcOnQICQkJUKlU2Lt3LyZPngx7e3sEBAQgISEBu3btwokTJ6pt++7duxg7diy6d++Ob7/9FuXl5Vi6dCkGDBiAo0ePwtDQsMr9IiMjERERIb3PzMyEq6srnJyc6uekiYiIiB7i77//hrm5eY11NBYAraysoFQqkZOTIyvPycmp8rLsPxUWFiI+Ph7z58+Xld+9exezZs3C5s2bpSeD3d3dkZaWhqVLlyIgIAC7du1CRkYGLCwsZPsOGjQIPXr0wO7du/HNN9/g6tWrSE1NhY7O/avk33zzDZo2bYqtW7di2LBhVfZLX18f+vr60nsTExNcv34dpqamVT5sUp38/Hw4OTnh+vXrMDMzq/V+9Pg49prBcdcMjrvmcOw1498+7kII/P3337C3t39oXY0FQD09PXh6eiI5ORlBQUEAALVajeTkZISHh9e478aNG1FcXIxRo0bJyktLS1FaWiqFtgpKpRJqtRoAMHPmTLz++uuy7W5ublixYgVefPFFAPfvEdTR0ZGFtor3Fe3Uho6ODhwdHWtd/0FmZmb/yg/o04Bjrxkcd83guGsOx14z/s3j/rCZvwoavQQcERGBkJAQdO3aFV5eXoiOjkZhYSFCQ0MBAMHBwXBwcMDixYtl+8XFxSEoKAiWlpaycjMzM/Tq1QvTp0+HoaEhVCoV9uzZg7Vr12L58uUAID1B/KAWLVqgZcuWAIDevXtj+vTpmDx5MqZMmQK1Wo0PPvgAurq6eP755xtiKIiIiIieGI0GwKFDh+LmzZuYM2cOsrOz0alTJyQmJkoPhly7dq3SbF56ejr279+PpKSkKtuMj49HZGQkRo4cidzcXKhUKixatAgTJkyodb/atWuHH3/8EfPmzYOPjw90dHTQuXNnJCYmws7O7tFPmIiIiKgR0PhDIOHh4dVe8t29e3elsrZt29b4dIutrS1Wr15dpz5U1V7v3r3Ru3fvOrVTX/T19REVFSW7n5CeDI69ZnDcNYPjrjkce83guP+PQtTmWWEiIiIi+tfQ+HcBExEREdGTxQBIREREpGUYAImIiIi0DAMgERERkZZhAGwgK1euhLOzMwwMDODt7Y0jR47UWH/jxo1o164dDAwM4Obmhm3btsm2CyEwZ84c2NnZwdDQEAEBAbh48aKsTm5uLkaOHAkzMzNYWFhg7NixKCgoqPdza8w0Me6LFi2Cr68vjIyMKn3DjLZ40uN+9epVjB07Fi1btoShoSFcXFwQFRWFkpKSBjm/xkwTn/mBAweiRYsWMDAwgJ2dHV577TX8+eef9X5ujZkmxr1CcXExOnXqBIVCgbS0tPo6paeCJsbd2dkZCoVC9vrggw/q/dyeOEH1Lj4+Xujp6Ykvv/xSnDlzRoSFhQkLCwuRk5NTZf0DBw4IpVIpPvroI3H27Fkxe/Zs0aRJE/Hbb79JdT744ANhbm4utmzZIk6ePCkGDhwoWrZsKe7evSvV6du3r/Dw8BCHDh0S+/btE61atRLDhw9v8PNtLDQ17nPmzBHLly8XERERwtzcvKFPs9HRxLj/8ssvYvTo0WL79u0iIyNDbN26VVhbW4u33nrriZxzY6Gpz/zy5ctFamqquHr1qjhw4IDw8fERPj4+DX6+jYWmxr3CG2+8Ifr16ycAiBMnTjTUaTY6mhp3lUol5s+fL7KysqRXQUFBg59vQ2MAbABeXl5i8uTJ0vvy8nJhb28vFi9eXGX9IUOGiAEDBsjKvL29xfjx44UQQqjVamFrayuWLFkibb9z547Q19cX3377rRBCiLNnzwoA4ujRo1KdX375RSgUCpGZmVlv59aYaWLc/2n16tVaGQA1Pe4VPvroI9GyZcvHOZWnTmMZ+61btwqFQiFKSkoe53SeGpoc923btol27dqJM2fOaF0A1NS4q1QqsWLFino8k8aBl4DrWUlJCY4dO4aAgACpTEdHBwEBAUhNTa1yn9TUVFl9AAgMDJTqX7lyBdnZ2bI65ubm8Pb2luqkpqbCwsICXbt2leoEBARAR0cHhw8frrfza6w0Ne7arjGNe15eHpo1a/Y4p/NUaSxjn5ubi/Xr18PX1xdNmjR53NNq9DQ57jk5OQgLC8PXX38NIyOj+jytRk/Tn/cPPvgAlpaW6Ny5M5YsWYKysrL6OjWNYQCsZ7du3UJ5ebn0dXYVbGxskJ2dXeU+2dnZNdav+O/D6lhbW8u26+rqolmzZtUe999EU+Ou7RrLuF+6dAkxMTEYP378I53H00jTYz9jxgwYGxvD0tIS165dw9atWx/rfJ4Wmhp3IQRGjx6NCRMmyP6hry00+Xl/4403EB8fj5SUFIwfPx7vv/8+3nnnncc+J01jACSip1pmZib69u2LwYMHIywsTNPd0RrTp0/HiRMnkJSUBKVSieDg4Bq/ppMeT0xMDP7++29ERkZquitaJyIiAn5+fnB3d8eECROwbNkyxMTEoLi4WNNdeywMgPXMysoKSqUSOTk5svKcnBzY2tpWuY+trW2N9Sv++7A6N27ckG0vKytDbm5utcf9N9HUuGs7TY/7n3/+ieeffx6+vr744osvHutcnjaaHnsrKyu0adMGvXv3Rnx8PLZt24ZDhw491jk9DTQ17rt27UJqair09fWhq6uLVq1aAQC6du2KkJCQxz+xRk7Tn/d/8vb2RllZGa5evVrX02hUGADrmZ6eHjw9PZGcnCyVqdVqJCcnw8fHp8p9fHx8ZPUBYMeOHVL9li1bwtbWVlYnPz8fhw8flur4+Pjgzp07OHbsmFRn165dUKvV8Pb2rrfza6w0Ne7aTpPjnpmZCT8/P3h6emL16tXQ0dGuv84a02derVYDwFM/I1Ibmhr3Tz75BCdPnkRaWhrS0tKk5Uw2bNiARYsW1es5NkaN6fOelpYGHR2dSrddPXU0/RTKv1F8fLzQ19cXa9asEWfPnhXjxo0TFhYWIjs7WwghxGuvvSZmzpwp1T9w4IDQ1dUVS5cuFefOnRNRUVFVPqpuYWEhtm7dKk6dOiVeeumlKpeB6dy5szh8+LDYv3+/aN26tdYtA6OJcf/999/FiRMnxLx584SJiYk4ceKEOHHihPj777+f3MlrkCbG/Y8//hCtWrUS/v7+4o8//pAtz6BNNDH2hw4dEjExMeLEiRPi6tWrIjk5Wfj6+goXFxdx7969JzsAGqKpv2v+6cqVK1r3FLAmxv3gwYNixYoVIi0tTWRkZIh169aJ5s2bi+Dg4Cd78g2AAbCBxMTEiBYtWgg9PT3h5eUlDh06JG3r1auXCAkJkdX/7rvvRJs2bYSenp7o0KGD+Pnnn2Xb1Wq1eO+994SNjY3Q19cX/v7+Ij09XVbn9u3bYvjw4cLExESYmZmJ0NBQrQkhFTQx7iEhIQJApVdKSkpDnWaj86THffXq1VWOuTb+m/ZJj/2pU6fE888/L5o1ayb09fWFs7OzmDBhgvjjjz8a9DwbG038XfNP2hgAhXjy437s2DHh7e0tzM3NhYGBgWjfvr14//33/xX/2FEIwbt2iYiIiLSJdt00Q0REREQMgERERETahgGQiIiISMswABIRERFpGQZAIiIiIi3DAEhERESkZRgAiYiIiLQMAyARERGRlmEAJKKnxujRoxEUFKTpbvzrKBQKbNmyBQBw9epVKBQKpKWlPXJ79dEGETUsXU13gIgIuB9CahIVFYWPP/4Y/PKihuXk5ISsrCxYWVnVqv7o0aNx584dKUA+ShtE9OQxABJRo5CVlSX9ecOGDZgzZw7S09OlMhMTE5iYmGiiaw2mtLQUTZo0aVRtKZVK2NraarwNImpYvARMRI2Cra2t9DI3N4dCoZCVmZiYVLoE7OfnhylTpmDq1Klo2rQpbGxssGrVKhQWFiI0NBSmpqZo1aoVfvnlF9mxTp8+jX79+sHExAQ2NjZ47bXXcOvWrWr7tmbNGlhYWGDLli1o3bo1DAwMEBgYiOvXr8vqbd26FV26dIGBgQGeeeYZzJs3D2VlZdJ2hUKBzz77DAMHDoSxsTEWLVpU5fGcnZ2xYMECDB8+HMbGxnBwcMDKlStldapr62F9uHjxInr27AkDAwO4urpix44dsnarunx75swZ/Oc//4GZmRlMTU3Ro0cPZGRkYO7cufjqq6+wdetWKBQKKBQK7N69u8o29uzZAy8vL+jr68POzg4zZ86U9cvPzw9vvPEG3nnnHTRr1gy2traYO3dutT8TIno8DIBE9FT76quvYGVlhSNHjmDKlCmYOHEiBg8eDF9fXxw/fhx9+vTBa6+9hqKiIgDAnTt38MILL6Bz58749ddfkZiYiJycHAwZMqTG4xQVFWHRokVYu3YtDhw4gDt37mDYsGHS9n379iE4OBhvvvkmzp49i88//xxr1qypFPLmzp2Ll19+Gb/99hvGjBlT7fGWLFkCDw8PnDhxAjNnzsSbb75ZKaw92NbD+qBWq/HKK69AT08Phw8fRmxsLGbMmFHjeWdmZqJnz57Q19fHrl27cOzYMYwZMwZlZWV4++23MWTIEPTt2xdZWVnIysqCr69vlW30798f3bp1w8mTJ/HZZ58hLi4OCxculNX76quvYGxsjMOHD+Ojjz7C/PnzK50zEdUTQUTUyKxevVqYm5tXKg8JCREvvfSS9L5Xr17iueeek96XlZUJY2Nj8dprr0llWVlZAoBITU0VQgixYMEC0adPH1m7169fFwBEenp6tf0BIA4dOiSVnTt3TgAQhw8fFkII4e/vL95//33Zfl9//bWws7OT3gMQU6dOfcjZC6FSqUTfvn1lZUOHDhX9+vWrsa2H9WH79u1CV1dXZGZmStt/+eUXAUBs3rxZCCHElStXBABx4sQJIYQQkZGRomXLlqKkpKTKvj74M6mqjVmzZom2bdsKtVot1Vm5cqUwMTER5eXlQojKP0shhOjWrZuYMWNGlcclosfDewCJ6Knm7u4u/VmpVMLS0hJubm5SmY2NDQDgxo0bAICTJ08iJSWlyvsJMzIy0KZNmyqPo6uri27duknv27VrBwsLC5w7dw5eXl44efIkDhw4IJvxKy8vx71791BUVAQjIyMAQNeuXWt1Xj4+PpXeR0dHy8oebOthfTh37hycnJxgb29f7XEelJaWhh49ejzW/YXnzp2Dj4+P7EGf7t27o6CgAH/88QdatGgBQP6zBAA7Ozvp50ZE9YsBkIieag8GE4VCISurCB1qtRoAUFBQgBdffBEffvhhpbbs7OweuR8FBQWYN28eXnnllUrbDAwMpD8bGxs/8jEe9GBbte1DXRgaGj7Sfo+iqp9lxc+NiOoXAyARaZUuXbrghx9+gLOzM3R1a/9XYFlZGX799Vd4eXkBANLT03Hnzh20b99eajc9PR2tWrWql34eOnSo0vuKY1XnYX1o3749rl+/jqysLCnsPnicB7m7u+Orr76q9iljPT09lJeX19hG+/bt8cMPP0AIIQXyAwcOwNTUFI6OjjXuS0QNgw+BEJFWmTx5MnJzczF8+HAcPXoUGRkZ2L59O0JDQ2sMMk2aNMGUKVNw+PBhHDt2DKNHj8azzz4rBcI5c+Zg7dq1mDdvHs6cOYNz584hPj4es2fPfqR+HjhwAB999BEuXLiAlStXYuPGjXjzzTdr3OdhfQgICECbNm0QEhKCkydPYt++fXj33XdrbDM8PBz5+fkYNmwYfv31V1y8eBFff/21tESPs7MzTp06hfT0dNy6dQulpaWV2pg0aRKuX7+OKVOm4Pz589i6dSuioqIQEREBHR3+b4hIE/ibR0Raxd7eHgcOHEB5eTn69OkDNzc3TJ06FRYWFjWGESMjI8yYMQMjRoxA9+7dYWJigg0bNkjbAwMD8dNPPyEpKQndunXDs88+ixUrVkClUj1SP9966y38+uuv6Ny5MxYuXIjly5cjMDCwxn0e1gcdHR1s3rwZd+/ehZeXF15//fVql6KpYGlpiV27dqGgoAC9evWCp6cnVq1aJc0GhoWFoW3btujatSuaN2+OAwcOVGrDwcEB27Ztw5EjR+Dh4YEJEyZg7NixjxyOiejxKYTgsvpERDVZs2YNpk6dijt37jyR4zk7O2Pq1KmYOnXqEzkeEWkfzgASERERaRkGQCIiIiItw0vARERERFqGM4BEREREWoYBkIiIiEjLMAASERERaRkGQCIiIiItwwBIREREpGUYAImIiIi0DAMgERERkZZhACQiIiLSMv8PXj8evJrhjUAAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 700x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.figure(figsize=(7,4))\n",
    "plt.scatter(baseline_time[1],baseline_results[\"f1\"],label=\"baseline\")\n",
    "plt.scatter(model_6_pretrained_time[1], model_6_pretrained_results[\"f1\"],label=\"TfHubSentenceEncoder\")\n",
    "plt.legend()\n",
    "plt.ylabel(\"F1 score\")\n",
    "plt.xlabel(\"Time per prediction\")\n",
    "plt.title(\"F1-Score versus time per prediction\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "  3/215 [..............................] - ETA: 16s - loss: 0.6927 - accuracy: 0.5000"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
      "I0000 00:00:1703430154.149104   67365 device_compiler.h:186] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "215/215 [==============================] - 8s 30ms/step - loss: 0.6099 - accuracy: 0.6922 - val_loss: 0.5360 - val_accuracy: 0.7559\n",
      "Epoch 2/5\n",
      "215/215 [==============================] - 2s 10ms/step - loss: 0.4413 - accuracy: 0.8193 - val_loss: 0.4690 - val_accuracy: 0.7835\n",
      "Epoch 3/5\n",
      "215/215 [==============================] - 2s 11ms/step - loss: 0.3466 - accuracy: 0.8608 - val_loss: 0.4587 - val_accuracy: 0.7887\n",
      "Epoch 4/5\n",
      "215/215 [==============================] - 2s 9ms/step - loss: 0.2851 - accuracy: 0.8914 - val_loss: 0.4637 - val_accuracy: 0.7900\n",
      "Epoch 5/5\n",
      "215/215 [==============================] - 2s 8ms/step - loss: 0.2382 - accuracy: 0.9121 - val_loss: 0.4762 - val_accuracy: 0.7861\n",
      "24/24 [==============================] - 0s 2ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 78.60892388451444,\n",
       " 'precision': 0.7903277546022673,\n",
       " 'recall': 0.7860892388451444,\n",
       " 'f1': 0.7832971347503846}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#1. Rebuild, compile and train model_1, model_2 and model_5 using the Keras Sequential API instead of the Functional API.\n",
    "#1.a\n",
    "model_1_exercise = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.Input(shape=(1,), dtype=tf.string),\n",
    "    text_vectorizer,\n",
    "    embedding,\n",
    "    tf.keras.layers.GlobalAveragePooling1D(),\n",
    "    tf.keras.layers.Dense(1, activation=tf.keras.activations.sigmoid)\n",
    "])\n",
    "model_1_exercise.compile(loss=tf.keras.losses.BinaryCrossentropy(),\n",
    "                        optimizer=tf.keras.optimizers.Adam(),\n",
    "                        metrics=['accuracy'])\n",
    "history_1_exercise = model_1_exercise.fit(train_sentences, train_labels,\n",
    "                                     epochs=5,\n",
    "                                     validation_data=(val_sentences, val_labels))\n",
    "model_1_preds_exercise = model_1_exercise.predict(val_sentences)\n",
    "calculate_results(val_labels, tf.squeeze(tf.round(model_1_preds_exercise)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "215/215 [==============================] - 15s 54ms/step - loss: 0.2238 - accuracy: 0.9165 - val_loss: 0.5366 - val_accuracy: 0.7795\n",
      "Epoch 2/5\n",
      "215/215 [==============================] - 5s 21ms/step - loss: 0.1668 - accuracy: 0.9381 - val_loss: 0.5980 - val_accuracy: 0.7769\n",
      "Epoch 3/5\n",
      "215/215 [==============================] - 4s 20ms/step - loss: 0.1317 - accuracy: 0.9515 - val_loss: 0.7259 - val_accuracy: 0.7677\n",
      "Epoch 4/5\n",
      "215/215 [==============================] - 4s 19ms/step - loss: 0.1102 - accuracy: 0.9590 - val_loss: 0.8589 - val_accuracy: 0.7848\n",
      "Epoch 5/5\n",
      "215/215 [==============================] - 4s 19ms/step - loss: 0.0821 - accuracy: 0.9651 - val_loss: 0.9306 - val_accuracy: 0.7808\n",
      "24/24 [==============================] - 1s 7ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 78.08398950131233,\n",
       " 'precision': 0.7828610537338687,\n",
       " 'recall': 0.7808398950131233,\n",
       " 'f1': 0.7787902197895524}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#1.b\n",
    "model_2_exercise = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.Input(shape=(1,), dtype=tf.string),\n",
    "    text_vectorizer,\n",
    "    embedding,\n",
    "    tf.keras.layers.LSTM(64, return_sequences=True),\n",
    "    tf.keras.layers.LSTM(64, return_sequences=True),\n",
    "    tf.keras.layers.LSTM(64),\n",
    "    tf.keras.layers.Dense(1, activation=tf.keras.activations.sigmoid)\n",
    "])\n",
    "model_2_exercise.compile(loss=tf.keras.losses.BinaryCrossentropy(),\n",
    "                        optimizer=tf.keras.optimizers.Adam(),\n",
    "                        metrics=['accuracy'])\n",
    "history_2_exercise = model_2_exercise.fit(train_sentences, train_labels,\n",
    "                                     epochs=5,\n",
    "                                     validation_data=(val_sentences, val_labels))\n",
    "model_2_preds_exercise = model_2_exercise.predict(val_sentences)\n",
    "calculate_results(val_labels, tf.squeeze(tf.round(model_2_preds_exercise)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "215/215 [==============================] - 13s 44ms/step - loss: 0.0741 - accuracy: 0.9707 - val_loss: 0.8903 - val_accuracy: 0.7730\n",
      "Epoch 2/5\n",
      "215/215 [==============================] - 4s 20ms/step - loss: 0.0577 - accuracy: 0.9746 - val_loss: 1.3991 - val_accuracy: 0.7769\n",
      "Epoch 3/5\n",
      "215/215 [==============================] - 4s 20ms/step - loss: 0.0491 - accuracy: 0.9768 - val_loss: 1.6260 - val_accuracy: 0.7638\n",
      "Epoch 4/5\n",
      "215/215 [==============================] - 4s 19ms/step - loss: 0.0464 - accuracy: 0.9781 - val_loss: 1.3402 - val_accuracy: 0.7756\n",
      "Epoch 5/5\n",
      "215/215 [==============================] - 4s 19ms/step - loss: 0.0411 - accuracy: 0.9806 - val_loss: 1.5429 - val_accuracy: 0.7677\n",
      "24/24 [==============================] - 1s 8ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 76.77165354330708,\n",
       " 'precision': 0.7681410880728078,\n",
       " 'recall': 0.7677165354330708,\n",
       " 'f1': 0.7662770891654436}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#1.c\n",
    "model_5_exercise = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.Input(shape=(1,), dtype=tf.string),\n",
    "    text_vectorizer,\n",
    "    embedding,\n",
    "    tf.keras.layers.Conv1D(filters=64, kernel_size=5, padding='same'),\n",
    "    tf.keras.layers.GlobalAveragePooling1D(),\n",
    "    tf.keras.layers.Dense(1, activation=tf.keras.activations.sigmoid)\n",
    "])\n",
    "model_2_exercise.compile(loss=tf.keras.losses.BinaryCrossentropy(),\n",
    "                        optimizer=tf.keras.optimizers.Adam(),\n",
    "                        metrics=['accuracy'])\n",
    "history_2_exercise = model_2_exercise.fit(train_sentences, train_labels,\n",
    "                                     epochs=5,\n",
    "                                     validation_data=(val_sentences, val_labels))\n",
    "model_2_preds_exercise = model_2_exercise.predict(val_sentences)\n",
    "calculate_results(val_labels, tf.squeeze(tf.round(model_2_preds_exercise)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "70.20997375328083"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#2. Retrain the baseline model with 10% of the training data. How does perform \n",
    "# compared to the Universal Sentence Encoder model with 10% of the training data?\n",
    "train_sentences_10_percent = train_sentences[:int(0.1*len(train_sentences))]\n",
    "train_labels_10_percent = train_labels[:int(0.1*len(train_labels))]\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "# Create tokenization and modelling pipeline\n",
    "model_0 = Pipeline([\n",
    "                    (\"tfidf\", TfidfVectorizer()), # convert words to numbers using tfidf\n",
    "                    (\"clf\", MultinomialNB()) # model the text\n",
    "])\n",
    "\n",
    "# Fit the pipeline to the training data\n",
    "model_0.fit(train_sentences_10_percent, train_labels_10_percent)\n",
    "model_0.score(val_sentences, val_labels)*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
      "I0000 00:00:1703425067.700393   40160 device_compiler.h:186] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "215/215 [==============================] - 103s 396ms/step - loss: 0.4555 - accuracy: 0.7986 - val_loss: 0.4411 - val_accuracy: 0.8150\n",
      "24/24 [==============================] - 1s 22ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accuracy': 81.49606299212599,\n",
       " 'precision': 0.8153632818598993,\n",
       " 'recall': 0.8149606299212598,\n",
       " 'f1': 0.8150930383740584}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#3. Try fine-tuning the TF Hub Universal Sentence Encoder model by setting training=True when instantiating it as a Keras layer.\n",
    "sentence_encoder_layer = hub.KerasLayer(embed,\n",
    "                                        input_shape=[],\n",
    "                                        dtype=tf.string,\n",
    "                                        trainable=True) #model is trainable now\n",
    "model_fine_tune_exercise = tf.keras.models.Sequential([\n",
    "    sentence_encoder_layer,\n",
    "    tf.keras.layers.Dense(64, activation=tf.keras.activations.relu),\n",
    "    tf.keras.layers.Dense(64, activation=tf.keras.activations.relu),\n",
    "    tf.keras.layers.Dense(1, activation=tf.keras.activations.sigmoid)\n",
    "])\n",
    "model_fine_tune_exercise.compile(loss=tf.keras.losses.BinaryCrossentropy(),\n",
    "                        optimizer=tf.keras.optimizers.Adam(),\n",
    "                        metrics=['accuracy'])\n",
    "history_fine_tune_exercise = model_fine_tune_exercise.fit(train_sentences, train_labels,\n",
    "                                     epochs=1,\n",
    "                                     validation_data=(val_sentences, val_labels))\n",
    "model_fine_tune_preds_exercise = model_fine_tune_exercise.predict(val_sentences)\n",
    "calculate_results(val_labels, tf.squeeze(tf.round(model_fine_tune_preds_exercise)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "238/238 [==============================] - 6s 20ms/step - loss: 0.4717 - accuracy: 0.7908\n",
      "Epoch 2/10\n",
      "238/238 [==============================] - 5s 19ms/step - loss: 0.3938 - accuracy: 0.8291\n",
      "Epoch 3/10\n",
      "238/238 [==============================] - 4s 19ms/step - loss: 0.3661 - accuracy: 0.8428\n",
      "Epoch 4/10\n",
      "238/238 [==============================] - 4s 19ms/step - loss: 0.3336 - accuracy: 0.8614\n",
      "Epoch 5/10\n",
      "238/238 [==============================] - 4s 19ms/step - loss: 0.2943 - accuracy: 0.8824\n",
      "Epoch 6/10\n",
      "238/238 [==============================] - 5s 19ms/step - loss: 0.2502 - accuracy: 0.9033\n",
      "Epoch 7/10\n",
      "238/238 [==============================] - 5s 19ms/step - loss: 0.2054 - accuracy: 0.9243\n",
      "Epoch 8/10\n",
      "238/238 [==============================] - 5s 19ms/step - loss: 0.1591 - accuracy: 0.9460\n",
      "Epoch 9/10\n",
      "238/238 [==============================] - 5s 19ms/step - loss: 0.1255 - accuracy: 0.9594\n",
      "Epoch 10/10\n",
      "238/238 [==============================] - 4s 19ms/step - loss: 0.1050 - accuracy: 0.9647\n",
      "102/102 [==============================] - 2s 16ms/step\n"
     ]
    }
   ],
   "source": [
    "#4 Retrain the best model you've got so far on the whole training set (no validation split). Then use this trained model to make predictions on the test dataset and format the predictions into the same format as the sample_submission.csv\n",
    "# Challenge, beat the baseline\n",
    "model_best_exercise = tf.keras.models.Sequential([\n",
    "    sentence_encoder_layer,\n",
    "    tf.keras.layers.Dense(128,activation=tf.keras.activations.relu),\n",
    "    tf.keras.layers.Dense(64,activation=tf.keras.activations.relu),\n",
    "    tf.keras.layers.Dense(1,activation=tf.keras.activations.sigmoid)\n",
    "])\n",
    "model_best_exercise.compile(loss=tf.keras.losses.BinaryCrossentropy(),\n",
    "                    optimizer=tf.keras.optimizers.Adam(),\n",
    "                    metrics=['accuracy'])\n",
    "model_best_exercise_history = model_best_exercise.fit(train_df_shuffled.text, train_df_shuffled.target,\n",
    "                                      epochs=10)\n",
    "\n",
    "model_best_exercise_pred_probs = model_best_exercise.predict(test_df.text)\n",
    "model_best_exercise_preds = tf.squeeze(tf.round(model_best_exercise_pred_probs))\n",
    "model_best_exercise_preds = [int(pred) for pred in model_best_exercise_preds.numpy()]\n",
    "test_df[\"target\"] = model_best_exercise_preds\n",
    "submission = test_df[[\"id\", \"target\"]]\n",
    "submission.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 79.79002624671917,\n",
       " 'precision': 0.798940036341611,\n",
       " 'recall': 0.7979002624671916,\n",
       " 'f1': 0.7965988626421697}"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#5. Combine the ensemble predictions using the majority vote (mode), how does this perform compare \n",
    "# to averaging the prediction probabilities of each model?\n",
    "#ensemble majority voting - take the preds which has max support\n",
    "models=[model_1_preds, model_2_preds, model_3_preds, model_4_preds, model_5_preds, model_6_preds]\n",
    "ensemble_preds_exercise=[]\n",
    "for i in range(len(model_1_preds)):\n",
    "    class0Count, class1Count = 0, 0\n",
    "    for model in models:\n",
    "        if model[i]==0:\n",
    "            class0Count+=1\n",
    "        elif model[i]==1:\n",
    "            class1Count+=1\n",
    "    if class0Count>class1Count:\n",
    "        ensemble_preds_exercise.append(0)\n",
    "    else:\n",
    "        ensemble_preds_exercise.append(1)\n",
    "calculate_results(val_labels, ensemble_preds_exercise)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxcAAAMWCAYAAACDfHuZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABZgklEQVR4nO3de5xN9f7H8fcazAxj9ozBzJjcL7nkrtJUbrlfikOnnJSh4lR0oSTlrlKipESnhIrTnaJyDxXlEhFSbpliXDNjhsFcfn8w+9c+Q2bxHXtt6/Wcx3o82muvtfZnz3nU8fb5rO+ysrOzswUAAAAAFynI3wUAAAAAuDwQLgAAAAAYQbgAAAAAYAThAgAAAIARhAsAAAAARhAuAAAAABhBuAAAAABgBOECAAAAgBEF/V0AAAAA4C/p6ek6efKkv8vIJTg4WKGhof4uwzbCBQAAAFwpPT1dhcOLSxnH/F1KLrGxsdq5c2fABQzCBQAAAFzp5MmTUsYxhdRIkAoE+7uc/5d5Ukmbp+vkyZOECwAAACCgFAiW5aBwke3vAi4C4QIAAADuZgWd3pzCSbXYFLiVAwAAAHAUwgUAAAAAIxiLAgAAgLtZkizL31X8PweVYhedCwAAAABGEC4AAAAAGEG4AAAAgLvlrBblpM2GSZMmqXbt2vJ4PPJ4PIqPj9eXX37pfb9p06ayLMtnu++++3yusXv3brVv315FihRRdHS0BgwYoIyMDNu/Su65AAAAAAJY6dKl9dxzz6lKlSrKzs7W9OnT1bFjR61bt05XXXWVJKlXr14aOXKk95wiRYp4/zkzM1Pt27dXbGysVqxYob1796p79+4qVKiQnn32WVu1EC4AAACAAHbzzTf7vH7mmWc0adIkfffdd95wUaRIEcXGxp71/AULFmjz5s1atGiRYmJiVLduXY0aNUoDBw7U8OHDFRyc9wcMMhYFAAAAd7Ms522SUlJSfLYTJ06c96tkZmbqvffeU1pamuLj4737Z8yYoRIlSqhmzZoaNGiQjh075n1v5cqVqlWrlmJiYrz7WrdurZSUFG3atMnWr5LOBQAAAOBAZcqU8Xk9bNgwDR8+/KzHbty4UfHx8UpPT1fRokU1a9Ys1ahRQ5J0xx13qFy5coqLi9OGDRs0cOBAbd26VZ988okkKSkpySdYSPK+TkpKslUz4QIAAABwoMTERHk8Hu/rkJCQcx5btWpVrV+/XsnJyfroo4+UkJCgZcuWqUaNGurdu7f3uFq1aqlUqVJq3ry5tm/frkqVKhmtmXABAAAAd7uAFZry1ZlaclZ/yovg4GBVrlxZktSgQQOtXr1aL7/8sl5//fVcxzZs2FCStG3bNlWqVEmxsbFatWqVzzH79u2TpHPep3EuDvotAgAAADAhKyvrnPdorF+/XpJUqlQpSVJ8fLw2btyo/fv3e49ZuHChPB6Pd7Qqr+hcAAAAAAFs0KBBatu2rcqWLaujR49q5syZWrp0qebPn6/t27dr5syZateunYoXL64NGzaoX79+aty4sWrXri1JatWqlWrUqKG77rpLY8aMUVJSkgYPHqw+ffr87SjW2RAuAAAA4G5/WaHJEWzWsn//fnXv3l179+5VRESEateurfnz56tly5ZKTEzUokWLNH78eKWlpalMmTLq0qWLBg8e7D2/QIECmjt3ru6//37Fx8crLCxMCQkJPs/FyHPp2dnZ2bbPAgAAAAJcSkqKIiIiFNLgQVkF7P0NfX7KzjyhE2tfUXJycp7vuXAK7rkAAAAAYARjUQAAAHA5h60WFcB//x+4lQMAAABwFMIFAAAAACMYiwIAAIC7BfhqUU5C5wIAAACAEYQLAAAAAEYwFgUAAAB3sxy2WpSTarEpcCsHAAAA4CiECwAAAABGMBYFAAAAd2O1KGPoXAAAAAAwgnABAAAAwAjGogAAAOBurBZlTOBWDgAAAMBRCBcAAAAAjGAsCgAAAO7GalHG0LkAAAAAYAThAgAAAIARjEUBAADA3VgtypjArRwAAACAoxAuAAAAABjBWBQAAADczbKcNYrEalEAAAAA3I5wAQAAAMAIxqIAAADgbkHW6c0pnFSLTXQuAAAAABhBuAAAAABgBGNRAAAAcDceomdM4FYOAAAAwFEIFwAAAACMYCwKAAAA7mZZznpwnZNqsYnOBQAAAAAjCBcAXO/XX39Vq1atFBERIcuyNHv2bKPX37VrlyzL0rRp04xeN5A1bdpUTZs29XcZAADDCBcAHGH79u3697//rYoVKyo0NFQej0c33HCDXn75ZR0/fjxfPzshIUEbN27UM888o3feeUdXX311vn7epdSjRw9ZliWPx3PW3+Ovv/4qy7JkWZbGjh1r+/p79uzR8OHDtX79egPVAoCf5KwW5aQtQHHPBQC/+/zzz/XPf/5TISEh6t69u2rWrKmTJ0/qm2++0YABA7Rp0yb95z//yZfPPn78uFauXKmnnnpKffv2zZfPKFeunI4fP65ChQrly/XPp2DBgjp27JjmzJmj2267zee9GTNmKDQ0VOnp6Rd07T179mjEiBEqX7686tatm+fzFixYcEGfBwBwNsIFAL/auXOnunbtqnLlymnJkiUqVaqU970+ffpo27Zt+vzzz/Pt8w8cOCBJioyMzLfPsCxLoaGh+Xb98wkJCdENN9yg//73v7nCxcyZM9W+fXt9/PHHl6SWY8eOqUiRIgoODr4knwcAuLQCt+cC4LIwZswYpaamasqUKT7BIkflypX18MMPe19nZGRo1KhRqlSpkkJCQlS+fHk9+eSTOnHihM955cuXV4cOHfTNN9/o2muvVWhoqCpWrKi3337be8zw4cNVrlw5SdKAAQNkWZbKly8v6fQ4Uc4//9Xw4cNl/c8qHgsXLtSNN96oyMhIFS1aVFWrVtWTTz7pff9c91wsWbJEjRo1UlhYmCIjI9WxY0dt2bLlrJ+3bds29ejRQ5GRkYqIiFDPnj117Nixc/9i/8cdd9yhL7/8UkeOHPHuW716tX799VfdcccduY4/fPiwHnvsMdWqVUtFixaVx+NR27Zt9eOPP3qPWbp0qa655hpJUs+ePb3jVTnfs2nTpqpZs6bWrl2rxo0bq0iRIt7fy//ec5GQkKDQ0NBc379169YqVqyY9uzZk+fvCgC25awW5aQtQBEuAPjVnDlzVLFiRV1//fV5Ov7ee+/V0KFDVb9+fb300ktq0qSJRo8era5du+Y6dtu2bbr11lvVsmVLjRs3TsWKFVOPHj20adMmSVLnzp310ksvSZL+9a9/6Z133tH48eNt1b9p0yZ16NBBJ06c0MiRIzVu3Djdcsst+vbbb//2vEWLFql169bav3+/hg8frv79+2vFihW64YYbtGvXrlzH33bbbTp69KhGjx6t2267TdOmTdOIESPyXGfnzp1lWZY++eQT776ZM2eqWrVqql+/fq7jd+zYodmzZ6tDhw568cUXNWDAAG3cuFFNmjTx/kG/evXqGjlypCSpd+/eeuedd/TOO++ocePG3uscOnRIbdu2Vd26dTV+/Hg1a9bsrPW9/PLLKlmypBISEpSZmSlJev3117VgwQK98soriouLy/N3BQD4D2NRAPwmJSVFf/zxhzp27Jin43/88UdNnz5d9957r9544w1J0gMPPKDo6GiNHTtWX331lc8fXrdu3arly5erUaNGkk7/Ab1MmTKaOnWqxo4dq9q1a8vj8ahfv36qX7++7rzzTtvfYeHChTp58qS+/PJLlShRIs/nDRgwQFFRUVq5cqWioqIkSZ06dVK9evU0bNgwTZ8+3ef4evXqacqUKd7Xhw4d0pQpU/T888/n6fPCw8PVoUMHzZw5U3fffbeysrL03nvv6f777z/r8bVq1dIvv/yioKD//zuou+66S9WqVdOUKVM0ZMgQxcTEqG3btho6dKji4+PP+vtLSkrS5MmT9e9///tv64uMjNSUKVPUunVrPffcc7rjjjv02GOPqVOnThf0vwsAwD/oXADwm5SUFEmn/+CbF1988YUkqX///j77H330UUnKdW9GjRo1vMFCkkqWLKmqVatqx44dF1zz/8q5V+PTTz9VVlZWns7Zu3ev1q9frx49eniDhSTVrl1bLVu29H7Pv7rvvvt8Xjdq1EiHDh3y/g7z4o477tDSpUuVlJSkJUuWKCkp6awjUdLp+zRygkVmZqYOHTrkHfn64Ycf8vyZISEh6tmzZ56ObdWqlf79739r5MiR6ty5s0JDQ/X666/n+bMA4IL5e2Woy2i1qMCtHEDA83g8kqSjR4/m6fjffvtNQUFBqly5ss/+2NhYRUZG6rfffvPZX7Zs2VzXKFasmP78888LrDi322+/XTfccIPuvfdexcTEqGvXrvrggw/+Nmjk1Fm1atVc71WvXl0HDx5UWlqaz/7//S7FihWTJFvfpV27dgoPD9f777+vGTNm6Jprrsn1u8yRlZWll156SVWqVFFISIhKlCihkiVLasOGDUpOTs7zZ15xxRW2bt4eO3asoqKitH79ek2YMEHR0dF5PhcA4H+ECwB+4/F4FBcXp59++snWef97Q/W5FChQ4Kz7s7OzL/gzcu4HyFG4cGEtX75cixYt0l133aUNGzbo9ttvV8uWLXMdezEu5rvkCAkJUefOnTV9+nTNmjXrnF0LSXr22WfVv39/NW7cWO+++67mz5+vhQsX6qqrrspzh0Y6/fuxY926ddq/f78kaePGjbbOBQD4H+ECgF916NBB27dv18qVK897bLly5ZSVlaVff/3VZ/++fft05MgR78pPJhQrVsxnZaUc/9sdkaSgoCA1b95cL774ojZv3qxnnnlGS5Ys0VdffXXWa+fUuXXr1lzv/fzzzypRooTCwsIu7gucwx133KF169bp6NGjZ70JPsdHH32kZs2aacqUKeratatatWqlFi1a5Pqd5DXo5UVaWpp69uypGjVqqHfv3hozZoxWr15t7PoAcE7+XhmK1aIAwIzHH39cYWFhuvfee7Vv375c72/fvl0vv/yypNNjPZJyrej04osvSpLat29vrK5KlSopOTlZGzZs8O7bu3evZs2a5XPc4cOHc52b8zC5/10eN0epUqVUt25dTZ8+3ecP6z/99JMWLFjg/Z75oVmzZho1apReffVVxcbGnvO4AgUK5OqKfPjhh/rjjz989uWEoLMFMbsGDhyo3bt3a/r06XrxxRdVvnx5JSQknPP3CABwHlaLAuBXlSpV0syZM3X77berevXqPk/oXrFihT788EP16NFDklSnTh0lJCToP//5j44cOaImTZpo1apVmj59ujp16nTOZU4vRNeuXTVw4ED94x//0EMPPaRjx45p0qRJuvLKK31uaB45cqSWL1+u9u3bq1y5ctq/f79ee+01lS5dWjfeeOM5r//CCy+obdu2io+P1z333KPjx4/rlVdeUUREhIYPH27se/yvoKAgDR48+LzHdejQQSNHjlTPnj11/fXXa+PGjZoxY4YqVqzoc1ylSpUUGRmpyZMnKzw8XGFhYWrYsKEqVKhgq64lS5botdde07Bhw7xL406dOlVNmzbVkCFDNGbMGFvXAwD4B+ECgN/dcsst2rBhg1544QV9+umnmjRpkkJCQlS7dm2NGzdOvXr18h775ptvqmLFipo2bZpmzZql2NhYDRo0SMOGDTNaU/HixTVr1iz1799fjz/+uCpUqKDRo0fr119/9QkXt9xyi3bt2qW33npLBw8eVIkSJdSkSRONGDFCERER57x+ixYtNG/ePA0bNkxDhw5VoUKF1KRJEz3//PO2/2CeH5588kmlpaVp5syZev/991W/fn19/vnneuKJJ3yOK1SokKZPn65BgwbpvvvuU0ZGhqZOnWrrOxw9elR333236tWrp6eeesq7v1GjRnr44Yc1btw4de7cWdddd52x7wcAPpy2QpOTarHJyrZzNyAAAABwmUhJSVFERIRCmj8jq2Cov8vxys5I14nFTyk5Odm7smKgCNxYBAAAAMBRGIsCAACAuzlthSYn1WITnQsAAAAARhAuAAAAABjBWBQAAABczmGrRQXw3/8HbuUAAAAAHCWgOxdZWVnas2ePwsPDZQXwjS8AAACXq+zsbB09elRxcXEKCuLvtS93AR0u9uzZozJlyvi7DAAAAJxHYmKiSpcu7e8yzo7VoowJ6HARHh4uSQqukSCrQLCfqwEAM3YvHevvEgDAmKMpKapcoYz3z224vAV0uMgZhbIKBBMuAFw2Au1prACQF4ywu0NAhwsAAADgolmWs1aLCuAg5qDfIgAAAIBARrgAAAAAYARjUQAAAHA3y2EP0XNSLTYFbuUAAAAAHIVwAQAAAMAIxqIAAADgbjxEzxg6FwAAAACMIFwAAAAAMIKxKAAAALgbq0UZE7iVAwAAAHAUwgUAAAAAIxiLAgAAgLuxWpQxdC4AAAAAGEG4AAAAAGAEY1EAAABwN1aLMiZwKwcAAADgKIQLAAAAAEYwFgUAAAB3Y7UoY+hcAAAAADCCcAEAAADACMaiAAAA4GqWZcly0iiSk2qxic4FAAAAACMIFwAAAACMYCwKAAAArsZYlDl0LgAAAAAYQbgAAAAAYARjUQAAAHA368zmFE6qxSY6FwAAAACMIFwAAAAAMIKxKAAAALgaq0WZQ+cCAAAAgBGECwAAAABGMBYFAAAAV2Msyhw6FwAAAACMIFwAAAAAMIKxKAAAALgaY1Hm0LkAAAAAYAThAgAAAIARjEUBAADA1RiLMofOBQAAAAAjCBcAAAAAjGAsCgAAAO5mndmcwkm12ETnAgAAAIARhAsAAAAARjAWBQAAAFdjtShz6FwAAAAAMIJwAQAAAMAIxqIAAADgapYlh41F+buAC0fnAgAAAIARhAsAAAAARjAWBQAAAFez5LDVogJ4LorOBQAAAAAjCBcAAAAAjGAsCgAAAK7GQ/TMoXMBAAAAwAjCBQAAAAAjGIsCAACAu1ly1gJNTqrFJjoXAAAAAIwgXAAAAAAwgrEoAAAAuJvDVovKdlAtdtG5AAAAAALYpEmTVLt2bXk8Hnk8HsXHx+vLL7/0vp+enq4+ffqoePHiKlq0qLp06aJ9+/b5XGP37t1q3769ihQpoujoaA0YMEAZGRm2ayFcAAAAAAGsdOnSeu6557R27VqtWbNGN910kzp27KhNmzZJkvr166c5c+boww8/1LJly7Rnzx517tzZe35mZqbat2+vkydPasWKFZo+fbqmTZumoUOH2q7Fys7Ozjb2zS6xlJQURUREKKRWL1kFgv1dDgAY8efqV/1dAgAYk5KSopjiEUpOTpbH4/F3OT5y/iwZdcdbCgou4u9yvLJOHtPhmXdf1O8sKipKL7zwgm699VaVLFlSM2fO1K233ipJ+vnnn1W9enWtXLlS1113nb788kt16NBBe/bsUUxMjCRp8uTJGjhwoA4cOKDg4Lz/OZvOBQAAAOBAKSkpPtuJEyfOe05mZqbee+89paWlKT4+XmvXrtWpU6fUokUL7zHVqlVT2bJltXLlSknSypUrVatWLW+wkKTWrVsrJSXF2/3IK8IFAAAA4EBlypRRRESEdxs9evQ5j924caOKFi2qkJAQ3XfffZo1a5Zq1KihpKQkBQcHKzIy0uf4mJgYJSUlSZKSkpJ8gkXO+znv2cFqUQAAAHA1y2GrReXUkpiY6DMWFRIScs5zqlatqvXr1ys5OVkfffSREhIStGzZsnyv9X8RLgAAAAAHyln9KS+Cg4NVuXJlSVKDBg20evVqvfzyy7r99tt18uRJHTlyxKd7sW/fPsXGxkqSYmNjtWrVKp/r5awmlXNMXjEWBQAAAFxmsrKydOLECTVo0ECFChXS4sWLve9t3bpVu3fvVnx8vCQpPj5eGzdu1P79+73HLFy4UB6PRzVq1LD1uXQuAAAA4G7Wmc0pbNYyaNAgtW3bVmXLltXRo0c1c+ZMLV26VPPnz1dERITuuece9e/fX1FRUfJ4PHrwwQcVHx+v6667TpLUqlUr1ahRQ3fddZfGjBmjpKQkDR48WH369PnbUayzIVwAAAAAAWz//v3q3r279u7dq4iICNWuXVvz589Xy5YtJUkvvfSSgoKC1KVLF504cUKtW7fWa6+95j2/QIECmjt3ru6//37Fx8crLCxMCQkJGjlypO1aeM4FADgMz7kAcDkJhOdcFL9zquOec3Ho3Z6O/J2dD50LAAAAuJpTV4sKRNzQDQAAAMAIwgUAAAAAIxiLAgAAgKsxFmUOnQsAAAAARhAuAAAAABjBWBQAAABcjbEoc+hcAAAAADCCcAEAAADACMaiAAAA4GqMRZlD5wIAAACAEYQLAAAAAEYwFgUAAAB3s85sTuGkWmyicwEAAADACMIFAAAAACMYiwIAAICrsVqUOXQuAAAAABhBuAAAAABgBGNRAAAAcDXGosyhcwEAAADACMIFAAAAACMYiwIAAICrMRZlDp0LAAAAAEYQLgAAAAAYwVgUAAAA3M06szmFk2qxic4FAAAAACMIFwAAAACMYCwKAAAArsZqUebQuQAAAABgBOECAAAAgBGMRQEAAMDVGIsyh84FAAAAACMIFwAAAACMYCwKAAAArmbJYWNRAfwUPToXAAAAAIwgXAAAAAAwgrEoAAAAuBqrRZlD5wIAAACAEYQLAAAAAEYwFgUAAAB3s85sTuGkWmyicwEAAADACMIFAAAAACMIFwAAAACM4J4LAAAAuBpL0ZpD5wIAAACAEYQLAAAAAEYwFgUAAABXYyzKHDoXAAAAAIwgXAAAAAAwgrEoAAAAuJplnd6cwkm12EXnAgAAAIARhAsAAAAARjAWBQAAAFc7PRblnFkkB5ViG50LAAAAAEYQLgAAAAAYwVgUAAAA3M1hq0XJSbXYROcCAAAAgBGECwAAAABGMBYFAAAAV7Msy2GrRTmnFrvoXAAAAAAwgnABAAAAwAjGogAAAOBqlsNWi3JSLXbRuQAAAABgBOECAAAAgBGMRcEVHrrzJl1ft6KuqhKnksXCFRpSUPsOHdXXa3/VS9MXa9O2PT7HP/Xvdhp8X7tzXm/s1AUaMuEzn32NGlTRgjcfPuc5qzbsVJOEcRf3RQDgPFo1b6qvly875/ufzv1SrVq38dl3+PBhvfD8aH326Sz9npioiIgI3dCosZ4YNFh16tbN54oB/wsKshQU5JxZpGwH1WIX4QKuMODuVgorHKyfft2jTb+eDhLVK5VStw4N9c/WDdT10Tf15dc/5Tpvxbrt2p54INf+dZsTz/lZ23cf0Ir123Pt3/H7wYv4BgBgT6fOXVQ0rGiu/XFxV/i83rt3r5o3vVE7d+xQbGysWrVpq31JSfp01if6Yu4cfTx7jlq0bHWpygYQ4AgXcIXb+v1HP2zZrRMnM3z29/5nI7385O16begdqtxmsDIzs3zenzprhd6d872tz1qxfrt6D3v3omsGgIvx3PNjVa58+fMe1/f+3tq5Y4dat2mrGe99qLCwMEnSZ5/O1r9u66Ke3btp8y87FB4ens8VA7gccM8FXGHljztyBQtJ+s+HX2v77gOKLeFR9YqxfqgMAPwnMTFRX3w+VwULFtTLr07yBgtJuqVjJ/3ztq46ePCgpk99y49VAvkvZ7UoJ22BinAB1zuVkSlJOnkq08+VAMCltX7dD5Kk8hUqqFy5crneb9K0mSRp7pxPL2ldAAIXY1FwtX+1v0ZXlo/Wr7/t17bd+3O93/TaK1WnammFhBTSH/v+1IJvN2vdlnPfbyFJlcuW1MgHb1FURJgOHUnVivXbteDbLcrOzs6vrwEAuUybOkWHDx2SFRSkKlWu1M0dO6ls2bI+x6SlpUmSIiOLnfUaUcWLS5I2bvgxf4sFcNkgXMBV+nVvruqVSimscLCqVojVVZXjtGf/ESUMmqqsrNx/+O/WoaHP6+F9btasRevUa+g7Sjt+8qyfEV+3kuLrVvLZt/GXP/SvAW9q++7cN4cDQH547tmnfV4PGviYBj01RIOeGuLdV7JkSUlS4u7fznqNXTt3Sjq9mlRqaqqKFs19gzhwObAsS5aDZpGcVItdhAu4Sovrq+umhtW8r3/bc0j3DHk7Vzdie+IBPfHiJ5r/7Wbt3nNYxTxFdGP9ynrmkY76R4t6KhAUpNsffcPnnJTU43px2kLNXrxe286EiNpVS2tE35vVsHYFzX2tjxp2fU4pqen5/0UBuNaNjRqr59336rr46xVbqpR+T0zUrE8+0nPPPq2Rw4cqPNyjvg+dXjb76muuVUhIiPbt26cF8+f5LFGbnZ2td9+e5n199OhRwgWA87KyA3hWIyUlRREREQqp1UtWgWB/l4MAElG0sGpWidOg3m3V/LpqGvbqHI2ZMv+858WW8Gj1B0+qRLGiatJ9rFZt3HXec4KCLM1/42HdWL+yhrzymca+tcDAN8Dl7M/Vr/q7BFyGFi1coJvbtVZkZKR27N6jwoULS5Ief6y/Xnn5JUVHR2vCxMlq2uwm7UtK0sgRQ/XprE+UkXF6MYxdvycpJibGn18BASolJUUxxSOUnJwsj8fj73J85PxZstpjs1QgJOz8J1wimSfS9PPYfzjyd3Y+3NANV0pOPa5v121Xpwdf09rNuzXsgfZqUKPsec9LOpiidz77TpLU8voaefqsrKxsjZu68PQ58dUvvGgAuAgtWrZS/QZX68iRI1q96v+X2B71zGj9o8ut2r9/v7r+s7NiS0SqTs1qmvPpbI198WXvcZGRkX6oGrg0/L0y1OW0WhRjUXC1jIwsfTx/rRrUKKt2TWpp7ebd5z0nZ+QptkTe/ybhQs4BANMqV6miH9au0d69e737QkJCNPO9D/XNN19r4fx5OnjggEqXKaN/3tbVO/ddqXJlhYSE+KtsAAGEcAHXO3jk9GopJYrlbZa4mOf0KMGx9LPf0P1355zrJnAAuBSO/PmnJPk8zyLHjTc20o03NvLZN+OdtyVJjRs3zffaAFweHDEWNXHiRJUvX16hoaFq2LChVq1a5e+S4CKNGlSWJO1MPJin429pVkeSzrsk7V91al5XkrT+57yfAwAmHThwQN9+87UkqW69+uc9Pjs7W5MnTZQk9bynV77WBvhbzmpRTtoCld/Dxfvvv6/+/ftr2LBh+uGHH1SnTh21bt1a+/fnfuYAcCHi61RUy+ur5/oXtWDBIN3ftYnuaH+tjh0/qY8WrJV0uoPx79saq2gR3xGAsMLBmvBUV11bu4L2HkjWp0vW+7zf946mKh0Tmevz7+lygx7sdpOysrL0xodfG/1uAPBXK1es0GefzlZmpu9DQX/btUu33/oPpaWlqcPNt6h06dLe93bv3p3r/3OPHz+uPvf11prVq3RX9x665tprL0n9AAKf38eiXnzxRfXq1Us9e/aUJE2ePFmff/653nrrLT3xxBN+rg6Xg0plS+qNkXfpwJ9HtW5zog4np6l4ZFHVrBKnUiUjdDz9pHoPe0e/7zsiSSoSGqzxg27TqIdu0dpNvynpYIpKFCuqutXKqESxovoz5Zi6DZii4+mnfD6nb7dmGt3vH1r/c6J2/XFIISGFVLNynCqULqHMzCw9OuYjW90OALBr26+/qPe9PRUbG6u69eorIiJSu3f/pnU/rFV6erpqXHWVJk72XUZ72VdL9MB9vVS/wdUqU7as0o8f18oV3+rw4cNq2aq1Jkyc5KdvAyAQ+TVcnDx5UmvXrtWgQYO8+4KCgtSiRQutXLky1/EnTpzQiRMnvK9TUlIuSZ0IbF+v3abn35yvRg0qq9aVV6h4ZJhOnsrUb3sOadaidZr436Xa8ZeRqMPJaRo7dYGurVVBVcpF67o6FZWZlaVdfxzSu3O+1yvvLtGeA8m5Pufld5ao+XXVVKNSKVWrWEqFCgYp6WCKZs5dpdf+uzRPN4sDwMW45tqG6v3v+7V61fdau2a1/vzzT4WFhal2nbrq3OWf6n3f/d4laHPUq99A/+hyq1Z9/502/LheISEhuqpmLXVP6KnuPXoG9HgGkFdOG0VyUi12+TVcHDx4UJmZmbnWzY6JidHPP/+c6/jRo0drxIgRl6o8XCZ+23NIwyfOyfPxqcdOaMiEz2x/zqT3lmnSe8tsnwcAplSrXl0vv/qarXNq1qqlt9/9bz5VBMBt/H7PhR2DBg1ScnKyd0tMZMQEAAAAcAq/di5KlCihAgUKaN++fT779+3bp9jY2FzHh4SEsM42AAAAjHLag+ucVItdfu1cBAcHq0GDBlq8eLF3X1ZWlhYvXqz4+Hg/VgYAAADALr+vFtW/f38lJCTo6quv1rXXXqvx48crLS3Nu3oUAAAAgMDg93Bx++2368CBAxo6dKiSkpJUt25dzZs3L9dN3gAAAEB+sOSw1aLknFrs8nu4kKS+ffuqb9++/i4DAAAAwEUIqNWiAAAAADiXIzoXAAAAgL+wWpQ5dC4AAAAAGEG4AAAAAGAEY1EAAABwNcty2GpRDqrFLjoXAAAAAIwgXAAAAAAwgrEoAAAAuBqrRZlD5wIAAACAEYQLAAAAAEYwFgUAAABXY7Uoc+hcAAAAADCCcAEAAADACMaiAAAA4GqsFmUOnQsAAAAARhAuAAAAABjBWBQAAABcjdWizKFzAQAAAMAIwgUAAAAAIxiLAgAAgLs5bLUoOakWm+hcAAAAADCCcAEAAADACMaiAAAA4GqsFmUOnQsAAAAARhAuAAAAABjBWBQAAABczXLYalFOqsUuOhcAAAAAjCBcAAAAADCCsSgAAAC4GqtFmUPnAgAAAIARhAsAAAAARjAWBQAAAFdjtShz6FwAAAAAAWz06NG65pprFB4erujoaHXq1Elbt271OaZp06bee0tytvvuu8/nmN27d6t9+/YqUqSIoqOjNWDAAGVkZNiqhc4FAAAAEMCWLVumPn366JprrlFGRoaefPJJtWrVSps3b1ZYWJj3uF69emnkyJHe10WKFPH+c2Zmptq3b6/Y2FitWLFCe/fuVffu3VWoUCE9++yzea6FcAEAAABXC/TVoubNm+fzetq0aYqOjtbatWvVuHFj7/4iRYooNjb2rNdYsGCBNm/erEWLFikmJkZ169bVqFGjNHDgQA0fPlzBwcF5qoWxKAAAAMCBUlJSfLYTJ07k6bzk5GRJUlRUlM/+GTNmqESJEqpZs6YGDRqkY8eOed9buXKlatWqpZiYGO++1q1bKyUlRZs2bcpzzXQuAAAAAAcqU6aMz+thw4Zp+PDhf3tOVlaWHnnkEd1www2qWbOmd/8dd9yhcuXKKS4uThs2bNDAgQO1detWffLJJ5KkpKQkn2Ahyfs6KSkpzzUTLgAAAOBqTh2LSkxMlMfj8e4PCQk577l9+vTRTz/9pG+++cZnf+/evb3/XKtWLZUqVUrNmzfX9u3bValSJUOVMxYFAAAAOJLH4/HZzhcu+vbtq7lz5+qrr75S6dKl//bYhg0bSpK2bdsmSYqNjdW+fft8jsl5fa77NM6GcAEAAAAEsOzsbPXt21ezZs3SkiVLVKFChfOes379eklSqVKlJEnx8fHauHGj9u/f7z1m4cKF8ng8qlGjRp5rYSwKAAAArhboD9Hr06ePZs6cqU8//VTh4eHeeyQiIiJUuHBhbd++XTNnzlS7du1UvHhxbdiwQf369VPjxo1Vu3ZtSVKrVq1Uo0YN3XXXXRozZoySkpI0ePBg9enTJ0/jWDnoXAAAAAABbNKkSUpOTlbTpk1VqlQp7/b+++9LkoKDg7Vo0SK1atVK1apV06OPPqouXbpozpw53msUKFBAc+fOVYECBRQfH68777xT3bt393kuRl7QuQAAAAACWHZ29t++X6ZMGS1btuy81ylXrpy++OKLi6qFcAEAAABXc+pqUYGIsSgAAAAARhAuAAAAABjBWBQAAABcLdBXi3ISOhcAAAAAjCBcAAAAADCCsSgAAAC4GqtFmUPnAgAAAIARhAsAAAAARjAWBQAAAFez5KwVmhxUim10LgAAAAAYQbgAAAAAYARjUQAAAHC1IMtSkIPmopxUi110LgAAAAAYQbgAAAAAYARjUQAAAHA1y3LYalEOqsUuOhcAAAAAjCBcAAAAADCCsSgAAAC4mmVZshw0i+SkWuyicwEAAADACMIFAAAAACMYiwIAAICrBVmnN6dwUi120bkAAAAAYAThAgAAAIARjEUBAADA3SyHrdDkoFLsonMBAAAAwAjCBQAAAAAjGIsCAACAq1nW6c0pnFSLXXQuAAAAABhBuAAAAABgBGNRAAAAcDXrzI9TOKkWu+hcAAAAADCCcAEAAADACMaiAAAA4GpB1unNKZxUi110LgAAAAAYQbgAAAAAYARjUQAAAHA1y7JkOejJdU6qxS46FwAAAACMIFwAAAAAMIKxKAAAALiaZZ3enMJJtdhF5wIAAACAEYQLAAAAAEYwFgUAAABXC7IsBTloFslJtdhF5wIAAACAEYQLAAAAAEYwFgUAAABXY7Uoc+hcAAAAADCCcAEAAADACMaiAAAA4GqWZcly0CySk2qxi84FAAAAACMIFwAAAACMYCwKAAAArsZqUebQuQAAAABgBOECAAAAgBGMRQEAAMDVgixLQQ6aRXJSLXbRuQAAAABgBOECAAAAgBGMRQEAAMDVrDObUzipFrvoXAAAAAAwgnABAAAAwAjGogAAAOBqlmXJctAKTU6qxS46FwAAAACMIFwAAAAAMIKxKAAAALhakHV6cwon1WIXnQsAAAAARhAuAAAAABjBWBQAAABcjdWizKFzAQAAAMAIwgUAAAAAIxiLAgAAgOsF8CSSo9C5AAAAAGAE4QIAAACAEYxFAQAAwNVYLcqcPIWLzz77LM8XvOWWWy64GAAAAACBK0/holOnTnm6mGVZyszMvJh6AAAAAASoPIWLrKys/K4DAAAA8Isg6/TmFE6qxa6LuqE7PT3dVB0AAAAAApztcJGZmalRo0bpiiuuUNGiRbVjxw5J0pAhQzRlyhTjBQIAAAAIDLbDxTPPPKNp06ZpzJgxCg4O9u6vWbOm3nzzTaPFAQAAAPktZ7UoJ22Byna4ePvtt/Wf//xH3bp1U4ECBbz769Spo59//tlocQAAAAACh+1w8ccff6hy5cq59mdlZenUqVNGigIAAAAQeGyHixo1aujrr7/Otf+jjz5SvXr1jBQFAAAAXCqWA7dAZfsJ3UOHDlVCQoL++OMPZWVl6ZNPPtHWrVv19ttva+7cuflRIwAAAIAAYLtz0bFjR82ZM0eLFi1SWFiYhg4dqi1btmjOnDlq2bJlftQIAAAAIADY7lxIUqNGjbRw4ULTtQAAAACXXJBlKchBKzQ5qRa7LihcSNKaNWu0ZcsWSafvw2jQoIGxogAAAAAEHtvh4vfff9e//vUvffvtt4qMjJQkHTlyRNdff73ee+89lS5d2nSNAAAAAAKA7Xsu7r33Xp06dUpbtmzR4cOHdfjwYW3ZskVZWVm6995786NGAAAAIN9YlvO2QGW7c7Fs2TKtWLFCVatW9e6rWrWqXnnlFTVq1MhocQAAAAACh+3ORZkyZc76sLzMzEzFxcUZKQoAAABA4LEdLl544QU9+OCDWrNmjXffmjVr9PDDD2vs2LFGiwMAAADym2VZjtsCVZ7GoooVK+bzJdPS0tSwYUMVLHj69IyMDBUsWFB33323OnXqlC+FAgAAAHC2PIWL8ePH53MZAAAAAAJdnsJFQkJCftcBAAAA+IXTVmhyUi12XfBD9CQpPT1dJ0+e9Nnn8XguqiAAAAAAgcn2Dd1paWnq27evoqOjFRYWpmLFivlsAAAAANzJdrh4/PHHtWTJEk2aNEkhISF68803NWLECMXFxentt9/OjxoBAACAfBNkWY7bApXtsag5c+bo7bffVtOmTdWzZ081atRIlStXVrly5TRjxgx169YtP+oEAAAA4HC2OxeHDx9WxYoVJZ2+v+Lw4cOSpBtvvFHLly83Wx0AAACAgGE7XFSsWFE7d+6UJFWrVk0ffPCBpNMdjcjISKPFAQAAAPktZ7UoJ22Byna46Nmzp3788UdJ0hNPPKGJEycqNDRU/fr104ABA4wXCAAAACAw2L7nol+/ft5/btGihX7++WetXbtWlStXVu3atY0WBwAAACBwXNRzLiSpXLlyKleunIlaAAAAgEvOsixZDppFclItduUpXEyYMCHPF3zooYcuuBgAAAAAgStP4eKll17K08Usy/JLuFj16dMKD+fJ4AAuD81fYuU9AJePjPQ0f5eASyhP4SJndSgAAADgchOkC1jlKB85qRa7Arl2AAAAAA5CuAAAAABgxEWvFgUAAAAEMlaLMofOBQAAAAAjCBcAAAAAjLigcPH111/rzjvvVHx8vP744w9J0jvvvKNvvvnGaHEAAABAfrMsKchBWwBPRdkPFx9//LFat26twoULa926dTpx4oQkKTk5Wc8++6zxAgEAAAAEBtvh4umnn9bkyZP1xhtvqFChQt79N9xwg3744QejxQEAAAAIHLZXi9q6dasaN26ca39ERISOHDlioiYAAADgkskZR3IKJ9Vil+3ORWxsrLZt25Zr/zfffKOKFSsaKQoAAABA4LEdLnr16qWHH35Y33//vSzL0p49ezRjxgw99thjuv/++/OjRgAAAAABwPZY1BNPPKGsrCw1b95cx44dU+PGjRUSEqLHHntMDz74YH7UCAAAAOQbHqJnju1wYVmWnnrqKQ0YMEDbtm1TamqqatSooaJFi+ZHfQAAAAACxAU/RC84OFg1atTQtddeS7AAAAAA/GT06NG65pprFB4erujoaHXq1Elbt271OSY9PV19+vRR8eLFVbRoUXXp0kX79u3zOWb37t1q3769ihQpoujoaA0YMEAZGRm2arHduWjWrNnftmqWLFli95IAAACA3wT6alHLli1Tnz59dM011ygjI0NPPvmkWrVqpc2bNyssLEyS1K9fP33++ef68MMPFRERob59+6pz58769ttvJUmZmZlq3769YmNjtWLFCu3du1fdu3dXoUKFbD3Lzna4qFu3rs/rU6dOaf369frpp5+UkJBg93IAAAAALsK8efN8Xk+bNk3R0dFau3atGjdurOTkZE2ZMkUzZ87UTTfdJEmaOnWqqlevru+++07XXXedFixYoM2bN2vRokWKiYlR3bp1NWrUKA0cOFDDhw9XcHBwnmqxHS5eeumls+4fPny4UlNT7V4OAAAAwFmkpKT4vA4JCVFISMh5z0tOTpYkRUVFSZLWrl2rU6dOqUWLFt5jqlWrprJly2rlypW67rrrtHLlStWqVUsxMTHeY1q3bq37779fmzZtUr169fJU8wXfc/G/7rzzTr311lumLgcAAABcEpblvE2SypQpo4iICO82evTo836XrKwsPfLII7rhhhtUs2ZNSVJSUpKCg4MVGRnpc2xMTIySkpK8x/w1WOS8n/NeXtnuXJzLypUrFRoaaupyAAAAgKslJibK4/F4X+ela9GnTx/99NNP+uabb/KztHOyHS46d+7s8zo7O1t79+7VmjVrNGTIEGOFAQAAAG7m8Xh8wsX59O3bV3PnztXy5ctVunRp7/7Y2FidPHlSR44c8ele7Nu3T7Gxsd5jVq1a5XO9nNWkco7JC9tjUX9tzURERCgqKkpNmzbVF198oWHDhtm9HAAAAOBXQZbluM2O7Oxs9e3bV7NmzdKSJUtUoUIFn/cbNGigQoUKafHixd59W7du1e7duxUfHy9Jio+P18aNG7V//37vMQsXLpTH41GNGjXyXIutzkVmZqZ69uypWrVqqVixYnZOBQAAAJAP+vTpo5kzZ+rTTz9VeHi49x6JiIgIFS5cWBEREbrnnnvUv39/RUVFyePx6MEHH1R8fLyuu+46SVKrVq1Uo0YN3XXXXRozZoySkpI0ePBg9enTJ0/jWDlsdS4KFCigVq1a6ciRI3ZOAwAAAJBPJk2apOTkZDVt2lSlSpXybu+//773mJdeekkdOnRQly5d1LhxY8XGxuqTTz7xvl+gQAHNnTtXBQoUUHx8vO688051795dI0eOtFWL7XsuatasqR07duRqtwAAAACBKEgGl1A1wG4t2dnZ5z0mNDRUEydO1MSJE895TLly5fTFF1/Y/HRftn+PTz/9tB577DHNnTtXe/fuVUpKis8GAAAAwJ3y3LkYOXKkHn30UbVr106SdMstt8j6y80m2dnZsixLmZmZ5qsEAAAA4Hh5DhcjRozQfffdp6+++io/6wEAAAAuqb8+uM4JnFSLXXkOFzmzXE2aNMm3YgAAAAAELlv3XFiBHKMAAAAA5Ctbq0VdeeWV5w0Yhw8fvqiCAAAAgEspSPYfXJefguScWuyyFS5GjBihiIiI/KoFAAAAQACzFS66du2q6Ojo/KoFAAAAQADLc7jgfgsAAABcjlgtypw839Cdlyf/AQAAAHCvPHcusrKy8rMOAAAAAAHO1j0XAAAAwOUmyDq9OYWTarHL1nMuAAAAAOBcCBcAAAAAjGAsCgAAAK5mWXLUQ/QcVIptdC4AAAAAGEG4AAAAAGAEY1EAAABwNR6iZw6dCwAAAABGEC4AAAAAGMFYFAAAAFyNh+iZQ+cCAAAAgBGECwAAAABGMBYFAAAAV7PO/DiFk2qxi84FAAAAACMIFwAAAACMYCwKAAAArsZqUebQuQAAAABgBOECAAAAgBGMRQEAAMDVGIsyh84FAAAAACMIFwAAAACMYCwKAAAArmZZlizLObNITqrFLjoXAAAAAIwgXAAAAAAwgrEoAAAAuBqrRZlD5wIAAACAEYQLAAAAAEYwFgUAAABXs6zTm1M4qRa76FwAAAAAMIJwAQAAAMAIxqIAAADgakGWpSAHzSI5qRa76FwAAAAAMIJwAQAAAMAIxqIAAADgajxEzxw6FwAAAACMIFwAAAAAMIKxKAAAALibwx6iJyfVYhOdCwAAAABGEC4AAAAAGMFYFAAAAFwtSJaCHDSL5KRa7KJzAQAAAMAIwgUAAAAAIxiLAgAAgKtZDlstykm12EXnAgAAAIARhAsAAAAARjAWBQAAAFcLsk5vTuGkWuyicwEAAADACMIFAAAAACMYiwIAAICrBVmWghy0RJOTarGLzgUAAAAAIwgXAAAAAIxgLAoAAACuxkP0zKFzAQAAAMAIwgUAAAAAIxiLAgAAgKsFyWGrRck5tdhF5wIAAACAEYQLAAAAAEYwFgUAAABXY7Uoc+hcAAAAADCCcAEAAADACMaiAAAA4GpBctbfuDupFrsCuXYAAAAADkK4AAAAAGAEY1EAAABwNcuyZDloiSYn1WIXnQsAAAAARhAuAAAAABjBWBQAAABczTqzOYWTarGLzgUAAAAAIwgXAAAAAIxgLAoAAACuFmRZCnLQCk1OqsUuOhcAAAAAjCBcAAAAADCCsSgAAAC4XuAOIjkLnQsAAAAARhAuAAAAABjBWBQAAABczbJOb07hpFrsonMBAAAAwAjCBQAAAAAjGIsCAACAq1mWJctBs0hOqsUuOhcAAAAAjCBcAAAAADCCsSgAAAC4WpCc9TfuTqrFrkCuHQAAAICDEC4AAAAAGMFYFAAAAFyN1aLMoXMBAAAAwAjCBQAAAAAjGIsCAACAq1lnNqdwUi120bkAAAAAYAThAgAAAIARjEUBAADA1Vgtyhw6FwAAAACMIFwAAAAAMIKxKAAAALhakJz1N+5OqsWuQK4dAAAAgIMQLgAAAAAYwVgUAAAAXI3VosyhcwEAAADACMIFAAAAACMYiwIAAICrWWc2p3BSLXbRuQAAAABgBOECAAAAgBGMRQEAAMDVLOv05hROqsUuwgVcb9svP+vVF5/Tyq+XKfnIYZWMidVNLdvq4ccHK6p4CZ9jP/rvO3r8od7nvFaHTrdqwhvv5HfJAFwupGCQri1fTDdUKq46pT2K8YQqKytbvx85rmW/HNR7a37X8VNZPufcfX053XNDuXNe853vd2vy8l0++yqVDFPH2rGqFhuuGE+IPKGFdDIzS7sOHdPCzfs168e9yszKzo+vCCBAES7gaiu+Xqped3bR8WPHVKlKVdW/pqF++Xmz3nnrdS2aN1cffblUpeJK5zqv+lW1VaNm7Vz76zS45hJUDcDtWlWP1hNtrpQk7TyYpm+2HVJYSAHVivPo3hvLq0X1aPV570cdOXYq17k//p6sP44cz7V/a1Jqrn11S0eoS/0rtDc5XTsPHdORY6cUWaSQasd5VLNFZTW5soT6fbhRGQQMAGcQLuBax48d0yP/7qHjx47pwceeVL+BQyRJ2dnZem7Ek3pj4ng98fD9mv7hnFzntmx3sx55fPClLhkAJEkZWdma/eNefbDmd/12+P+DQvGwYL3Q5SpVjQnXw80qacTnP+c6d+6GJH2xaV+ePmfljsP6539WaU9yus/+YkUK6eXbaqt+2Uh1rFNKH6/bc3FfCPCzIFkKctAaTU6qxS5u6IZrzf/8Ux08sE8VK1+phwc85d1vWZYee2qkSpctp6+XLtKWnzb4sUoAyO3LTfv0woJffYKFJB1KO6lxi7ZJkppcWUIFgy7uDyh7ktNzBQtJ+vPYKb27KlGS1KBs5EV9BoDLC+ECrrXxxx8kSdfG36igIN9/FQoVKqQG18ZLkhbOm3vJawOAC7Vtf5qk0/dlRBQulG+fk5l5ehTqVGbWeY4E4CaEC7jW8WPHJEmeyMizvl+sWJQknbVz8dOP6zR6+CA99WhfvfT8KH3/7df5VicA2BEXGSrp9B/6U9Jz33NRv1ykHm5WUQNaVlbCdWVVNaao7c8IDymortdcIUlasePwxRUMOEDOalFO2uxavny5br75ZsXFxcmyLM2ePdvn/R49esiyLJ+tTZs2PsccPnxY3bp1k8fjUWRkpO655x6lpua+H+vvcM8FXCtnJag9ibvP+n7i7t8kSX/8npjrvSULvtCSBV94X78y9lk1vL6RJrzxjkpGx+RDtQCQN7fVP/2H/u93/qlTmblvtG57le9/o3o3Kq+vth7QM19uzbXCVI7SkaFKiC8ry7IUVaSQal3hUZHggpq1fo8WbN5v/ksAsC0tLU116tTR3Xffrc6dO5/1mDZt2mjq1Kne1yEhIT7vd+vWTXv37tXChQt16tQp9ezZU71799bMmTPzXAfhAq51bfyNem38GH21cJ4OHzros+xs0t4/9M2yxZKktNSj3v3RMbF6+PHBatmmg8qUr6D048e1Yd0aPTfiKX2/4mvd262zPpm3XAUKFLjk3wcA4isUU4fasTqVmaU3vtnl897vR47rla+267udfyopJV3hIQVVt0yEHmhSUc2qllRQkKUnZ28+63WjwoLVrmasz74P1v6hN77ZJdaJApyhbdu2atu27d8eExISotjY2LO+t2XLFs2bN0+rV6/W1VdfLUl65ZVX1K5dO40dO1ZxcXF5qoOxKLhWo2YtVLN2PaWlpapn14768YfVSktN1Q+rv1PPrp2UmZEhST73YzS+qaUeHvCUatSqo/Bwj0pGx6h56/aaveAbVahURRvX/6DPP/3IX18JgIuVjSqsoe2rKciyNHHpDm07kObz/oLN+/Xemj+069AxpZ/K0oHUk1q45YDufWedjhw7pSZVSuiqUuFnvfaGP1J0wwvL1Wjsct36n1Wa8NV2tasZoyl31VOsJ+Ss5wCBxHLgjySlpKT4bCdOnLio77l06VJFR0eratWquv/++3Xo0CHveytXrlRkZKQ3WEhSixYtFBQUpO+//z7Pn0G4gGtZlqVJ0/6rK6vV0Mb1P+gfrRurVoWSurVdMx06cEAPDzi91GzEOe7J+KuwokWV0OsBSdLXSxblZ9kAkEuJosF68dZa8hQupP+u/l0f/pD3pWEPpZ3UFz8lSZIaVoj622OzsqW9yel6f80fevbLrSobVUT9mle+qNoBnFuZMmUUERHh3UaPHn3B12rTpo3efvttLV68WM8//7yWLVumtm3bKjMzU5KUlJSk6Ohon3MKFiyoqKgoJSUl5flzGIuCq11RppzmfvW9Fnz+qdau/k7p6em6smp1dby1q+bP/VSSVKVqjTxdq0LF0/8Hu39f3v8FBICLFR5aUOP/WUulIkI1d2OSXl26w/Y1Ev88vaRt8aLBeT5n2a+HdOxkhq6rUEwFgywepAfkg8TERHk8Hu/r/71Hwo6uXbt6/7lWrVqqXbu2KlWqpKVLl6p58+YXVedfES7gegULFlS7jl3UrmMXn/1rV38nSWp4Q6M8XSf5yJ+SpMJhRcwWCADnULhQkMZ1qakKJcK09JcDen7+Lxd0nfDQ038cSD+Vaeu8lOMZio0IVXhoQf15lqeBA4HiQldoyi85tXg8Hp9wYVLFihVVokQJbdu2Tc2bN1dsbKz27/ddoCEjI0OHDx8+530aZ+PXsajzLZkF+MuBfUmaN2eWikUVV5v2nfJ0zry5syVJNWvVy7/CAOCMQgUsPfePq3RVnEff7TysYXN+1oU2D5pUOb2gxdZ9eV9yMi4iVNGeEKWeyFDycYIFEGh+//13HTp0SKVKlZIkxcfH68iRI1q7dq33mCVLligrK0sNGzbM83X9Gi5ylsyaOHGiP8uAi23dskkn0n2fPrt3z+/q3f2fSk09qidHPKfQwoW97702/gUdPnTQ5/hTp07p5Ree0ReffaLQwoV16x13XZLaAbhXkCWN6FBdV5crpvWJyXpy9ua/HUuKLFxIneuVUpFCvivZFS4UpAEtK+uqOI8Opp7Qsl98//t2a704RYXlfhBf2WKFNbzD6ZvH523ad8GhBoA5qampWr9+vdavXy9J2rlzp9avX6/du3crNTVVAwYM0Hfffaddu3Zp8eLF6tixoypXrqzWrVtLkqpXr642bdqoV69eWrVqlb799lv17dtXXbt2zfNKUZKfx6LysmQWkJ/enDhe87/4TDVr11XJmFgdOnhAa75foZMnTujBRwepS9c7fY4f+8xQTRj7jGrVqa9SV5RW6tGj2vLTj9qXtFchoaF68bW3FFvqCj99GwBu0aVenJpcebrbkHz8lB5refabql9dukPJxzMUWihIj7aoovsbV9CWvak6lHZSkUUK6croooosUkgp6ac0+LMtOpHh+5yLrteU1kM3VdK2A6n6/c90WZYU6wlR1ZhwFQiytC7xiCYv35nv3xfIb5YsBck5c1HWBdSyZs0aNWvWzPu6f//+kqSEhARNmjRJGzZs0PTp03XkyBHFxcWpVatWGjVqlM99HDNmzFDfvn3VvHlzBQUFqUuXLpowYYKtOgLqnosTJ074LMGVkpLix2pwOWjZ7mYd2L9PWzZt1NpVK+WJLKYmN7VUz38/qOtuaJzr+Acfe1Lr1nyvHdt+0aaN65Wdna3YUlfoXwn36p77HlTFylf64VsAcJvw0P/vJuSEjLOZ8u1vSj6eoZTjGXrn+926qpRHZaIKq+YVHmVlZWtvcrq+2JSk99f8oYOpJ3Od/5+vdym+YpSqxRZVwwrFFFIwSCnpGVr9259atOWA5m3ax3MuAIdo2rSpsrPP/W/k/Pnzz3uNqKgoWw/MO5uAChejR4/WiBEj/F0GLiOt2t2iVu1uyfPx/QYOycdqACBv3lrxm95a8Vuejz92KlOTl++y/TkLtuzXgi08gRtA3gXUcy4GDRqk5ORk75aYmOjvkgAAABDgclaLctIWqAKqcxESEnJR6/sCAAAAyD8B1bkAAAAA4Fx+7VykpqZq27Zt3tc5S2ZFRUWpbNmyfqwMAAAAbuG0USQn1WKXX8PF3y2ZNW3aND9VBQAAAOBC+DVcnG/JLAAAAACBI6Bu6AYAAABMs878OIWTarGLG7oBAAAAGEG4AAAAAGAEY1EAAABwtSDr9OYUTqrFLjoXAAAAAIwgXAAAAAAwgrEoAAAAuBqrRZlD5wIAAACAEYQLAAAAAEYwFgUAAABXs6zTm1M4qRa76FwAAAAAMIJwAQAAAMAIxqIAAADgapactUKTcyqxj84FAAAAACMIFwAAAACMYCwKAAAArhZknd6cwkm12EXnAgAAAIARhAsAAAAARjAWBQAAAFezzvw4hZNqsYvOBQAAAAAjCBcAAAAAjGAsCgAAAK5mWac3p3BSLXbRuQAAAABgBOECAAAAgBGMRQEAAMDVrDObUzipFrvoXAAAAAAwgnABAAAAwAjGogAAAOBqQbIU5KAlmoICeDCKzgUAAAAAIwgXAAAAAIwgXAAAAAAwgnsuAAAA4GosRWsOnQsAAAAARhAuAAAAABjBWBQAAADcjbkoY+hcAAAAADCCcAEAAADACMaiAAAA4GrWmR+ncFItdtG5AAAAAGAE4QIAAACAEYxFAQAAwN0syXLSJJKTarGJzgUAAAAAIwgXAAAAAIxgLAoAAACuxjP0zKFzAQAAAMAIwgUAAAAAIxiLAgAAgLsxF2UMnQsAAAAARhAuAAAAABjBWBQAAABczTrz4xROqsUuOhcAAAAAjCBcAAAAADCCsSgAAAC4mmWd3pzCSbXYRecCAAAAgBGECwAAAABGMBYFAAAAV+MZeubQuQAAAABgBOECAAAAgBGMRQEAAMDdmIsyhs4FAAAAACMIFwAAAACMYCwKAAAArmad+XEKJ9ViF50LAAAAAEYQLgAAAAAYwVgUAAAAXM2yTm9O4aRa7KJzAQAAAMAIwgUAAAAAIxiLAgAAgKvxDD1z6FwAAAAAMIJwAQAAAMAIxqIAAADgbsxFGUPnAgAAAIARhAsAAAAARjAWBQAAAFezzvw4hZNqsYvOBQAAAAAjCBcAAAAAjGAsCgAAAK5mWac3p3BSLXbRuQAAAABgBOECAAAAgBGMRQEAAMDVeIaeOXQuAAAAABhBuAAAAABgBGNRAAAAcDfmooyhcwEAAADACMIFAAAAACMYiwIAAICrWWd+nMJJtdhF5wIAAACAEYQLAAAAAEYwFgUAAABXs6zTm1M4qRa76FwAAAAAMIJwAQAAAMAIxqIAAADgajxDzxw6FwAAAACMIFwAAAAAMIKxKAAAALgbc1HG0LkAAAAAYAThAgAAAIARjEUBAADA1awzP07hpFrsonMBAAAAwAjCBQAAAAAjGIsCAACAq1nW6c0pnFSLXXQuAAAAABhBuAAAAABgBGNRAAAAcDWeoWcOnQsAAAAARhAuAAAAABjBWBQAAADcjbkoY+hcAAAAADCCcAEAAADACMaiAAAA4GrWmR+ncFItdtG5AAAAAGAE4QIAAACAEYQLAAAAuJslWQ7aLmQqavny5br55psVFxcny7I0e/Zsn/ezs7M1dOhQlSpVSoULF1aLFi3066+/+hxz+PBhdevWTR6PR5GRkbrnnnuUmppqqw7CBQAAABDg0tLSVKdOHU2cOPGs748ZM0YTJkzQ5MmT9f333yssLEytW7dWenq695hu3bpp06ZNWrhwoebOnavly5erd+/eturghm4AAAAgwLVt21Zt27Y963vZ2dkaP368Bg8erI4dO0qS3n77bcXExGj27Nnq2rWrtmzZonnz5mn16tW6+uqrJUmvvPKK2rVrp7FjxyouLi5PddC5AAAAgKtZDtwkKSUlxWc7ceLEBX2/nTt3KikpSS1atPDui4iIUMOGDbVy5UpJ0sqVKxUZGekNFpLUokULBQUF6fvvv8/zZxEuAAAAAAcqU6aMIiIivNvo0aMv6DpJSUmSpJiYGJ/9MTEx3veSkpIUHR3t837BggUVFRXlPSYvGIsCAAAAHCgxMVEej8f7OiQkxI/V5A2dCwAAALibv2egzjEX5fF4fLYLDRexsbGSpH379vns37dvn/e92NhY7d+/3+f9jIwMHT582HtMXhAuAAAAgMtYhQoVFBsbq8WLF3v3paSk6Pvvv1d8fLwkKT4+XkeOHNHatWu9xyxZskRZWVlq2LBhnj+LsSgAAAAgwKWmpmrbtm3e1zt37tT69esVFRWlsmXL6pFHHtHTTz+tKlWqqEKFChoyZIji4uLUqVMnSVL16tXVpk0b9erVS5MnT9apU6fUt29fde3aNc8rRUmECwAAALicdebHKS6kljVr1qhZs2be1/3795ckJSQkaNq0aXr88ceVlpam3r1768iRI7rxxhs1b948hYaGes+ZMWOG+vbtq+bNmysoKEhdunTRhAkTbNVBuAAAAAACXNOmTZWdnX3O9y3L0siRIzVy5MhzHhMVFaWZM2deVB3ccwEAAADACDoXAAAAcDXLOr05hZNqsYvOBQAAAAAjCBcAAAAAjGAsCgAAAK72l+fWOYKTarGLzgUAAAAAIwgXAAAAAIxgLAoAAADuxlyUMXQuAAAAABhBuAAAAABgBGNRAAAAcDXrzI9TOKkWu+hcAAAAADCCcAEAAADACMaiAAAA4GqWJMtBk0gOKsU2OhcAAAAAjCBcAAAAADCCsSgAAAC4Gs/QM4fOBQAAAAAjCBcAAAAAjGAsCgAAAK5mWQ5bLcpBtdhF5wIAAACAEYQLAAAAAEYwFgUAAACXY70oU+hcAAAAADAioDsX2dnZkqTUo0f9XAkAmJORnubvEgDAmMz0Y5L+/89tuLwFdLg4eiZU3FCnsp8rAQAAwN85evSoIiIi/F3GWbFalDkBHS7i4uKUmJio8PBwWYH8vwIcLyUlRWXKlFFiYqI8Ho+/ywGAi8Z/13CpZGdn6+jRo4qLi/N3KbgEAjpcBAUFqXTp0v4uAy7i8Xj4P2EAlxX+u4ZLwakdC5gX0OECAAAAuFisFWUOq0UBAAAAMIJwAeRBSEiIhg0bppCQEH+XAgBG8N81APnBymZdMAAAALhQSkqKIiIitHX3AYU76N6joykpqlq2pJKTkwPunig6FwAAAACMIFwAAAAAMILVogAAAOBq1pkfp3BSLXbRuQAAAABgBOECyIOJEyeqfPnyCg0NVcOGDbVq1Sp/lwQAF2T58uW6+eabFRcXJ8uyNHv2bH+XBOAyQrgAzuP9999X//79NWzYMP3www+qU6eOWrdurf379/u7NACwLS0tTXXq1NHEiRP9XQrgHJYDtwDFUrTAeTRs2FDXXHONXn31VUlSVlaWypQpowcffFBPPPGEn6sDgAtnWZZmzZqlTp06+bsUwC9ylqL9JfGg45aivbJMCZaiBS43J0+e1Nq1a9WiRQvvvqCgILVo0UIrV670Y2UAAADOQ7gA/sbBgweVmZmpmJgYn/0xMTFKSkryU1UAAMAkf09AXUZTUYQLAAAAAGYQLoC/UaJECRUoUED79u3z2b9v3z7Fxsb6qSoAAABnIlwAfyM4OFgNGjTQ4sWLvfuysrK0ePFixcfH+7EyAABgimU5bwtUPKEbOI/+/fsrISFBV199ta699lqNHz9eaWlp6tmzp79LAwDbUlNTtW3bNu/rnTt3av369YqKilLZsmX9WBmAywHhAjiP22+/XQcOHNDQoUOVlJSkunXrat68eblu8gaAQLBmzRo1a9bM+7p///6SpISEBE2bNs1PVQG4XPCcCwAAALhSznMutv9+yHHPuahUujjPuQAAAADgXoQLAAAAAEZwzwUAAADczWlPrnNSLTbRuQAAAABgBOECAAAAgBGMRQEAAMDVmIoyh84FAAAAACMIFwBgU48ePdSpUyfv66ZNm+qRRx655HUsXbpUlmXpyJEj5zzGsizNnj07z9ccPny46tate1F17dq1S5Zlaf369Rd1HQBA4CFcALgs9OjRQ5ZlybIsBQcHq3Llyho5cqQyMjLy/bM/+eQTjRo1Kk/H5iUQAAAuLcty3haouOcCwGWjTZs2mjp1qk6cOKEvvvhCffr0UaFChTRo0KBcx548eVLBwcFGPjcqKsrIdQAACHR0LgBcNkJCQhQbG6ty5crp/vvvV4sWLfTZZ59J+v9RpmeeeUZxcXGqWrWqJCkxMVG33XabIiMjFRUVpY4dO2rXrl3ea2ZmZqp///6KjIxU8eLF9fjjjys7O9vnc/93LOrEiRMaOHCgypQpo5CQEFWuXFlTpkzRrl271KxZM0lSsWLFZFmWevToIUnKysrS6NGjVaFCBRUuXFh16tTRRx995PM5X3zxha688koVLlxYzZo186kzrwYOHKgrr7xSRYoUUcWKFTVkyBCdOnUq13Gvv/66ypQpoyJFiui2225TcnKyz/tvvvmmqlevrtDQUFWrVk2vvfaa7VoAAJcfOhcALluFCxfWoUOHvK8XL14sj8ejhQsXSpJOnTql1q1bKz4+Xl9//bUKFiyop59+Wm3atNGGDRsUHByscePGadq0aXrrrbdUvXp1jRs3TrNmzdJNN910zs/t3r27Vq5cqQkTJqhOnTrauXOnDh48qDJlyujjjz9Wly5dtHXrVnk8HhUuXFiSNHr0aL377ruaPHmyqlSpouXLl+vOO+9UyZIl1aRJEyUmJqpz587q06ePevfurTVr1ujRRx+1/TsJDw/XtGnTFBcXp40bN6pXr14KDw/X448/7j1m27Zt+uCDDzRnzhylpKTonnvu0QMPPKAZM2ZIkmbMmKGhQ4fq1VdfVb169bRu3Tr16tVLYWFhSkhIsF0TAPifJctRazQ5qRZ7CBcALjvZ2dlavHix5s+frwcffNC7PywsTG+++aZ3HOrdd99VVlaW3nzzTVlnBlynTp2qyMhILV26VK1atdL48eM1aNAgde7cWZI0efJkzZ8//5yf/csvv+iDDz7QwoUL1aJFC0lSxYoVve/njFBFR0crMjJS0ulOx7PPPqtFixYpPj7ee84333yj119/XU2aNNGkSZNUqVIljRs3TpJUtWpVbdy4Uc8//7yt383gwYO9/1y+fHk99thjeu+993zCRXp6ut5++21dccUVkqRXXnlF7du317hx4xQbG6thw4Zp3Lhx3t9JhQoVtHnzZr3++uuECwBwOcIFgMvG3LlzVbRoUZ06dUpZWVm64447NHz4cO/7tWrV8rnP4scff9S2bdsUHh7uc5309HRt375dycnJ2rt3rxo2bOh9r2DBgrr66qtzjUblWL9+vQoUKKAmTZrkue5t27bp2LFjatmypc/+kydPql69epKkLVu2+NQhyRtE7Hj//fc1YcIEbd++XampqcrIyJDH4/E5pmzZst5gkfM5WVlZ2rp1q8LDw7V9+3bdc8896tWrl/eYjIwMRURE2K4HAHB5IVwAuGw0a9ZMkyZNUnBwsOLi4lSwoO9/4sLCwnxep6amqkGDBt5xn78qWbLkBdWQM+ZkR2pqqiTp888/9/lDvXT6PhJTVq5cqW7dumnEiBFq3bq1IiIi9N5773m7IXZqfeONN3KFnQIFChirFQAuJaet0OSkWuwiXAC4bISFhaly5cp5Pr5+/fp6//33FR0dnetv73OUKlVK33//vRo3bizp9N/Qr127VvXr1z/r8bVq1VJWVpaWLVvmHYv6q5zOSWZmpndfjRo1FBISot27d5+z41G9enXvzek5vvvuu/N/yb9YsWKFypUrp6eeesq777fffst13O7du7Vnzx7FxcV5PycoKEhVq1ZVTEyM4uLitGPHDnXr1s3W5wMALn+sFgXAtbp166YSJUqoY8eO+vrrr7Vz504tXbpUDz30kH7//XdJ0sMPP6znnntOs2fP1s8//6wHHnjgb59RUb58eSUkJOjuu+/W7Nmzvdf84IMPJEnlypWTZVmaO3euDhw4oNTUVIWHh+uxxx5Tv379NH36dG3fvl0//PCDXnnlFU2fPl2SdN999+nXX3/VgAEDtHXrVs2cOVPTpk2z9X2rVKmi3bt367333tP27ds1YcIEzZo1K9dxoaGhSkhI0I8//qivv/5aDz30kG677TbFxsZKkkaMGKHRo0drwoQJ+uWXX7Rx40ZNnTpVL774oq16AACXH8IFANcqUqSIli9frrJly6pz586qXr267rnnHqWnp3s7GY8++qjuuusuJSQkKD4+XuHh4frHP/7xt9edNGmSbr31Vj3wwAOqVq2aevXqpbS0NEnSFVdcoREjRuiJJ55QTEyM+vbtK0kaNWqUhgwZotGjR6t69epq06aNPv/8c1WoUEHS6fsgPv74Y82ePVt16tTR5MmT9eyzz9r6vrfccov69eunvn37qm7dulqxYoWGDBmS67jKlSurc+fOateunVq1aqXatWv7LDV777336s0339TUqVNVq1YtNWnSRNOmTfPWCgBwLyv7XHclAgAAAJexlJQURUREaNfew+ccj/WHlJQUlS8VpeTkZEfVlRd0LgAAAAAYwQ3dAAAAcDVWizKHzgUAAAAAIwgXAAAAAIxgLAoAAACuZp35cQon1WIXnQsAAAAARhAuAAAAABjBWBQAAABcjdWizKFzAQAAAMAIwgUAAAAAIxiLAgAAgKtZZzancFItdtG5AAAAAGAE4QIAAACAEYxFAQAAwN2YizKGzgUAAAAAIwgXAAAAAIxgLAoAAACuZp35cQon1WIXnQsAAAAARhAuAAAAABjBWBQAAABczbJOb07hpFrsonMBAAAAwAjCBQAAAAAjGIsCAACAq/EMPXPoXAAAAAAwgnABAAAAwAjGogAAAOBuzEUZQ+cCAAAAgBGECwAAAABGMBYFAAAAV7PO/DiFk2qxi84FAAAAACMIFwAAAACMYCwKAAAArmZZpzencFItdtG5AAAAAGAEnQsAAAC4WkpKir9L8OG0euwgXAAAAMCVgoODFRsbqyoVyvi7lFxiY2MVHBzs7zJss7Kzs7P9XQQAAADgD+np6Tp58qS/y8glODhYoaGh/i7DNsIFAAAAACO4oRsAAACAEYQLAAAAAEYQLgAAAAAYQbgAAAAAYAThAgAAAIARhAsAAAAARhAuAAAAABjxf4JzA8qYUFPmAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1000x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#6. Make a confusion matrix with the best performing model's\n",
    "#predictions on the validation set and the validation ground truth labels.\n",
    "from helper_functions import make_confusion_matrix\n",
    "make_confusion_matrix(val_labels, ensemble_preds_exercise)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}