pwd_ds_done.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pipreqsnb --force\n",
    "!pip3 install -q -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "K38HKbXTQbek"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "dCOG9WcZWitH"
   },
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, f1_score\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from sklearn.model_selection import train_test_split, cross_val_score, RandomizedSearchCV\n",
    "\n",
    "\n",
    "RSEED = 42\n",
    "N_JOBS = -1 #4\n",
    "N_FOLDS = 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0YjIX9Zt6CDd"
   },
   "source": [
    "I chose gradient boosting as my baseline since it's SOTA amongst classic ML algorithms. As i have 3-4 hours for this task only and don't have any GPU here, i stick to this baseline and its proper implementation. In general, CatBoost might be faster than XGBoost, but I don't have any plans for categorical features or context-based password text processing thus stick to XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "id": "jGWEP0e3Qh6Z",
    "outputId": "cab1930e-881b-4946-ca69-98d91adf5ec9"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-283a414d-043f-4392-b284-1269c0e61967\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>password</th>\n",
       "      <th>strength</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>yrtzuab476</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>yEdnN9jc1NgzkkBP</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sarita99</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Suramerica2015</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PPRbMvDIxMQ19TMo</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-283a414d-043f-4392-b284-1269c0e61967')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-283a414d-043f-4392-b284-1269c0e61967 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-283a414d-043f-4392-b284-1269c0e61967');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-53db3482-6362-45a2-bc34-2993c6e45340\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-53db3482-6362-45a2-bc34-2993c6e45340')\"\n",
       "            title=\"Suggest charts.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-53db3482-6362-45a2-bc34-2993c6e45340 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "           password  strength\n",
       "0        yrtzuab476         1\n",
       "1  yEdnN9jc1NgzkkBP         2\n",
       "2          sarita99         1\n",
       "3    Suramerica2015         2\n",
       "4  PPRbMvDIxMQ19TMo         2"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ = pd.read_csv('passwords.csv')\n",
    "df_.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ajHecbSAQ9p4",
    "outputId": "4cf46d9c-dc09-47e0-e6dc-0469f84f4da5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(password    False\n",
       " strength    False\n",
       " dtype: bool,\n",
       " False)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check fo NaNs and duplicated rows\n",
    "df_.isna().any(), df_['password'].duplicated().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "X6nzAgGIR47n",
    "outputId": "d90fb976-dddc-47aa-e2de-b4e284c732d7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    100000.000000\n",
       "mean          9.975540\n",
       "std           2.766384\n",
       "min           1.000000\n",
       "25%           8.000000\n",
       "50%           9.000000\n",
       "75%          11.000000\n",
       "max          45.000000\n",
       "Name: password, dtype: float64"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# shallow copy of original dataframe, measure sequence length (1st feature)\n",
    "df = df_.copy()\n",
    "length = df_['password'].str.len()\n",
    "length.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "id": "UOHIGaZO7iaG",
    "outputId": "70b54f3d-26b0-4b90-9809-3a990220f59c"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-0553bc61-9ab0-484b-a0e5-5dea04b8596d\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>74.278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13.428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12.294</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0553bc61-9ab0-484b-a0e5-5dea04b8596d')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-0553bc61-9ab0-484b-a0e5-5dea04b8596d button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-0553bc61-9ab0-484b-a0e5-5dea04b8596d');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-e5f27487-4f86-4e1b-a9c0-5acdb957395c\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-e5f27487-4f86-4e1b-a9c0-5acdb957395c')\"\n",
       "            title=\"Suggest charts.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-e5f27487-4f86-4e1b-a9c0-5acdb957395c button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "   counts\n",
       "1  74.278\n",
       "0  13.428\n",
       "2  12.294"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check if there are imbalanced classes\n",
    "def check_balance(dfr):\n",
    "  N = dfr.shape[0]\n",
    "  try:\n",
    "    # last column treated as target for dataframe input\n",
    "    tar = dfr[dfr.columns[-1]]\n",
    "  except AttributeError:\n",
    "    # single array input case\n",
    "    tar = dfr\n",
    "  finally:\n",
    "    return (100*tar.value_counts()/N).to_frame('counts')\n",
    "\n",
    "check_balance(df_)\n",
    "# sheesh ... our strategy is to oversample last 2 classes (later)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "id": "ZZwoND5KUnLp"
   },
   "outputs": [],
   "source": [
    "# count pattern ocurrences,\n",
    "df['fd'] = df['password'].str.count('\\d') # all digits\n",
    "df['fu'] = df['password'].str.count('[A-Z]') # all uppercase letters\n",
    "df['fl'] = df['password'].str.count('[a-z]') # all lowercase letters\n",
    "df['fx'] = df['password'].str.count('[\\W_]') # all non-word w/ underscores\n",
    "# There are umlauts, accents and some math symbols (plusminus, equality...) left"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "id": "ZMSfUyv1vnKh"
   },
   "outputs": [],
   "source": [
    "# separate target\n",
    "target = df['strength']\n",
    "# gonna use ocurrences as features thus get rid of password string column\n",
    "features_ = df.drop(['password', 'strength'], axis=1)\n",
    "# I prefer 'the rest symbols quantity' instead of 'overall length' feature\n",
    "# using both leads to multicollinearity (=> overfitting)\n",
    "features_['fr'] = length - features_.sum(axis=1, numeric_only=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2jOPeXPg9gCP"
   },
   "source": [
    "I decided to match 4 independent regular expression patterns within each password string cause these are obviously distinctive numeric features, moreover their proportions aka frequencies are independent of password length, this allows to handle various input sequences\n",
    "\n",
    "Therefore, I have 4+1 non-collinear features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 300
    },
    "id": "km339I8Cbe5J",
    "outputId": "18a4799d-96da-488f-f1f3-27427bd0f88f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-194be188-8c22-4b35-94aa-ff30208c9950\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fd</th>\n",
       "      <th>fu</th>\n",
       "      <th>fl</th>\n",
       "      <th>fx</th>\n",
       "      <th>fr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>100000.000000</td>\n",
       "      <td>100000.000000</td>\n",
       "      <td>100000.000000</td>\n",
       "      <td>100000.000000</td>\n",
       "      <td>100000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.328459</td>\n",
       "      <td>0.053249</td>\n",
       "      <td>0.614548</td>\n",
       "      <td>0.003699</td>\n",
       "      <td>0.000044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.198993</td>\n",
       "      <td>0.153707</td>\n",
       "      <td>0.221159</td>\n",
       "      <td>0.027665</td>\n",
       "      <td>0.002613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.187500</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-194be188-8c22-4b35-94aa-ff30208c9950')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-194be188-8c22-4b35-94aa-ff30208c9950 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-194be188-8c22-4b35-94aa-ff30208c9950');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-6e61bf84-bb57-47ad-b023-c7e4729fa406\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-6e61bf84-bb57-47ad-b023-c7e4729fa406')\"\n",
       "            title=\"Suggest charts.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-6e61bf84-bb57-47ad-b023-c7e4729fa406 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "                  fd             fu             fl             fx  \\\n",
       "count  100000.000000  100000.000000  100000.000000  100000.000000   \n",
       "mean        0.328459       0.053249       0.614548       0.003699   \n",
       "std         0.198993       0.153707       0.221159       0.027665   \n",
       "min         0.000000       0.000000       0.000000       0.000000   \n",
       "25%         0.187500       0.000000       0.500000       0.000000   \n",
       "50%         0.300000       0.000000       0.666667       0.000000   \n",
       "75%         0.400000       0.000000       0.777778       0.000000   \n",
       "max         1.000000       1.000000       1.000000       1.000000   \n",
       "\n",
       "                  fr  \n",
       "count  100000.000000  \n",
       "mean        0.000044  \n",
       "std         0.002613  \n",
       "min         0.000000  \n",
       "25%         0.000000  \n",
       "50%         0.000000  \n",
       "75%         0.000000  \n",
       "max         0.333333  "
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# divide by each element's overall length to get features within [0..1] range\n",
    "features = features_.apply(lambda f: f/length, axis=0)\n",
    "features.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "id": "q-wK2cMIhGsU",
    "outputId": "eaa32294-2c78-49e2-ca00-cd76ada05b97"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-e3b72405-074b-4151-897d-afa6fb036add\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>33.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>33.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>33.333333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e3b72405-074b-4151-897d-afa6fb036add')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-e3b72405-074b-4151-897d-afa6fb036add button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-e3b72405-074b-4151-897d-afa6fb036add');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "<div id=\"df-896b1025-e182-4309-aa2f-f14ac7e62708\">\n",
       "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-896b1025-e182-4309-aa2f-f14ac7e62708')\"\n",
       "            title=\"Suggest charts.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "  </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "  <script>\n",
       "    async function quickchart(key) {\n",
       "      const quickchartButtonEl =\n",
       "        document.querySelector('#' + key + ' button');\n",
       "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "      try {\n",
       "        const charts = await google.colab.kernel.invokeFunction(\n",
       "            'suggestCharts', [key], {});\n",
       "      } catch (error) {\n",
       "        console.error('Error during call to suggestCharts:', error);\n",
       "      }\n",
       "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "    }\n",
       "    (() => {\n",
       "      let quickchartButtonEl =\n",
       "        document.querySelector('#df-896b1025-e182-4309-aa2f-f14ac7e62708 button');\n",
       "      quickchartButtonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "    })();\n",
       "  </script>\n",
       "</div>\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "      counts\n",
       "1  33.333333\n",
       "2  33.333333\n",
       "0  33.333333"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# handle imbalanced classes\n",
    "ros = RandomOverSampler(random_state=RSEED)\n",
    "X, y = ros.fit_resample(X=features, y=target)\n",
    "check_balance(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "id": "1gFWJRHNEvTS"
   },
   "outputs": [],
   "source": [
    "# first of all, put aside test dataset\n",
    "# btw, we ensure that train/test parts have same distribution of labels\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,\n",
    "                                                    y,\n",
    "                                                    test_size=0.2,\n",
    "                                                    shuffle=True,\n",
    "                                                    stratify=y.values,\n",
    "                                                    random_state=RSEED,\n",
    "                                                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "id": "yh8RfCN1iDka"
   },
   "outputs": [],
   "source": [
    "cl = xgb.XGBClassifier(objective='multi:softmax',\n",
    "                       num_class = 3,\n",
    "                       n_jobs=N_JOBS,\n",
    "                       seed=RSEED,\n",
    "                       early_stopping_rounds=50,\n",
    "                       )\n",
    "\n",
    "# parameters to be passed to fit method (directly)\n",
    "cv_fp = {\"eval_set\": [(X_test, y_test)], \"verbose\": 50}\n",
    "# NB test dataset is only used for early stopping, not for learning!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "id": "NZreqM_uUJRl"
   },
   "outputs": [],
   "source": [
    "# parameter grid for gridsearch (some options turned off to make it faster)\n",
    "opt_parameters = {\n",
    "    \"learning_rate\": [0.08, 0.1, 0.12],\n",
    "    \"max_depth\": [5, 6],\n",
    "    \"n_estimators\": [1000],\n",
    "    # \"min_child_weight\": [2, 5],\n",
    "    # \"gamma\": [0, 0.01],\n",
    "    # \"subsample\": [0.9, 1.0],\n",
    "    # \"early_stopping_rounds\": [50, 100],\n",
    "}\n",
    "\n",
    "random_search = RandomizedSearchCV(\n",
    "    estimator=cl,\n",
    "    param_distributions=opt_parameters,\n",
    "    n_iter=4, # just 4 random options are chosen to save time\n",
    "    scoring=\"f1_macro\", # as we don't care about imbalanced classes anymore\n",
    "    n_jobs=N_JOBS,\n",
    "    verbose=3,\n",
    "    cv=N_FOLDS - 1, # utilizes stratified k-fold w/o shuffling by default\n",
    "    random_state=RSEED,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dsaQYHxInedV",
    "outputId": "b8e73b18-734b-4789-844e-a9fbbf323665"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 4 candidates, totalling 8 fits\n",
      "[0]\tvalidation_0-mlogloss:1.00560\n",
      "[50]\tvalidation_0-mlogloss:0.16207\n",
      "[100]\tvalidation_0-mlogloss:0.12876\n",
      "[150]\tvalidation_0-mlogloss:0.12470\n",
      "[200]\tvalidation_0-mlogloss:0.12346\n",
      "[250]\tvalidation_0-mlogloss:0.12293\n",
      "[300]\tvalidation_0-mlogloss:0.12243\n",
      "[350]\tvalidation_0-mlogloss:0.12220\n",
      "[400]\tvalidation_0-mlogloss:0.12213\n",
      "[450]\tvalidation_0-mlogloss:0.12210\n",
      "[500]\tvalidation_0-mlogloss:0.12208\n",
      "[550]\tvalidation_0-mlogloss:0.12207\n",
      "[600]\tvalidation_0-mlogloss:0.12206\n",
      "[650]\tvalidation_0-mlogloss:0.12204\n",
      "[700]\tvalidation_0-mlogloss:0.12203\n",
      "[750]\tvalidation_0-mlogloss:0.12201\n",
      "[800]\tvalidation_0-mlogloss:0.12201\n",
      "[850]\tvalidation_0-mlogloss:0.12199\n",
      "[900]\tvalidation_0-mlogloss:0.12198\n",
      "[940]\tvalidation_0-mlogloss:0.12198\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'n_estimators': 1000, 'max_depth': 5, 'learning_rate': 0.08}"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_search.fit(X_train, y_train, **cv_fp)\n",
    "optimal_cfg = random_search.best_params_\n",
    "optimal_cfg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "id": "3M_6QOFLIcoS"
   },
   "outputs": [],
   "source": [
    "optimal_cfg = {'n_estimators': 1000, 'max_depth': 5, 'learning_rate': 0.08}\n",
    "\n",
    "# reinit w/ best parameters (according to Randomized GSCV)\n",
    "cl = xgb.XGBClassifier(objective='multi:softmax',\n",
    "                       num_class = 3,\n",
    "                       n_jobs=N_JOBS,\n",
    "                       seed=RSEED,\n",
    "                       early_stopping_rounds=50,\n",
    "                       **optimal_cfg,\n",
    "                       )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "A-lOUdxAvKyi",
    "outputId": "2dcfb6b6-f91c-44ec-9f7e-320751856a31"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-mlogloss:1.00561\n",
      "[50]\tvalidation_0-mlogloss:0.16232\n",
      "[100]\tvalidation_0-mlogloss:0.12863\n",
      "[150]\tvalidation_0-mlogloss:0.12447\n",
      "[200]\tvalidation_0-mlogloss:0.12327\n",
      "[250]\tvalidation_0-mlogloss:0.12259\n",
      "[300]\tvalidation_0-mlogloss:0.12239\n",
      "[350]\tvalidation_0-mlogloss:0.12238\n",
      "[400]\tvalidation_0-mlogloss:0.12237\n",
      "[450]\tvalidation_0-mlogloss:0.12235\n",
      "[500]\tvalidation_0-mlogloss:0.12232\n",
      "[550]\tvalidation_0-mlogloss:0.12230\n",
      "[600]\tvalidation_0-mlogloss:0.12227\n",
      "[650]\tvalidation_0-mlogloss:0.12227\n",
      "[700]\tvalidation_0-mlogloss:0.12224\n",
      "[750]\tvalidation_0-mlogloss:0.12221\n",
      "[800]\tvalidation_0-mlogloss:0.12221\n",
      "[809]\tvalidation_0-mlogloss:0.12221\n",
      "[0]\tvalidation_0-mlogloss:1.00561\n",
      "[50]\tvalidation_0-mlogloss:0.16267\n",
      "[100]\tvalidation_0-mlogloss:0.12921\n",
      "[150]\tvalidation_0-mlogloss:0.12452\n",
      "[200]\tvalidation_0-mlogloss:0.12338\n",
      "[250]\tvalidation_0-mlogloss:0.12291\n",
      "[300]\tvalidation_0-mlogloss:0.12259\n",
      "[350]\tvalidation_0-mlogloss:0.12249\n",
      "[400]\tvalidation_0-mlogloss:0.12246\n",
      "[450]\tvalidation_0-mlogloss:0.12241\n",
      "[500]\tvalidation_0-mlogloss:0.12241\n",
      "[550]\tvalidation_0-mlogloss:0.12239\n",
      "[600]\tvalidation_0-mlogloss:0.12235\n",
      "[650]\tvalidation_0-mlogloss:0.12234\n",
      "[700]\tvalidation_0-mlogloss:0.12232\n",
      "[750]\tvalidation_0-mlogloss:0.12231\n",
      "[800]\tvalidation_0-mlogloss:0.12229\n",
      "[847]\tvalidation_0-mlogloss:0.12229\n",
      "[0]\tvalidation_0-mlogloss:1.00563\n",
      "[50]\tvalidation_0-mlogloss:0.16223\n",
      "[100]\tvalidation_0-mlogloss:0.12862\n",
      "[150]\tvalidation_0-mlogloss:0.12464\n",
      "[200]\tvalidation_0-mlogloss:0.12335\n",
      "[250]\tvalidation_0-mlogloss:0.12282\n",
      "[300]\tvalidation_0-mlogloss:0.12247\n",
      "[350]\tvalidation_0-mlogloss:0.12234\n",
      "[400]\tvalidation_0-mlogloss:0.12225\n",
      "[450]\tvalidation_0-mlogloss:0.12219\n",
      "[500]\tvalidation_0-mlogloss:0.12218\n",
      "[550]\tvalidation_0-mlogloss:0.12218\n",
      "[600]\tvalidation_0-mlogloss:0.12216\n",
      "[650]\tvalidation_0-mlogloss:0.12213\n",
      "[700]\tvalidation_0-mlogloss:0.12209\n",
      "[750]\tvalidation_0-mlogloss:0.12208\n",
      "[800]\tvalidation_0-mlogloss:0.12207\n",
      "[850]\tvalidation_0-mlogloss:0.12206\n",
      "[900]\tvalidation_0-mlogloss:0.12206\n",
      "[908]\tvalidation_0-mlogloss:0.12206\n"
     ]
    },
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'3-fold CV F1 score: 0.94'"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# N_FOLD cross-validation over train dataset\n",
    "kf_cv_scores = cross_val_score(cl,\n",
    "                               X_train,\n",
    "                               y_train,\n",
    "                               scoring=\"f1_macro\",\n",
    "                               cv=N_FOLDS,\n",
    "                               fit_params=cv_fp,\n",
    "                               )\n",
    "\n",
    "f\"{N_FOLDS}-fold CV F1 score: {kf_cv_scores.mean():.2f}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 439
    },
    "id": "B3lmWX6sGnV4",
    "outputId": "e52caf03-47af-46df-a287-c9086445ceb1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-mlogloss:1.00566\tvalidation_1-mlogloss:1.00560\n",
      "[100]\tvalidation_0-mlogloss:0.12770\tvalidation_1-mlogloss:0.12876\n",
      "[200]\tvalidation_0-mlogloss:0.12198\tvalidation_1-mlogloss:0.12346\n",
      "[300]\tvalidation_0-mlogloss:0.12068\tvalidation_1-mlogloss:0.12243\n",
      "[400]\tvalidation_0-mlogloss:0.12024\tvalidation_1-mlogloss:0.12213\n",
      "[500]\tvalidation_0-mlogloss:0.12011\tvalidation_1-mlogloss:0.12208\n",
      "[600]\tvalidation_0-mlogloss:0.12005\tvalidation_1-mlogloss:0.12206\n",
      "[700]\tvalidation_0-mlogloss:0.11999\tvalidation_1-mlogloss:0.12203\n",
      "[800]\tvalidation_0-mlogloss:0.11994\tvalidation_1-mlogloss:0.12201\n",
      "[900]\tvalidation_0-mlogloss:0.11989\tvalidation_1-mlogloss:0.12198\n",
      "[939]\tvalidation_0-mlogloss:0.11987\tvalidation_1-mlogloss:0.12198\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, early_stopping_rounds=50,\n",
       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "              gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=0.08, max_bin=None,\n",
       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "              max_delta_step=None, max_depth=5, max_leaves=None,\n",
       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "              n_estimators=1000, n_jobs=-1, num_class=3, num_parallel_tree=None,\n",
       "              objective=&#x27;multi:softmax&#x27;, ...)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">XGBClassifier</label><div class=\"sk-toggleable__content\"><pre>XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, early_stopping_rounds=50,\n",
       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "              gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=0.08, max_bin=None,\n",
       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "              max_delta_step=None, max_depth=5, max_leaves=None,\n",
       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "              n_estimators=1000, n_jobs=-1, num_class=3, num_parallel_tree=None,\n",
       "              objective=&#x27;multi:softmax&#x27;, ...)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "XGBClassifier(base_score=None, booster=None, callbacks=None,\n",
       "              colsample_bylevel=None, colsample_bynode=None,\n",
       "              colsample_bytree=None, early_stopping_rounds=50,\n",
       "              enable_categorical=False, eval_metric=None, feature_types=None,\n",
       "              gamma=None, gpu_id=None, grow_policy=None, importance_type=None,\n",
       "              interaction_constraints=None, learning_rate=0.08, max_bin=None,\n",
       "              max_cat_threshold=None, max_cat_to_onehot=None,\n",
       "              max_delta_step=None, max_depth=5, max_leaves=None,\n",
       "              min_child_weight=None, missing=nan, monotone_constraints=None,\n",
       "              n_estimators=1000, n_jobs=-1, num_class=3, num_parallel_tree=None,\n",
       "              objective='multi:softmax', ...)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# evaluate on two sets at once (eval_metric is same as objective by default)\n",
    "fp = {\"eval_set\": [(X_train, y_train), (X_test, y_test)], \"verbose\": 100}\n",
    "# train our final model on a whole train dataset\n",
    "cl.fit(X_train, y_train, **fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 36
    },
    "id": "-UoWoqkxlZ1b",
    "outputId": "4b50e2c6-1fdd-4c5a-f0b1-3679463da147"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'Final F1-score of XGB learner on test datset equals 0.94'"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# finally let's use X_test portion of the dataset\n",
    "y_pred = cl.predict(X_test)\n",
    "f1_fin = f1_score(y_test, y_pred, average='macro')\n",
    "f\"Final F1-score of XGB learner on test datset equals {f1_fin:.2f}\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "t8Q92b5kACDf"
   },
   "source": [
    "I think, confusion matrix is the most revealing metric for multi-class task, nevertheless I use f1-score (macro) during cross-validation and overall performance assessment. The only drawback of that metric is that it's harder to interpret its values, that is why i plot confusion matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 447
    },
    "id": "YabKZ25IYe8S",
    "outputId": "0faac73f-dd7f-4bfa-92b9-b6c714571410"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: >"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhkAAAGdCAYAAAC/02HYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABHjElEQVR4nO3deVhUZfsH8C/IKsogIAy4p6XigjviViYvuCYuFYo7SRqoiHsqiVkoZu5KWrmU5pJphoUSqFQiKoogilnuy4DIJqgDOPP7w59HzxEU8EyD8P1c17nemec858xzaF65ue/nPMdAq9VqQURERCQzQ30PgIiIiComBhlERESkEwwyiIiISCcYZBAREZFOMMggIiIinWCQQURERDrBIIOIiIh0gkEGERER6QSDDCIiItIJI30P4DH1hSP6HgKVIw5tRup7CFSO5Kjv6XsIVM4U5t/Q6fkL0i/Kdi5j29dkO9erptwEGUREROWG5qG+R1AhsFxCREREOsFMBhERkZRWo+8RVAgMMoiIiKQ0DDLkwCCDiIhIQstMhiw4J4OIiIh0gpkMIiIiKZZLZMEgg4iISIrlElmwXEJEREQ6wUwGERGRFBfjkgWDDCIiIimWS2TBcgkRERHpBDMZREREUry7RBYMMoiIiCS4GJc8WC4hIiIinWAmg4iISIrlElkwk0FERCSl1ci3lUJMTAz69esHR0dHGBgYYM+ePcX2HTduHAwMDLBs2TJRe0ZGBry9vWFpaQkrKyv4+PggNzdX1CcxMRFdu3aFmZkZ6tSpg9DQ0GfOv3PnTjRp0gRmZmZo0aIFfv3111JdC8Agg4iI6Fmah/JtpZCXlwdnZ2esXr36uf12796No0ePwtHR8Zl93t7eSE5ORmRkJMLDwxETEwNfX19hf05ODtzd3VGvXj3Ex8dj8eLFmDdvHtatWyf0OXLkCIYMGQIfHx+cOnUKnp6e8PT0xJkzZ0p1PQZarVZbqiN0RH3hiL6HQOWIQ5uR+h4ClSM56nv6HgKVM4X5N3R6fnXKYdnOZdrkzTIdZ2BggN27d8PT01PUfuPGDbi4uGD//v3o06cPAgICEBAQAAA4d+4cnJyccPz4cbRr1w4AEBERgd69e+P69etwdHTE2rVrMXv2bKhUKpiYmAAAZs6ciT179iAlJQUA8P777yMvLw/h4eHC53bs2BGtWrVCWFhYia+BmQwiIiIpGcslarUaOTk5ok2tVpdpWBqNBsOHD8e0adPQrFmzZ/bHxsbCyspKCDAAwM3NDYaGhoiLixP6dOvWTQgwAMDDwwPnz59HZmam0MfNzU10bg8PD8TGxpZqvAwyiIiIpDQa2baQkBAoFArRFhISUqZhLVq0CEZGRpg4cWKR+1UqFezs7ERtRkZGsLa2hkqlEvrY29uL+jx+/6I+j/eXFO8uISIi0qFZs2YhMDBQ1GZqalrq88THx2P58uU4efIkDAwM5BqeTjGTQUREJCVjucTU1BSWlpairSxBxh9//IG0tDTUrVsXRkZGMDIywpUrVzBlyhTUr18fAKBUKpGWliY6rrCwEBkZGVAqlUKf1NRUUZ/H71/U5/H+kmKQQUREJCVjuUQuw4cPR2JiIhISEoTN0dER06ZNw/79+wEArq6uyMrKQnx8vHBcdHQ0NBoNXFxchD4xMTEoKCgQ+kRGRqJx48aoUaOG0CcqKkr0+ZGRkXB1dS3VmFkuISIiKidyc3Pxzz//CO8vXbqEhIQEWFtbo27durCxsRH1NzY2hlKpROPGjQEATZs2Rc+ePTF27FiEhYWhoKAA/v7+8PLyEm53HTp0KIKDg+Hj44MZM2bgzJkzWL58OZYuXSqcd9KkSXjzzTexZMkS9OnTB9u2bcOJEydEt7mWBDMZREREElrtQ9m20jhx4gRat26N1q1bAwACAwPRunVrBAUFlfgcW7ZsQZMmTdCjRw/07t0bXbp0EQUHCoUCBw4cwKVLl9C2bVtMmTIFQUFBorU0OnXqhK1bt2LdunVwdnbGjz/+iD179qB58+aluh6uk0HlEtfJoKdxnQyS0vU6GQ8Swl/cqYTMWvWV7VyvGmYyiIiISCc4J4OIiEiKD0iTBYMMIiIiqVI+2IyKxiCDiIhIqpQPNqOicU4GERER6QQzGURERFIsl8iCQQYREZEUJ37KguUSIiIi0glmMoiIiKRYLpEFgwwiIiIplktkwXIJERER6QQzGURERFLMZMiCQQYREZFEaZ+eSkVjuYSIiIh0gpkMIiIiKZZLZMEgg4iISIq3sMqCQQYREZEUMxmy4JwMIiIi0glmMoiIiKRYLpEFgwwiIiIplktkwXIJERER6QQzGURERFIsl8iCQQYREZEUyyWyYLmEiIiIdIKZDCIiIilmMmTBIIOIiEiKczJkUeogo7CwEMnJyVCpVAAApVIJJycnGBsbyz44IiIienWVOMjQaDQICgrC6tWrkZ2dLdqnUCjg7++P4OBgGBpymgcREb3iWC6RRYkjgpkzZ2LdunVYuHAhLl68iLy8POTl5eHixYtYtGgR1q1bh1mzZulyrHpx4sx5+AcvQ48Rk9Gy72hEx54s8bGnzl5A63d88O6EIB2O8JEDfx7HO+Nmod2AsRjoNwd/HD9dbN9PV21Cy76j8d3PB3Q+rorGwcEeYeu/wD9XjuFGWhL+PBqOVq2bF9t/VdgiZNy98Mx25NivOh1nf8+eOBofgZu3z+DPo+Fwc39T2GdkZIRP5k/Dn0fDcU11Gsl//4k1X4VCqbTT6Zio7Lp2ccGe3Rtx9XI8CvNv4J13PPQ9pIpPq5Fvq8RKHGRs3rwZ3333HT788EPUr18f5ubmMDc3R/369eHr64vNmzdj48aNOhyqftx/oEbj1+rg43HDSnVcTu49zP5yPVycm770GI4npqDnmKnF7k84dwEzQsMw4H/dsGNFMN7u2AaTPluJC5evP9M36kg8Es//Cztrq5ceV2WjsLLEb5HbUFhYiPcGfgDX9r0w9+OFyMrKKfaYWdM/RZOGrsLWvHFXZGRk4ufdv5V5HJ27dEDCmYPF7u/g0hrrNyzFls0/4q0u/fFr+O/4/oc1aNr0dQCAeVUzODs3wxeLVqN7V0+M9PbH66+/hi3bw8o8JtItC4uqSEw8iwmTZut7KJWHRiPfVomVuFxy9+5dODo6FrvfwcEBeXl5sgyqPOnariW6tmtZ6uMWrN6E3m92hKGhIQ4eFWc/NBoNvv3xV/y4/zDuZGajnqMSvl794N6lfZnGuGVvJDq3bYHRg3oBAPyHD0RsQjK2hUdhrv9IoV9qeiZCvtqCsPlT4B+8tEyfVZlNmuyLGzduwX/8TKHt6pVnA7mn3c3Jxd2cXOF9775usLJSYMv3u4Q2AwMDTAr0xchR78POvib+/ecSvli0Bnt/jijTOD8cPxJRv/+Blcu/BgB8vmAZ3nq7Mz74cDimBAThbk4uBvYfJTpm+tRgRB3+CbVqO+DG9Vtl+lzSnYj9BxGxv/jAkqi8KnEm46233sLUqVORnp7+zL709HTMmDEDb731lpxje2XtifwD11W3MW5o/yL3f71zH36JPoK5H43A7jULMNzTHR8vWYcTSSll+rzTKf/CpZWTqK1Tm+Y4nfKv8F6j0eDjL9dh1MCeaFSvVpk+p7Lr1bsHEk6ewYbNK3D+4lEc+vNnjBj1XqnOMWzEuzh88AiuX7sptE2eMg5eQwZgSkAQOnXojbWrNyLs6y/QqXOHMo2zfYfWOHzwiKgt+vc/0L5Dq2KPsbSsDo1Gg5zsu2X6TKIKh+USWZQ4kxEWFobevXvDwcEBLVq0gL29PQAgNTUVSUlJcHJyQnh4eInOpVaroVarxY35+TA1MSn5yMupKzdUWLbpR2xcNAtGVao8sz+/oABf7wjH+gXT4Ny0EQCgttIOJ8/+jZ0Rh9CuRZNSf2Z6ZjZsrCxFbTZWCqRnPZmg++2Pv8KoShV4v/O/Up+fHqlXvw5GfzAUa1Z9iy+/CEObti0QEjoX+fkF2LZ19wuPVyrt4Pa/bvAdEyi0mZiYYPLUcRj4zkgcP5YAALhy+Ro6urbFqDFeOPLXsVKP087eFmlp4j8G0tLSYWdfs8j+pqYm+GT+NOzaGY67d3OL7ENU6VTyModcShxk1KlTB6dPn8b+/ftx9OhR4RbWDh064PPPP4e7u3uJ7ywJCQlBcHCwqG22/xjMnehTiqGXPw8fajDzi6/w0VBP1K+lLLLP1ZtpeKDOh+/cL0TtBYWFaPJaPeG9y+BxwmuNRoP8gkJRW9+3XEWlkOc5+89lbNkbie3L58HAwKA0l0RPMTQ0QMKpM1gQ/CUAICnxLJo0fQOjfYaUKMjw8h6A7Owc7Av/XWh77bW6sLCoil0/bxT1NTExRtLpc8L7q7cShNdVqlSBqamJqG3n9r2YElD6CcZGRkb4dvMKGBgYYOrkT0p9PBHR85Q4yJg/fz6mTp2KXr16oVevXi/1obNmzUJgYKC48VrJ79oor/LuP0DyhctI+fcqQsK+BwBotFpotVq0fscHYZ9OgbmZKQBg9ScBsLOpITrexPjJf46dK54EYUnnL2Lpxp34NmSG0GZR1Vx4bVtDgTuSyYd3srJha6UAAMQn/42M7LvwGP1k8uhDjQZLvtmGLT8fQMS34oCHipaquo3zKf+I2v4+/y/69Xcv0fHewwZjxw8/o6CgQGizqGYBAPAa7Itbt1Si/vnqfOH1m53fEV63bdcK8+ZPQ7/e3kLb0/M+0lLTYWdnKzqXnZ0t0lJvi9oeBRjLUaeOI/r3HcEsBtHTmMmQRYmDjODgYIwbNw5Vq1Z96Q81NTWFqampqE1dAUol1aqaYdeqT0Vt23+NxrHEc1gy0w+1lDWh1WhgYmyEW7cznlsaqetoL7xOTc+EURVDUdvTnJs0RFzCWQx/6pfd0VPJcG7SEADQr3sndHQWz9kYH7QEfd/uhP5uXUp9nZVV3NGTaPR6A1Fbo0b1RfMritO5Swc0bFQf32/eKWo/n/IPHjxQo3Ydh+eWRi5dvCq8dnRUorCwUNT2tOPHTqHbW64IW7NRaHvr7c5COQZ4EmA0bFgf7/QZjsyMrBdeA1GlotXqewQVQomDDG0l/YHfu/8AV2+lCe9vpN5GysWrUFSzgIOdDZZv3InUO1n4fMpYGBoa4vX6tUXHWyssYWpsLGofObAnFn/9AzRaDdo4vYG7efeQcO4fWFQ1Q/8epf+l7/3O/zBm5iJs+ikC3do747eYOCT/cxlB/qMAAFaW1WBlWU10jJFRFdjUUKBBbYdSf15ltXb1BkT8vh2Tp47Dnp9+RZu2zhgx+n1MnjhX6DN33hQ4ONjjow+ni44dNuJdnDiegHPnLojac3PzsGrFN/hs4ccwNDTE0dh4WFpWg0vHtrh7N7dEZRipr9Zuwi+/bYHfhDE4sP8QBg7qg1atm2PyhDkAHgUYG79fCWfnZvB61xdVDA2FzEdmZrYo00Llg4VFVTRq9CTAbVC/LpydmyEjIxPXShDkEulLqZYVr4z1/OQLl+Hz8SLh/eKvtwEA3unRGQsmf4DbmdlQ3b5TqnP6DxuIGpbV8c3OfQhWbUR1i6po2rAePnivb5nG2Krp61g47UOs/O4nrNi8C3Ud7bF89oRnAh56OadOJmH4UD8EzZuCaTP8cfXKdcye+Rl+3LFX6GOvtEPtOuJbvatbVkO//h74eMaCIs/7+adLcSc9AwFTPkT9+nWQnX0XiQnJ+HJJ2datOBZ3Cr5jAvFx0GTM+WQKLv57GcOGfCQEOA6O9ujdxw0A8EfsL6Jj+/Xyxl9/ln6yKelWu7bOiPr9R+H9ki/mAQA2bd4Bnw8m62lUFRzLJbIw0JYwRWFoaAiFQvHCQCMjI6NMA1FfOPLiTlRpOLQp2aRWqhxy1Pf0PQQqZwrzb+j0/Pe3zH1xpxIy9/70xZ0qqFJlMoKDg6FQKHQ1FiIiokotJiYGixcvRnx8PG7duoXdu3fD09MTAFBQUIA5c+bg119/xcWLF6FQKODm5oaFCxeKFsvMyMjAhAkT8Msvv8DQ0BCDBg3C8uXLUa3ak7J5YmIi/Pz8cPz4cdSsWRMTJkzA9OniMu/OnTsxd+5cXL58Ga+//joWLVqE3r17l+p6ShVkeHl5wc6OzzcgIqIKTk+LaOXl5cHZ2RljxozBwIEDRfvu3buHkydPYu7cuXB2dkZmZiYmTZqEd955BydOnBD6eXt749atW4iMjERBQQFGjx4NX19fbN26FQCQk5MDd3d3uLm5ISwsDElJSRgzZgysrKzg6+sLADhy5AiGDBmCkJAQ9O3bF1u3boWnpydOnjyJ5s2Lf16TVInLJVWqVMGtW7d0FmSwXEJPY7mEnsZyCUnpvFyyWb4HfpqPCCnTcQYGBqJMRlGOHz+ODh064MqVK6hbty7OnTsHJycnHD9+HO3atQMAREREoHfv3rh+/TocHR2xdu1azJ49GyqVCib/f2fnzJkzsWfPHqSkPFp5+v3330deXp5okc2OHTuiVatWCAsr+XyxEi8rXlnvLiEiokpIq5VtU6vVyMnJEW3PrHpdRtnZ2TAwMICVlRUAIDY2FlZWVkKAAQBubm4wNDREXFyc0Kdbt25CgAEAHh4eOH/+PDIzM4U+bm5uos/y8PBAbGxsqcZX4iBDo9GwVEJERFRKISEhUCgUoi0kpGzZjac9ePAAM2bMwJAhQ2Bp+ejREiqV6pnf1UZGRrC2thZW6lapVMKjQR57/P5FfR7vL6lSzckgIiKqFGS8hbWoVa6lC1KWVkFBAd577z1otVqsXbv2pc6lSwwyiIiIpGQMMopa5fplPA4wrly5gujoaCGLAQBKpRJpaWmi/oWFhcjIyIBSqRT6pKamivo8fv+iPo/3l1SJyyVERESkX48DjAsXLuD333+HjY2NaL+rqyuysrIQHx8vtEVHR0Oj0cDFxUXoExMTI1rdNzIyEo0bN0aNGjWEPlFRUaJzR0ZGwtXVtVTjZZBBREQkpdXIt5VCbm4uEhISkJCQAAC4dOkSEhIScPXqVRQUFGDw4ME4ceIEtmzZgocPH0KlUkGlUiE//9EDFZs2bYqePXti7NixOHbsGP766y/4+/vDy8tLWEtj6NChMDExgY+PD5KTk7F9+3YsX75cVNKZNGkSIiIisGTJEqSkpGDevHk4ceIE/P39S3U9Jb6FVdd4Cys9jbew0tN4CytJ6foW1nvr5Fuuvarv0hL3PXToELp37/5M+8iRIzFv3jw0aNCgiKOAgwcP4q233gLwaDEuf39/0WJcK1asKHYxLltbW0yYMAEzZswQnXPnzp2YM2eOsBhXaGhoqRfjYpBB5RKDDHoagwySqqhBRkXDiZ9ERERSfECaLBhkEBERSelpWfGKhhM/iYiISCeYySAiIpLSlIvpiq88BhlERERSnJMhCwYZREREUgwyZME5GURERKQTzGQQERFJlY8lpF55DDKIiIikWC6RBcslREREpBPMZBAREUnxFlZZMMggIiKS4oqfsmC5hIiIiHSCmQwiIiIplktkwSCDiIhIQsu7S2TBcgkRERHpBDMZREREUiyXyIJBBhERkRTvLpEFgwwiIiIpZjJkwTkZREREpBPMZBAREUnx7hJZMMggIiKSYrlEFiyXEBERkU4wk0FERCTFu0tkwSCDiIhIiuUSWbBcQkRERDrBTAYREZEEn10iDwYZREREUiyXyILlEiIiItIJZjKIiIikmMmQBYMMIiIiKd7CKgsGGURERFLMZMiCczKIiIhIJ5jJICIiktAykyELBhlERERSDDJkwXIJERER6QQzGURERFJc8VMWDDKIiIikWC6RBcslRERE5URMTAz69esHR0dHGBgYYM+ePaL9Wq0WQUFBcHBwgLm5Odzc3HDhwgVRn4yMDHh7e8PS0hJWVlbw8fFBbm6uqE9iYiK6du0KMzMz1KlTB6Ghoc+MZefOnWjSpAnMzMzQokUL/Prrr6W+HgYZREREUhqtfFsp5OXlwdnZGatXry5yf2hoKFasWIGwsDDExcXBwsICHh4eePDggdDH29sbycnJiIyMRHh4OGJiYuDr6yvsz8nJgbu7O+rVq4f4+HgsXrwY8+bNw7p164Q+R44cwZAhQ+Dj44NTp07B09MTnp6eOHPmTKmux0Cr1ZaLnJD6whF9D4HKEYc2I/U9BCpHctT39D0EKmcK82/o9Pw5H3rIdi7Lr/aX6TgDAwPs3r0bnp6eAB5lMRwdHTFlyhRMnToVAJCdnQ17e3ts3LgRXl5eOHfuHJycnHD8+HG0a9cOABAREYHevXvj+vXrcHR0xNq1azF79myoVCqYmJgAAGbOnIk9e/YgJSUFAPD+++8jLy8P4eHhwng6duyIVq1aISwsrMTXwEwGERGRDqnVauTk5Ig2tVpd6vNcunQJKpUKbm5uQptCoYCLiwtiY2MBALGxsbCyshICDABwc3ODoaEh4uLihD7dunUTAgwA8PDwwPnz55GZmSn0efpzHvd5/DklxSCDiIhISsZySUhICBQKhWgLCQkp9ZBUKhUAwN7eXtRub28v7FOpVLCzsxPtNzIygrW1tahPUed4+jOK6/N4f0nx7hIiIiIpGe8umTVrFgIDA0Vtpqamsp2/PGOQQUREJCHnsuKmpqayBBVKpRIAkJqaCgcHB6E9NTUVrVq1EvqkpaWJjissLERGRoZwvFKpRGpqqqjP4/cv6vN4f0mVmyCjy5sz9T0EKkdubPPT9xCoHKk+YLG+h0Ckdw0aNIBSqURUVJQQVOTk5CAuLg7jx48HALi6uiIrKwvx8fFo27YtACA6OhoajQYuLi5Cn9mzZ6OgoADGxsYAgMjISDRu3Bg1atQQ+kRFRSEgIED4/MjISLi6upZqzJyTQUREJKWnW1hzc3ORkJCAhIQEAI8meyYkJODq1aswMDBAQEAAFixYgL179yIpKQkjRoyAo6OjcAdK06ZN0bNnT4wdOxbHjh3DX3/9BX9/f3h5ecHR0REAMHToUJiYmMDHxwfJycnYvn07li9fLirpTJo0CREREViyZAlSUlIwb948nDhxAv7+/qW6nnKTySAiIio39LSq+IkTJ9C9e3fh/eNf/CNHjsTGjRsxffp05OXlwdfXF1lZWejSpQsiIiJgZmYmHLNlyxb4+/ujR48eMDQ0xKBBg7BixQphv0KhwIEDB+Dn54e2bdvC1tYWQUFBorU0OnXqhK1bt2LOnDn4+OOP8frrr2PPnj1o3rx5qa6n3KyT0d6xm76HQOVIzPqB+h4ClSMsl5CUrtfJyB7eQ7ZzKb6Lku1crxpmMoiIiCTknPhZmTHIICIikmKQIQtO/CQiIiKdYCaDiIhISk8TPysaBhlEREQSnJMhD5ZLiIiISCeYySAiIpJiuUQWDDKIiIgkWC6RB4MMIiIiKWYyZME5GURERKQTzGQQERFJaJnJkAWDDCIiIikGGbJguYSIiIh0gpkMIiIiCZZL5MEgg4iISIpBhixYLiEiIiKdYCaDiIhIguUSeTDIICIikmCQIQ8GGURERBIMMuTBORlERESkE8xkEBERSWkN9D2CCoFBBhERkQTLJfJguYSIiIh0gpkMIiIiCa2G5RI5MMggIiKSYLlEHiyXEBERkU4wk0FERCSh5d0lsmCQQUREJMFyiTxYLiEiIiKdYCaDiIhIgneXyINBBhERkYRWq+8RVAwMMoiIiCSYyZAH52QQERGRTjCTQUREJMFMhjwYZBAREUlwToY8WC4hIiIinWAmg4iISILlEnkwyCAiIpLgsuLyYLmEiIiIdIJBBhERkYRWI99WGg8fPsTcuXPRoEEDmJubo2HDhvj000+hfWomqlarRVBQEBwcHGBubg43NzdcuHBBdJ6MjAx4e3vD0tISVlZW8PHxQW5urqhPYmIiunbtCjMzM9SpUwehoaFl/nkVh0EGERGRhEZrINtWGosWLcLatWuxatUqnDt3DosWLUJoaChWrlwp9AkNDcWKFSsQFhaGuLg4WFhYwMPDAw8ePBD6eHt7Izk5GZGRkQgPD0dMTAx8fX2F/Tk5OXB3d0e9evUQHx+PxYsXY968eVi3bt3L//CewjkZRERE5cSRI0fQv39/9OnTBwBQv359/PDDDzh27BiAR1mMZcuWYc6cOejfvz8AYPPmzbC3t8eePXvg5eWFc+fOISIiAsePH0e7du0AACtXrkTv3r3xxRdfwNHREVu2bEF+fj6+/fZbmJiYoFmzZkhISMCXX34pCkZeFjMZREREElqtgWybWq1GTk6OaFOr1UV+bqdOnRAVFYW///4bAHD69Gn8+eef6NWrFwDg0qVLUKlUcHNzE45RKBRwcXFBbGwsACA2NhZWVlZCgAEAbm5uMDQ0RFxcnNCnW7duMDExEfp4eHjg/PnzyMzMlO3nyCCDiIhIQqsxkG0LCQmBQqEQbSEhIUV+7syZM+Hl5YUmTZrA2NgYrVu3RkBAALy9vQEAKpUKAGBvby86zt7eXtinUqlgZ2cn2m9kZARra2tRn6LO8fRnyIHlEiIiIgk5V/ycNWsWAgMDRW2mpqZF9t2xYwe2bNmCrVu3CiWMgIAAODo6YuTIkfIN6j/CIIOIiEiHTE1Niw0qpKZNmyZkMwCgRYsWuHLlCkJCQjBy5EgolUoAQGpqKhwcHITjUlNT0apVKwCAUqlEWlqa6LyFhYXIyMgQjlcqlUhNTRX1efz+cR85sFxCREQkIWe5pDTu3bsHQ0Pxr+YqVapAo3l0L2yDBg2gVCoRFRUl7M/JyUFcXBxcXV0BAK6ursjKykJ8fLzQJzo6GhqNBi4uLkKfmJgYFBQUCH0iIyPRuHFj1KhRo3Q/rOdgkEFERCShr1tY+/Xrh88++wz79u3D5cuXsXv3bnz55ZcYMGAAAMDAwAABAQFYsGAB9u7di6SkJIwYMQKOjo7w9PQEADRt2hQ9e/bE2LFjcezYMfz111/w9/eHl5cXHB0dAQBDhw6FiYkJfHx8kJycjO3bt2P58uXPlHVeFsslRERE5cTKlSsxd+5cfPTRR0hLS4OjoyM+/PBDBAUFCX2mT5+OvLw8+Pr6IisrC126dEFERATMzMyEPlu2bIG/vz969OgBQ0NDDBo0CCtWrBD2KxQKHDhwAH5+fmjbti1sbW0RFBQk6+2rAGCg1ZaPB9q2d+ym7yFQORKzfqC+h0DlSPUBi/U9BCpnCvNv6PT8SQ36yXauFpd+ke1crxrZyiWFhYW4evWqXKcjIiLSG61Wvq0yky3ISE5ORoMGDeQ6HREREb3iOCfjOUb5e6N7726o16ge1A/USDxxBqs+C8OVf68Ve8xrb9THh9N80KTlG3Cs44Avg1bih6936nysPfq+hXHTfeBQW4lrl25g5WdhOBJ9VNg/dspouPd/G/aOdijIL0RK0nmsWbgeyafO6XxsFUn8vzex6WACzl2/jds59/Dl6J54u0XxwfWpi7ewLPwoLqdl4kF+IRysq2OQqxOGv+ms03EeSPgXayKO4WbGXdS1VWBS347o6lSvyL4Ldh7Gj7FnMbV/JwzT8bjoxbp2ccGUKePRpnULODoqMXDwGOzdu1/YHzQ3EO+91x91ajsiPz8fJ08mYW7QIhw7fkqPo654Sjthk4pW4kxGmzZtnrs9vqe3Imnj2go7N+7GmL7j4O8VCCMjI6z8YQnMzM2KPcbM3Aw3rt7Eqs+/QnrqHdnG8XPc9mL3t2zXHAvWBOHnH/ZhmPsHOBzxB7749jM0bPzkl9/Vi9ewePYyDHl7FMZ6+uHmNRVW/bAEVtYKWcZYWdzPL8AbjjaYNbBrifqbmxjBq0tzfOPviZ9memGsW1us/u0Yfow9W+YxHP/nBnp9+n2x+xMuqTDr+0h4dmiCbVPeRfcWDTB5QwT+ufXs9zE68SISr6SipqVFmcdD8rKwqIrExLOYMGl2kfv/vnARkybNQas2PfBm9wG4fOUafvt1K2xtrf/jkVZsci4rXpmVOJNx9uxZeHl5FVsSuXXrlrDWekUx0Xua6H1wwOeIPPMLmrZsjFNxp4s85uzpFJw9nQIA8P/4wyL7GBgYYKTfUHgOewc2Na1x9eI1fLNsE6L3HS7TOL0+GIzYg8fw/dptAICwxd+gQ7d2eHf0QCycuQQAsH/376Jjls1bBc+hffG6U0Mc//NkmT63MurStB66NC06I1CUJrVrokntmsL7WtaWiEq6iFMXb2GwqxMAQKPRYkP0Kew6ehZ3cu6hXk0rjHVvi/85NyzTGLf+kYhOTepi1NutAQB+vTrg6Plr2PbnGcx5902hX2pWLhbu/hNrPuyLCet/LdNnkfwi9h9ExP6Dxe7ftm2P6P3UacHwGTMULVs4IfrgnzoeHVHplDjIaN68OVxcXDB+/Pgi9yckJGD9+vWyDaw8qmZZDQCQk5XzUucZNWEYeg1yx8IZX+Dapeto3dEZ81fOQdadLJw8WnTw8jwt2jbD1q92iNqOHj6GNz2K/mvbyNgIA4a9g7vZd/H32X/LdA1UNinXb+P0ZRX8enUQ2r6JOolf4//GnMHdULemFeL/vYnZW6JQw8Ic7Ro5lvozEi+nYtibLUVtrk3q4FDSZeG9RqPFnK1RGNm9FRop+Rfwq8rY2BhjP/BGVlY2Ticm63s4FUpln7AplxIHGZ07d8b58+eL3V+9enV061ay21DVavUzT6DTaDUwNCi/a4MZGBggMHgCEo4l4t/zl8p8HmMTY4yeOAx+7wciKf7RPwo3rt6Cc4eWGDD8nTIFGTY1rXEnPUPUlnE7EzZ24l8eXdxc8dnaT2Bmbob01Dvw95qC7IzsMl8LlZx78GZk5t7HQ40W4zzaYWDHR1mM/MKH+CbqJL4a1w/O9R8t5VvbxhIJl27hx9jkMgUZ6XfvwaZ6VVGbTfWqSL97T3i/IfoUqhgaYmjXFi9xVaQvfXq7Ycv3a1C1qjlu3UpFz15DcOeOfE/OJM7JkEuJg4zly5c/d3/Dhg1x8GDxKb6nhYSEIDg4WNTmUK0ualUveRr6vzb988lo2KQBxnr6v9R56tSvBfOq5li1bYmo3djYGOfPXBDeH74QIbw2NKwCE1NjUdtvuyKFUkhJnfjrFLz/5wMrawU8vfvh86+CMbrPh8i8k1W2i6ES2+DviXvqAiReScWKfUdRx1aBXm1ex9X0bDzIL8S4MPF99AUPNWhSy1Z47zrzSZZQo9Ei/+FDUVuftm+ISiHPc/babWz9IxE/BL4LAwP+Q/oqOnjoL7Rt7w5bG2v4+AzFD1vD0KlLX9y+Lc88MEKln0shlxIHGfPnz8fUqVNRtWrVF3d+gaKeSNe9ce+XPq+uTPssAF3/1wm+AyYg7dbtlzqXuYU5AGDy8BlIU6WL9hXkP1lD3vt/PsLr5q2d4D/7Q4wbPEloy7ubJ7y+czsDNpJJX9Y1a+BOmji78eD+A1y/fAPXL9/AmZNnsevPreg/pA82rtryUtdEL1bLxhIA8LqjDTJy7yNs/3H0avM67qsf/Tdf+UEf2CnEky9NjKoIr7dPeU94nXQ1FcvDj+Lrj/oLbdXMjIXXttWr4s5TWQsAuHP3Hmz/P7tx8uJNZOTeR69PvxP2P9Ro8eXeWGyJScJvc4e97OWSjt27dx///nsZ//57GXHHTuJc8p8YM3oIFoWu0vfQiERKHGQEBwdj3LhxsgQZRT2RrryWSqZ9FoC3enbFuMGTcPParZc+36W/L0P9QA37WvbPLY1cv/xkNTs7h5p4+PChqO1pSfHJaN+1jehWWZdu7YVyTHEMDQ1gbGpSyiugl6XRaJFf+OhhR6/Z14CJURWosu4+tzRSt+aTu4BSs3NRxdBQ1Pa0lvXtcezCDdHtqEf/vo6W9e0BAH3bNUbHN2qLjhn/1T70bfcG+ndoXObrIv0xNDSAKf+/LCuWS+RR4iCjnKw+/p+a8flkeAxww9TRH+Ne7j3Y1HyULci9mwv1g3wAwLzlH+O2Kh2rQ9YBeDSp8rU36gN4VAKp6WCLN5o1wr28+7h++Qbu5d3H92HbERjsD0NDQyQcS0Q1y2pwbt8ceXfvYd/OiCLH8jzbvv4RX+1aAe8P38efUbFw798DTVs2xufTHi3FbGZuhjGThiPmwF9IT70DK2sF3h09ADWVtoj6pWQlLnrknroAV9OfzGO5kZGDlBvpUFQ1hUON6lgRfhRpOXlYMLQHAGDbn2fgUKMa6ttZAQBO/nsLmw8lYMj/z4WwMDPBiLec8cXPR6DRatG6gQNyH+Qj4ZIKFmbGeKd9k1KPcWjXlvhg9c/YfCgBXZvWQ8Spf3D22m0E/X85xcrCDFYW4tuwjaoYwqa6Oerbyff0RSobC4uqaNToyV18DerXhbNzM2RkZOLOnUx8PGsSfvnlAG6pUmFrY43x40ehVi0lftwVrsdRVzyV7zeebpRqMa7KVr8dPOrRU++++mmlqD044HOE73gUDChr2UOrefJ1rGlviy2R3wrvh48fguHjhyD+yCmh3BEW+jWy7mRh1ARv1Ko7DXdzcnE+6W9sWFH82gfPk3jiDOb4zcf4GR/go5ljce3SdUwdM1uYoKrRaFC/UT30ebcnrKwVyM7MwdnTKfAdMAEX/75cps+srJKvpWHsmr3C+yU/HwEA9GvfGJ8OeRu3797DrcxcYb9Wq8WKfUdxI+MujAwNUdvGEpP6dsRg12ZCH79eHVCjmjm+jTqF63cOo7q5CZrWqgkftzZlGmOrBkp8PswNq3+Lw8p9cahbU4Glo3uikYNNGa+a/kvt2joj6vcfhfdLvpgHANi0eQc+8puJxo0bYviwdbC1tcadO5k4EX8ab3UfiLNnK9YSAlQxlPgBaYaGhlAoFC8MNDIyMp67vzh8QBo9jQ9Io6fxAWkkpesHpB1xGCTbuTrd2iXbuV41pcpkBAcHQ6HgCpFERFSx8e4SeZQqyPDy8oKdnZ2uxkJEREQVSImDjMo2H4OIiCovjb4HUEHw7hIiIiIJLfiHtRxKHGRoNIzriIiIqORKNSeDiIioMtAweS8LBhlEREQSGpZLZMEgg4iISIJzMuRRPh8YQkRERK88ZjKIiIgkeKuDPBhkEBERSbBcIg+WS4iIiEgnmMkgIiKSYLlEHgwyiIiIJBhkyIPlEiIiItIJZjKIiIgkOPFTHgwyiIiIJDSMMWTBcgkRERHpBDMZREREEnx2iTwYZBAREUnwIazyYJBBREQkwVtY5cE5GURERKQTzGQQERFJaAw4J0MODDKIiIgkOCdDHiyXEBERkU4wk0FERCTBiZ/yYCaDiIhIQmMg31ZaN27cwLBhw2BjYwNzc3O0aNECJ06cEPZrtVoEBQXBwcEB5ubmcHNzw4ULF0TnyMjIgLe3NywtLWFlZQUfHx/k5uaK+iQmJqJr164wMzNDnTp1EBoaWqaf1fMwyCAiIionMjMz0blzZxgbG+O3337D2bNnsWTJEtSoUUPoExoaihUrViAsLAxxcXGwsLCAh4cHHjx4IPTx9vZGcnIyIiMjER4ejpiYGPj6+gr7c3Jy4O7ujnr16iE+Ph6LFy/GvHnzsG7dOlmvh+USIiIiCX2t+Llo0SLUqVMHGzZsENoaNGggvNZqtVi2bBnmzJmD/v37AwA2b94Me3t77NmzB15eXjh37hwiIiJw/PhxtGvXDgCwcuVK9O7dG1988QUcHR2xZcsW5Ofn49tvv4WJiQmaNWuGhIQEfPnll6Jg5GUxk0FERCShlXFTq9XIyckRbWq1usjP3bt3L9q1a4d3330XdnZ2aN26NdavXy/sv3TpElQqFdzc3IQ2hUIBFxcXxMbGAgBiY2NhZWUlBBgA4ObmBkNDQ8TFxQl9unXrBhMTE6GPh4cHzp8/j8zMzLL/4CQYZBAREelQSEgIFAqFaAsJCSmy78WLF7F27Vq8/vrr2L9/P8aPH4+JEydi06ZNAACVSgUAsLe3Fx1nb28v7FOpVLCzsxPtNzIygrW1tahPUed4+jPkwHIJERGRhJyPep81axYCAwNFbaampkV/rkaDdu3a4fPPPwcAtG7dGmfOnEFYWBhGjhwp36D+I8xkEBERSWhk3ExNTWFpaSnaigsyHBwc4OTkJGpr2rQprl69CgBQKpUAgNTUVFGf1NRUYZ9SqURaWppof2FhITIyMkR9ijrH058hBwYZREREEnLOySiNzp074/z586K2v//+G/Xq1QPwaBKoUqlEVFSUsD8nJwdxcXFwdXUFALi6uiIrKwvx8fFCn+joaGg0Gri4uAh9YmJiUFBQIPSJjIxE48aNRXeyvCwGGUREROXE5MmTcfToUXz++ef4559/sHXrVqxbtw5+fn4AAAMDAwQEBGDBggXYu3cvkpKSMGLECDg6OsLT0xPAo8xHz549MXbsWBw7dgx//fUX/P394eXlBUdHRwDA0KFDYWJiAh8fHyQnJ2P79u1Yvnz5M2Wdl8U5GURERBJyzskojfbt22P37t2YNWsW5s+fjwYNGmDZsmXw9vYW+kyfPh15eXnw9fVFVlYWunTpgoiICJiZmQl9tmzZAn9/f/To0QOGhoYYNGgQVqxYIexXKBQ4cOAA/Pz80LZtW9ja2iIoKEjW21cBwECr1ZaL58C0d+ym7yFQORKzfqC+h0DlSPUBi/U9BCpnCvNv6PT862sPk+1cY69/L9u5XjUslxAREZFOsFxCREQkwQekyYNBBhERkYRWT3MyKhqWS4iIiEgnmMkgIiKSYLlEHgwyiIiIJBhkyIPlEiIiItIJZjKIiIgkysUCUhUAgwwiIiIJfa34WdEwyCAiIpLgnAx5cE4GERER6QQzGURERBLMZMiDQQYREZEEJ37Kg+USIiIi0glmMoiIiCR4d4k8GGQQERFJcE6GPFguISIiIp1gJoOIiEiCEz/lwSCDiIhIQsMwQxblJsg4lf6vvodA5Uj1AYv1PQQqR+7f/EPfQyCiMig3QQYREVF5wYmf8mCQQUREJMFiiTwYZBAREUkwkyEP3sJKREREOsFMBhERkQRX/JQHgwwiIiIJ3sIqD5ZLiIiISCeYySAiIpJgHkMeDDKIiIgkeHeJPFguISIiIp1gJoOIiEiCEz/lwSCDiIhIgiGGPFguISIiIp1gJoOIiEiCEz/lwSCDiIhIgnMy5MEgg4iISIIhhjw4J4OIiIh0gpkMIiIiCc7JkAeDDCIiIgktCyayYLmEiIioHFq4cCEMDAwQEBAgtD148AB+fn6wsbFBtWrVMGjQIKSmpoqOu3r1Kvr06YOqVavCzs4O06ZNQ2FhoajPoUOH0KZNG5iamqJRo0bYuHGjTq6BQQYREZGERsatLI4fP46vvvoKLVu2FLVPnjwZv/zyC3bu3InDhw/j5s2bGDhwoLD/4cOH6NOnD/Lz83HkyBFs2rQJGzduRFBQkNDn0qVL6NOnD7p3746EhAQEBATggw8+wP79+8s42uIZaLXacpETMjKppe8hEFE5df/mH/oeApUzxrav6fT8H9V/T7Zzrbm8o1T9c3Nz0aZNG6xZswYLFixAq1atsGzZMmRnZ6NmzZrYunUrBg8eDABISUlB06ZNERsbi44dO+K3335D3759cfPmTdjb2wMAwsLCMGPGDNy+fRsmJiaYMWMG9u3bhzNnzgif6eXlhaysLERERMh23QAzGURERDqlVquRk5Mj2tRqdbH9/fz80KdPH7i5uYna4+PjUVBQIGpv0qQJ6tati9jYWABAbGwsWrRoIQQYAODh4YGcnBwkJycLfaTn9vDwEM4hJwYZREREEloZt5CQECgUCtEWEhJS5Odu27YNJ0+eLHK/SqWCiYkJrKysRO329vZQqVRCn6cDjMf7H+97Xp+cnBzcv3//xT+cUuDdJURERBJyrvg5a9YsBAYGitpMTU2f6Xft2jVMmjQJkZGRMDMzk+3z9YmZDCIiIh0yNTWFpaWlaCsqyIiPj0daWhratGkDIyMjGBkZ4fDhw1ixYgWMjIxgb2+P/Px8ZGVliY5LTU2FUqkEACiVymfuNnn8/kV9LC0tYW5uLtdlA2CQQURE9Ax93F3So0cPJCUlISEhQdjatWsHb29v4bWxsTGioqKEY86fP4+rV6/C1dUVAODq6oqkpCSkpaUJfSIjI2FpaQknJyehz9PneNzn8TnkxHIJERGRhD4W46pevTqaN28uarOwsICNjY3Q7uPjg8DAQFhbW8PS0hITJkyAq6srOnbsCABwd3eHk5MThg8fjtDQUKhUKsyZMwd+fn5C9mTcuHFYtWoVpk+fjjFjxiA6Oho7duzAvn37ZL8mBhlEREQS5XVZ8aVLl8LQ0BCDBg2CWq2Gh4cH1qxZI+yvUqUKwsPDMX78eLi6usLCwgIjR47E/PnzhT4NGjTAvn37MHnyZCxfvhy1a9fG119/DQ8PD9nHy3UyiKjc4zoZJKXrdTLG1B8s27m+vfyjbOd61TCTQUREJMFnl8iDQQYREZFEeS2XvGp4dwkRERHpBDMZREREEpryMV3xlccgg4iISIIhhjxYLiEiIiKdYCaDiIhIQs5nl1RmDDKIiIgkeAurPFguISIiIp1gJoOIiEiC62TIg0EGERGRBOdkyINBBhERkQTnZMiDczKIiIhIJ5jJICIikuCcDHkwyCAiIpLQcllxWbBcQkRERDrBTAYREZEE7y6RB4MMIiIiCc7JkAfLJURERKQTzGQQERFJcJ0MeTDIICIikuCcDHmwXEJEREQ6wUwGERGRBNfJkAeDDCIiIgneXSIPBhlEREQSnPgpj1LNyVizZg3c3Nzw3nvvISoqSrQvPT0dr732mqyDq0zGjxuJf/4+itycf3Hkz1/Qvl0rfQ+J9IzfifLvREIS/KZ/gu7veKN5516Iijny3P7HTiaieedez2zpdzJ0Os790X+g35CxaNP9HQwYPh4xR44V2zc4dCWad+6F77bv1umYqHIocZCxYsUKTJs2DU2aNIGpqSl69+6NkJAQYf/Dhw9x5coVnQyyonv33XfwxeJP8OmCL9HepSdOJ57Fr/u2oGZNG30PjfSE34lXw/37D9C40WuYPeWjUh0X/sN6HNq7Rdisa1iVeQzHTibCfdDIYvefSjqL6fMWYkBfD+zcsApvd3XFxFmf4sLFy8/0/f3wX0hMToGdLb9nGmhl2yqzEgcZX331FdavX49Vq1bhu+++w8GDB7F06VIEBQXpcnyVwuRJY/H1N1uxafMOnDt3AR/5zcS9e/cxepSXvodGesLvxKuhq2t7TPQdCbc3O5fqOOsaVrC1sRY2Q8Mn/xRrNBqs37wdHoNHoW33/hg48iMcOPhHmcf4/Y6f0dmlHcZ4D0bD+nUxwXcEnN5oiK0//iLql3o7HSFL12LRJ9NhZFSlzJ9XUWi1Wtm2yqzEczIuXbqETp06Ce87deqE6OhouLm5oaCgAAEBAboYX4VnbGyMNm1aYmHoKqFNq9UiKvpPdOzYVo8jI33hd6LiGzzKD/kFBWjUoD4+8vFGm5bNhH3rv9uO8P0HETRtAurWdkR8whnMnL8YNawUaN+6Zak/63TyOYx8f4CorZNLW0T/ESu812g0mDX/C4waOhiNXqtX9gsjkihxkGFra4tr166hfv36Qlvz5s0RHR2Nt99+Gzdv3tTF+Co8W1trGBkZIS01XdSelnYbTRo31NOoSJ/4nai4atpYI2jaBDRr8jryCwqw65cIjPGfga3rl8GpcSPk5+fj683bsX55CFo1bwoAqFPLAScTk7Hz59/KFGSk38mEjXUNUZutdQ2k38kU3n/z/U5UqWKIYe/2f7kLrEAqe5lDLiUOMrp06YKffvoJXbt2FbU7OTkhKioK3bt3L/GHqtVqqNVqUZtWq4WBgUGJz0FE9KppUK82GtSrLbxv3cIJ12/cwubtu7EwaBquXr+F+w/UGBvwsei4goJCNH3jSYDZ3u1JZkLzUIP8ggJRW1/3t/HJ9AklGlNyygV8v/Nn7Px2Jf8NfgrvLpFHiYOMmTNnIj4+vsh9zZo1Q3R0NHbt2lWic4WEhCA4OFjUZmBYDQZVLEs6nAojPT0DhYWFsLO3FbXb2dWEKvW2nkZF+sTvROXSvGljnEpMBgDcu38fALBmcTDsa4r/+xsbGwuvd21cLbxOTE7B0rXfYsOqUKHNwqKq8NrWpgbuZDzJWgBAekYmbG0eZTdOnj6DjMws/G/QCGH/w4caLF71Nb7bsQcHdm162UukSqzEQcaePXswderUYvc3b94czZs3L9G5Zs2ahcDAQFFbDZsmJR1KhVJQUICTJxPxdvcu2Lt3PwDAwMAAb3fvgjVrN+h5dKQP/E5ULikXLsLWxhoA0LB+XZiYGONW6u3nlkbq1nYUXqvS0lGlShVR29OcmzXF0fgEDH9qXkbs8VNwbvaoHNOvZw90bN9adMyHk+egX8+34dnbvczX9arTVPIJm3IpcZARHByMcePGoWrVqi/u/AKmpqYwNTUVtVXmNN3S5eux4ZuliD+ZiOPHT2HihLGwsDDHxk3b9T000hN+J14N9+7dx9XrT+aj3biZipS//4XCsjoclHZYunYD0tLvIGTuoz/Qvtu+G7UclWjUoB7U+fnYtTcCx06exrqlCwA8ykCMGjIIoSvWQavRoHXLZsjNu4dTicmoZlEV/Xv/r9RjHPZef4z2m46NP+xCt04d8Nvvh5GccgHzZkwEAFgpLGGlEGeRjYyqwNa6hqi0U9kwxJBHiYOMyn4bji7t3LkXNW2tMS9oKpTKmjh9Ohl9+g5DWlr6iw+mConfiVfDmZQLGDNhhvA+dOU6AED/Xm74bM4UpN/JwK3UNGF/QWEhFq9cj7Tbd2BmZoo3GjbA18s+R4e2zkKfCWNHoIaVAl9/twPXbqpgWc0CTRs3wtgR75dpjK1bOGHRvBlYuW4Tln+1EfVq18KKkLl4/bX6ZbtoolIw0JYwejA0NERqaipq1qypk4EYmdTSyXmJ6NV3/2bZ14mgisnYVrcrTHeu9bZs5/rrRrRs53rVlOrZJW+88cYLyxoZGbpdHpeIiEjXeAurPEoVZAQHB0OhUOhqLEREROUCpwjIo1RBhpeXF+zs7HQ1FiIiokotJCQEP/30E1JSUmBubo5OnTph0aJFaNy4sdDnwYMHmDJlCrZt2wa1Wg0PDw+sWbMG9vb2Qp+rV69i/PjxOHjwIKpVq4aRI0ciJCQERkZPfu0fOnQIgYGBSE5ORp06dTBnzhyMGjVK1usp8bNLKvPdH0REVLno6wFphw8fhp+fH44ePYrIyEgUFBTA3d0deXl5Qp/Jkyfjl19+wc6dO3H48GHcvHkTAwcOFPY/fPgQffr0QX5+Po4cOYJNmzZh48aNomeNXbp0CX369EH37t2RkJCAgIAAfPDBB9i/f//L//CeUqqJnyqVSmeZDE78JKLicOInSel64md7x26ynev4zZgyH3v79m3Y2dnh8OHD6NatG7Kzs1GzZk1s3boVgwcPBgCkpKSgadOmiI2NRceOHfHbb7+hb9++uHnzppDdCAsLw4wZM3D79m2YmJhgxowZ2LdvH86cOSN8lpeXF7KyshAREfFyF/yUEmcyNBoNSyVERET/oezsbACAtfWjBdvi4+NRUFAANzc3oU+TJk1Qt25dxMY+euhdbGwsWrRoISqfeHh4ICcnB8nJyUKfp8/xuM/jc8ilVHMyiIiIKgM5J34W9byuohallNJoNAgICEDnzp2FFbVVKhVMTExgZWUl6mtvbw+VSiX0eTrAeLz/8b7n9cnJycH9+/dhbm5euossRokzGURERJWFnHMyQkJCoFAoRFtISMgLx+Dn54czZ85g27Zt/8EV6wYzGURERDpU1PO6XpTF8Pf3R3h4OGJiYlC79pPl3ZVKJfLz85GVlSXKZqSmpkKpVAp9jh07JjpfamqqsO/x/z5ue7qPpaWlbFkMgJkMIiKiZ2i1Wtk2U1NTWFpairbiggytVgt/f3/s3r0b0dHRaNCggWh/27ZtYWxsjKioKKHt/PnzuHr1KlxdXQEArq6uSEpKQlrakyXtIyMjYWlpCScnJ6HP0+d43OfxOeTCTAYREZGEvlb89PPzw9atW/Hzzz+jevXqwhwKhUIBc3NzKBQK+Pj4IDAwENbW1rC0tMSECRPg6uqKjh07AgDc3d3h5OSE4cOHIzQ0FCqVCnPmzIGfn58Q3IwbNw6rVq3C9OnTMWbMGERHR2PHjh3Yt2+frNdT4ltYdY23sBJRcXgLK0np+hZWZ2Un2c51WnWkxH2LW5Nqw4YNwkJZjxfj+uGHH0SLcT0uhQDAlStXMH78eBw6dAgWFhYYOXIkFi5c+MxiXJMnT8bZs2dRu3ZtzJ07V/bFuBhkEFG5xyCDpHQdZLRUylc2SFTJe1voq4TlEiIiIglN+fj7+5XHIIOIiEhCy6ewyoJ3lxAREZFOMJNBREQkwXKJPBhkEBERSbBcIg+WS4iIiEgnmMkgIiKSYLlEHgwyiIiIJFgukQfLJURERKQTzGQQERFJsFwiDwYZREREEiyXyIPlEiIiItIJZjKIiIgktFqNvodQITDIICIiktCwXCILBhlEREQSWk78lAXnZBAREZFOMJNBREQkwXKJPBhkEBERSbBcIg+WS4iIiEgnmMkgIiKS4Iqf8mCQQUREJMEVP+XBcgkRERHpBDMZREREEpz4KQ8GGURERBK8hVUeLJcQERGRTjCTQUREJMFyiTwYZBAREUnwFlZ5MMggIiKSYCZDHpyTQURERDrBTAYREZEE7y6RB4MMIiIiCZZL5MFyCREREekEMxlEREQSvLtEHgwyiIiIJPiANHmwXEJEREQ6wUwGERGRBMsl8mCQQUREJMG7S+TBcgkRERHpBDMZREREEpz4KQ9mMoiIiCS0Wq1sW2mtXr0a9evXh5mZGVxcXHDs2DEdXOF/g0EGERGRhL6CjO3btyMwMBCffPIJTp48CWdnZ3h4eCAtLU1HV6pbDDKIiIjKiS+//BJjx47F6NGj4eTkhLCwMFStWhXffvutvodWJgwyiIiIJLQybmq1Gjk5OaJNrVY/85n5+fmIj4+Hm5ub0GZoaAg3NzfExsbq7Fp1qdxM/CzMv6HvIeidWq1GSEgIZs2aBVNTU30Ph/SM3wd6Gr8P/y05fyfNmzcPwcHBorZPPvkE8+bNE7Wlp6fj4cOHsLe3F7Xb29sjJSVFtvH8lwy0vBm43MjJyYFCoUB2djYsLS31PRzSM34f6Gn8Pry61Gr1M5kLU1PTZ4LFmzdvolatWjhy5AhcXV2F9unTp+Pw4cOIi4v7T8Yrp3KTySAiIqqIigooimJra4sqVaogNTVV1J6amgqlUqmr4ekU52QQERGVAyYmJmjbti2ioqKENo1Gg6ioKFFm41XCTAYREVE5ERgYiJEjR6Jdu3bo0KEDli1bhry8PIwePVrfQysTBhnliKmpKT755BNO6iIA/D6QGL8PlcP777+P27dvIygoCCqVCq1atUJERMQzk0FfFZz4SURERDrBORlERESkEwwyiIiISCcYZBAREZFOMMggIiIinWCQoQejRo2CgYEBDAwMYGJigkaNGmH+/PkoLCwEACQmJqJr164wMzNDnTp1EBoaqucRky497/vw4MEDjBo1Ci1atICRkRE8PT31PVzSsed9Hw4dOoT+/fvDwcEBFhYWaNWqFbZs2aLvIRMVi7ew6knPnj2xYcMGqNVq/Prrr/Dz84OxsTH8/Pzg7u4ONzc3hIWFISkpCWPGjIGVlRV8fX31PWzSkeK+DxMnToS5uTkmTpyIXbt26XuY9B8p7vug1WrRsmVLzJgxA/b29ggPD8eIESOgUCjQt29ffQ+b6Bm8hVUPRo0ahaysLOzZs0doc3d3x927dzFixAjMnj0bKpUKJiYmAICZM2diz549r+wDcuj5nvd9ePrJi0X1o4qnpN+Hx/r06QN7e/tX9lHgVLGxXFJOmJubIz8/H7GxsejWrZsQYACAh4cHzp8/j8zMTD2OkP5Lj78PRMDzvw/Z2dmwtrb+j0dEVDIMMvRMq9Xi999/x/79+/H2229DpVIV+ZhfAFCpVPoYIv2HpN8Hqtxe9H3YsWMHjh8//souOU0VH+dk6El4eDiqVauGgoICaDQaDB06FPPmzcOAAQP0PTTSg+K+D1Q5leT7cPDgQYwePRrr169Hs2bN9DNQohdgkKEn3bt3x9q1a2FiYgJHR0cYGT36T6FUKot8zO/jfVQxFfd9oMrpRd+Hw4cPo1+/fli6dClGjBihp1ESvRj/JdMTCwsLNGrU6Jl2V1dXzJ49GwUFBTA2NgYAREZGonHjxqhRo8Z/PUz6jxT3faDK6Xnfh0OHDqFv375YtGgR7zijco9zMsqZoUOHwsTEBD4+PkhOTsb27duxfPlyBAYG6ntopCdnz55FQkICMjIykJ2djYSEBCQkJOh7WKQHBw8eRJ8+fTBx4kQMGjQIKpUKKpUKGRkZ+h4aUZGYyShnFAoFDhw4AD8/P7Rt2xa2trYICgriXyyVWO/evXHlyhXhfevWrQE8mhRIlcumTZtw7949hISEICQkRGh/8803cejQIf0NjKgYXCeDiIiIdILlEiIiItIJBhlERESkEwwyiIiISCcYZBAREZFOMMggIiIinWCQQURERDrBIIOIiIh0gkEGERER6QSDDCIiItIJBhlERESkEwwyiIiISCcYZBAREZFO/B/7SFMF3iXvEQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cm = confusion_matrix(y_test, y_pred)\n",
    "# each row contains all predictions for true label\n",
    "df_cm = pd.DataFrame(cm, index=[f\"T{i}\" for i in range(3)],\n",
    "                     columns=[f\"P{j}\" for j in range(3)])\n",
    "sns.heatmap(df_cm, annot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 472
    },
    "id": "btQWkZwgnozk",
    "outputId": "6f000332-72f5-4efc-a4e2-b60f7f2093fe"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA24AAAHHCAYAAADH3QtoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA++klEQVR4nO3deVxUZf//8feAOogwoGiASuIurrhvIaiUe2q5ZJnLbWq3t6K3puaOmuXdamZlWq5fzaUUy6Vyg9QsV9QUlwyVu1xyYVESEub3hz/mdsIFFJijvJ6PxzxirnOd63zOYU769jrnjMlqtVoFAAAAADAsJ0cXAAAAAAC4O4IbAAAAABgcwQ0AAAAADI7gBgAAAAAGR3ADAAAAAIMjuAEAAACAwRHcAAAAAMDgCG4AAAAAYHAENwAAAAAwOIIbAOChcOPGDY0aNUp+fn5ycnJSp06dHF1SnuvTp4/8/f0dXQYAwAEIbgDwCFuwYIFMJpNMJpO2b9+eabnVapWfn59MJpPat2/vgAqzbt68eXrrrbfUpUsXLVy4UP/+979zZTtLly7VjBkzcmXs/O7IkSMKDw/XqVOnHF0KADx0Cji6AABA7nNxcdHSpUv1xBNP2LVHRUXpv//9r8xms4Mqy7otW7aoVKlSeu+993J1O0uXLtXPP/+sYcOG5ep27sfcuXOVnp7u6DLu25EjRzR58mSFhIQwcwgA2cSMGwDkA23bttXKlSt148YNu/alS5eqbt268vHxcVBlWXfhwgV5eno6uoz7lpyc/MBjFCxY8KEI2X93/fr1hzpwAoARENwAIB/o0aOHLl26pI0bN9raUlNT9cUXX+j555+/7Trp6emaMWOGqlWrJhcXF3l7e2vgwIG6cuWKXb81a9aoXbt2KlmypMxms8qXL6+pU6cqLS3Nrl9ISIiqV6+uI0eOqHnz5nJ1dVWpUqX05ptv3rX2U6dOyWQyaevWrTp8+LDt0s/IyMgcrzMkJETr1q3T6dOnbdvJmBnKuOz075f5RUZG2tVz677u3btXzZo1k6urq8aOHStJSklJ0aRJk1ShQgWZzWb5+flp1KhRSklJuetxkDLf45ZxbN5++219+OGHKleunFxdXfXUU08pLi5OVqtVU6dOVenSpVW4cGF17NhRly9fthvT399f7du313fffafAwEC5uLioatWqWrVqVabt//rrr+ratauKFSsmV1dXNWrUSOvWrbvt8Vi2bJnGjx+vUqVKydXVVTNnzlTXrl0lSc2bN8/0e8yNz9H169cVHh6uSpUqycXFRb6+vnrmmWd08uRJW5+sfn4AwNG4VBIA8gF/f381btxYn3/+udq0aSNJ2rBhgxISEvTcc89p5syZmdYZOHCgFixYoL59+yosLEyxsbGaNWuW9u/frx07dqhgwYKSbgYaNzc3DR8+XG5ubtqyZYsmTpyoxMREvfXWW3ZjXrlyRa1bt9Yzzzyjbt266YsvvtDo0aNVo0YNW11/V6JECS1evFjTpk3T1atX9cYbb0iSAgICcrzOcePGKSEhQf/9739tl2S6ubnd1zG/dOmS2rRpo+eee049e/aUt7e30tPT9fTTT2v79u0aMGCAAgICdOjQIb333ns6fvy4IiIi7mtbS5YsUWpqqoYMGaLLly/rzTffVLdu3dSiRQtFRkZq9OjR+uWXX/TBBx/olVde0bx58+zWP3HihLp3766XX35ZvXv31vz589W1a1d98803evLJJyVJ58+fV5MmTZScnKywsDB5eXlp4cKFevrpp/XFF1+oc+fOdmNOnTpVhQoV0iuvvKKUlBQ99dRTCgsL08yZMzV27Fjb7y/jvzn9OUpLS1P79u21efNmPffccxo6dKiSkpK0ceNG/fzzzypfvrykrH9+AMDhrACAR9b8+fOtkqy7d++2zpo1y+ru7m5NTk62Wq1Wa9euXa3Nmze3Wq1Wa5kyZazt2rWzrbdt2zarJOuSJUvsxvvmm28ytWeMd6uBAwdaXV1drdevX7e1BQcHWyVZFy1aZGtLSUmx+vj4WJ999tl77ktwcLC1WrVqdm25UWe7du2sZcqUydQ341jGxsbatW/dutUqybp169ZM+zp79my7vosXL7Y6OTlZt23bZtc+e/ZsqyTrjh07brvvGXr37m1XW2xsrFWStUSJEtb4+Hhb+5gxY6ySrLVq1bL+9ddftvYePXpYCxUqZLe/ZcqUsUqyfvnll7a2hIQEq6+vr7V27dq2tmHDhlkl2dWelJRkLVu2rNXf39+alpZmdzzKlSuX6ZivXLky07HKkNOfo3nz5lklWd99991M46anp1ut1ux9fgDA0bhUEgDyiW7duunPP//U2rVrlZSUpLVr197xMsmVK1fKw8NDTz75pC5evGh71a1bV25ubtq6dautb+HChW0/JyUl6eLFiwoKClJycrKOHj1qN66bm5t69uxpe1+oUCE1aNBAv/76633tU27VmRPMZrP69u2bqd6AgABVqVLFrt4WLVpIkl292dG1a1d5eHjY3jds2FCS1LNnTxUoUMCuPTU1Vb/99pvd+iVLlrSbMbNYLOrVq5f279+vc+fOSZLWr1+vBg0a2D3gxs3NTQMGDNCpU6d05MgRuzF79+5td8zvJac/R19++aWKFy+uIUOGZNqWyWSSlL3PDwA4GpdKAkA+UaJECYWGhmrp0qVKTk5WWlqaunTpctu+J06cUEJCgh577LHbLr9w4YLt58OHD2v8+PHasmWLEhMT7folJCTYvS9durTtL80ZihYtqoMHD97PLuVanTmhVKlSKlSoUKZ6Y2JiVKJEiXvWmx2PP/643fuMEOfn53fb9r/fv1WhQoVMv5dKlSpJunkfnY+Pj06fPm0LhLfKuNTx9OnTql69uq29bNmy2dqHnP4cnTx5UpUrV7YLrn+Xnc8PADgawQ0A8pHnn39e/fv317lz59SmTZs7PqUxPT1djz32mJYsWXLb5RnBIz4+XsHBwbJYLJoyZYrKly8vFxcX7du3T6NHj870JEFnZ+fbjme1Wu9rf3Krztv5e1DI8PeHZ2S43WxTenq6atSooXffffe26/w9aGXVnY5rTh/v7MjObJujPkdZ/fwAgBEQ3AAgH+ncubMGDhyoH3/8UcuXL79jv/Lly2vTpk1q2rTpXf8CHhkZqUuXLmnVqlVq1qyZrT02NjZH676T3KjzTgGtaNGikm6GjFudPn06W/UeOHBALVu2vON2HOGXX36R1Wq1q+n48eOSZHuKZZkyZXTs2LFM62ZcxlimTJl7budO+5wbn6Py5cvrp59+0l9//XXHB4xk9fMDAEbAPW4AkI+4ubnp448/Vnh4uDp06HDHft26dVNaWpqmTp2aadmNGzds4SVj5uPWmY7U1FR99NFHOVt4HtZZpEiR2146mfEUwu+//97WlpaWpjlz5mSr3t9++01z587NtOzPP//UtWvXsjxWTvr999+1evVq2/vExEQtWrRIgYGBtu/4a9u2rXbt2qWdO3fa+l27dk1z5syRv7+/qlates/tFClSRFLm8Jsbn6Nnn31WFy9e1KxZszIty9hOVj8/AGAEzLgBQD7Tu3fve/YJDg7WwIED9cYbbyg6OlpPPfWUChYsqBMnTmjlypV6//331aVLFzVp0kRFixZV7969FRYWJpPJpMWLF+fJpXi5VWfdunW1fPlyDR8+XPXr15ebm5s6dOigatWqqVGjRhozZowuX76sYsWKadmyZZm+1PxuXnzxRa1YsUIvv/yytm7dqqZNmyotLU1Hjx7VihUr9O2336pevXo5eYiypFKlSurXr592794tb29vzZs3T+fPn9f8+fNtfV599VXb10mEhYWpWLFiWrhwoWJjY/Xll1/Kyene/xYcGBgoZ2dn/ec//1FCQoLMZrNatGiRK5+jXr16adGiRRo+fLh27dqloKAgXbt2TZs2bdKgQYPUsWPHLH9+AMAICG4AgNuaPXu26tatq08++URjx45VgQIF5O/vr549e6pp06aSJC8vL61du1YjRozQ+PHjVbRoUfXs2VMtW7ZUq1atHso6Bw0apOjoaM2fP1/vvfeeypQpY5udXLJkiQYOHKjp06fL09NT/fr1U/PmzW3fdXYvTk5OioiI0HvvvadFixZp9erVcnV1Vbly5TR06FDbA0HyWsWKFfXBBx9o5MiROnbsmMqWLavly5fbHRtvb2/98MMPGj16tD744ANdv35dNWvW1Ndff6127dplaTs+Pj6aPXu23njjDfXr109paWnaunWrQkJCcvxz5OzsrPXr12vatGlaunSpvvzyS3l5eemJJ55QjRo1bP2y8vkBACMwWfPqn0UBAIDh+Pv7q3r16lq7dq2jSwEA3AX3uAEAAACAwRHcAAAAAMDgCG4AAAAAYHDc4wYAAAAABseMGwAAAAAYHMENAAAAAAyO73HLY+np6fr999/l7u4uk8nk6HIAAAAAOIjValVSUpJKliwpJ6e7z6kR3PLY77//Lj8/P0eXAQAAAMAg4uLiVLp06bv2IbjlMXd3d0k3fzkWi8XB1QAAAABwlMTERPn5+dkywt0Q3PJYxuWRFouF4AYAAAAgS7dQ8XASAAAAADA4ghsAAAAAGBzBDQAAAAAMjuAGAAAAAAZHcAMAAAAAgyO4AQAAAIDBEdwAAAAAwOAIbgAAAABgcAQ3AAAAADA4ghsAAAAAGBzBDQAAAAAMjuAGAAAAAAZHcAMAAAAAgyO4AQAAAIDBEdwAAAAAwOAIbgAAAABgcAUcXUB+VX3St3Iyuzq6DAAAACDfOTW9naNLyDZm3AAAAADA4AhuAAAAAGBwBDcAAAAAMDiCGwAAAAAYHMENAAAAAAyO4AYAAAAABkdwAwAAAACDI7gBAAAAgMER3AAAAADA4AhuAAAAAGBwBDcAAAAAMDiCGwAAAAAYHMENAAAAAAyO4AYAAAAABpet4BYSEqJhw4Zlal+wYIE8PT1zqCQAAAAAwK0emRm3tLQ0paenO7oMAAAAAMhxOR7c+vTpo06dOmny5MkqUaKELBaLXn75ZaWmptr6hISEaPDgwRo8eLA8PDxUvHhxTZgwQVar1dYnJSVFr7zyikqVKqUiRYqoYcOGioyMtC3PmOX76quvVLVqVZnNZp05c0YpKSkaPXq0/Pz8ZDabVaFCBX322WeSboa7fv36qWzZsipcuLAqV66s999/367+yMhINWjQQEWKFJGnp6eaNm2q06dP25avWbNGderUkYuLi8qVK6fJkyfrxo0bOX0YAQAAAMCmQG4MunnzZrm4uCgyMlKnTp1S37595eXlpWnTptn6LFy4UP369dOuXbu0Z88eDRgwQI8//rj69+8vSRo8eLCOHDmiZcuWqWTJklq9erVat26tQ4cOqWLFipKk5ORk/ec//9Gnn34qLy8vPfbYY+rVq5d27typmTNnqlatWoqNjdXFixclSenp6SpdurRWrlwpLy8v/fDDDxowYIB8fX3VrVs33bhxQ506dVL//v31+eefKzU1Vbt27ZLJZJIkbdu2Tb169dLMmTMVFBSkkydPasCAAZKkSZMm3fZYpKSkKCUlxfY+MTEx5w84AAAAgEeayXrrNNc9hISEKDAwUDNmzLBrX7BggYYNG6b4+Hj16dNHX3/9teLi4uTq6ipJmj17tkaOHKmEhAQ5OTkpJCREFy5c0OHDh22h6NVXX9VXX32lI0eO6MyZMypXrpzOnDmjkiVL2rYTGhqqBg0a6PXXX9eCBQvUt29fRUdHq1atWpKk48ePq3Llytq4caNCQ0OztE+DBw/WuXPn9MUXX+jy5cvy8vJSZGSkgoODM/UNDQ1Vy5YtNWbMGFvb//3f/2nUqFH6/fffbzt+eHi4Jk+enKndb9gKOZlds1QjAAAAgJxzano7R5cg6eakjoeHhxISEmSxWO7aN1fucatVq5YttElS48aNdfXqVcXFxdnaGjVqZAttGX1OnDihtLQ0HTp0SGlpaapUqZLc3Nxsr6ioKJ08edK2TqFChVSzZk3b++joaDk7O982dGX48MMPVbduXZUoUUJubm6aM2eOzpw5I0kqVqyY+vTpo1atWqlDhw56//33dfbsWdu6Bw4c0JQpU+xq6t+/v86ePavk5OTbbm/MmDFKSEiwvW49BgAAAACQFdm6VNJisSghISFTe3x8vDw8PHKsqKtXr8rZ2Vl79+6Vs7Oz3TI3Nzfbz4ULF7YLf4ULF77ruMuWLdMrr7yid955R40bN5a7u7veeust/fTTT7Y+8+fPV1hYmL755hstX75c48eP18aNG9WoUSNdvXpVkydP1jPPPJNpbBcXl9tu02w2y2w2Z2m/AQAAAOB2shXcKleurO+++y5T+759+1SpUiXb+wMHDujPP/+0Bakff/xRbm5u8vPzs/W5NSxl9KlYsaKcnZ1Vu3ZtpaWl6cKFCwoKCspyfTVq1FB6erqioqJue6nkjh071KRJEw0aNMjWdusMXobatWurdu3aGjNmjBo3bqylS5eqUaNGqlOnjo4dO6YKFSpkuSYAAAAAeFDZulTyn//8p44fP66wsDAdPHhQx44d07vvvqvPP/9cI0aMsPVLTU1Vv379dOTIEa1fv16TJk3S4MGD5eT0v82dOXNGw4cP17Fjx/T555/rgw8+0NChQyVJlSpV0gsvvKBevXpp1apVio2N1a5du/TGG29o3bp1d6zP399fvXv31j/+8Q9FREQoNjZWkZGRWrFihSSpYsWK2rNnj7799lsdP35cEyZM0O7du23rx8bGasyYMdq5c6dOnz6t7777TidOnFBAQIAkaeLEiVq0aJEmT56sw4cPKyYmRsuWLdP48eOzcxgBAAAAIFuyNeNWrlw5ff/99xo3bpxCQ0OVmpqqKlWqaOXKlWrdurWtX8uWLVWxYkU1a9ZMKSkp6tGjh8LDw+3G6tWrl/788081aNBAzs7OGjp0qO0JjdLNSxZfe+01jRgxQr/99puKFy+uRo0aqX379net8eOPP9bYsWM1aNAgXbp0SY8//rjGjh0rSRo4cKD279+v7t27y2QyqUePHho0aJA2bNggSXJ1ddXRo0e1cOFCXbp0Sb6+vvrXv/6lgQMHSpJatWqltWvXasqUKfrPf/6jggULqkqVKnrppZeycxgBAAAAIFuy9VTJrOjTp4/i4+MVERFxxz53ejplfpDx5BieKgkAAAA4Bk+VBAAAAADkOIIbAAAAABhctu5xy4oFCxbcs09kZGRObxYAAAAAHlnMuAEAAACAwRHcAAAAAMDgCG4AAAAAYHAENwAAAAAwOIIbAAAAABgcwQ0AAAAADI7gBgAAAAAGR3ADAAAAAIMjuAEAAACAwRVwdAH51c+TW8lisTi6DAAAAAAPAWbcAAAAAMDgCG4AAAAAYHAENwAAAAAwOIIbAAAAABgcwQ0AAAAADI7gBgAAAAAGR3ADAAAAAIMjuAEAAACAwRHcAAAAAMDgCG4AAAAAYHAENwAAAAAwOIIbAAAAABgcwQ0AAAAADI7gBgAAAAAGR3ADAAAAAIMjuAEAAACAwRHcAAAAAMDgCG4AAAAAYHAENwAAAAAwOIIbAAAAABgcwQ0AAAAADI7gBgAAAAAGR3ADAAAAAIMjuAEAAACAwRHcAAAAAMDgCG4AAAAAYHAENwAAAAAwOIIbAAAAABgcwQ0AAAAADI7gBgAAAAAGR3ADAAAAAIMjuAEAAACAwRHcAAAAAMDgCG4AAAAAYHAENwAAAAAwOIIbAAAAABgcwQ0AAAAADI7gBgAAAAAGV8DRBeRX1Sd9Kyezq6PLAAAAwF2cmt7O0SUAkphxAwAAAADDI7gBAAAAgMER3AAAAADA4AhuAAAAAGBwBDcAAAAAMDiCGwAAAAAYHMENAAAAAAyO4AYAAAAABkdwAwAAAACDI7gBAAAAgMER3AAAAADA4AhuAAAAAGBwBDcAAAAAMDiCGwAAAAAYXLaDW58+fdSpU6dcKAUAAAAAcDuP1Ixbamqqo0sAAAAAgByXo8EtKipKDRo0kNlslq+vr1599VXduHFDkrR27Vp5enoqLS1NkhQdHS2TyaRXX33Vtv5LL72knj172t5v375dQUFBKly4sPz8/BQWFqZr167Zlvv7+2vq1Knq1auXLBaLBgwYIEnasWOHQkJC5OrqqqJFi6pVq1a6cuWKJOmbb77RE088IU9PT3l5eal9+/Y6efKkbczU1FQNHjxYvr6+cnFxUZkyZfTGG2/YlsfHx+ull15SiRIlZLFY1KJFCx04cCAnDyMAAAAA2Mmx4Pbbb7+pbdu2ql+/vg4cOKCPP/5Yn332mV577TVJUlBQkJKSkrR//35JN0Ne8eLFFRkZaRsjKipKISEhkqSTJ0+qdevWevbZZ3Xw4EEtX75c27dv1+DBg+22+/bbb6tWrVrav3+/JkyYoOjoaLVs2VJVq1bVzp07tX37dnXo0MEWGK9du6bhw4drz5492rx5s5ycnNS5c2elp6dLkmbOnKmvvvpKK1as0LFjx7RkyRL5+/vbtte1a1dduHBBGzZs0N69e1WnTh21bNlSly9fvu1xSUlJUWJiot0LAAAAALLDZLVardlZoU+fPoqPj1dERIRd+7hx4/Tll18qJiZGJpNJkvTRRx9p9OjRSkhIkJOTk+rWrasePXrolVdeUefOnVW/fn1NnjxZly5dUkJCgkqXLq3jx4+rYsWKeumll+Ts7KxPPvnEto3t27crODhY165dk4uLi/z9/VW7dm2tXr3a1uf555/XmTNntH379iztz8WLF1WiRAkdOnRI1atXV1hYmA4fPqxNmzbZ9uPW7bdr104XLlyQ2Wy2tVeoUEGjRo2yzfjdKjw8XJMnT87U7jdshZzMrlmqEQAAAI5xano7R5eAR1hiYqI8PDyUkJAgi8Vy1745NuMWExOjxo0b24Wdpk2b6urVq/rvf/8rSQoODlZkZKSsVqu2bdumZ555RgEBAdq+fbuioqJUsmRJVaxYUZJ04MABLViwQG5ubrZXq1atlJ6ertjYWNs26tWrZ1dHxozbnZw4cUI9evRQuXLlZLFYbLNpZ86ckXQzmEZHR6ty5coKCwvTd999Z1v3wIEDunr1qry8vOzqio2Ntbvc8lZjxoxRQkKC7RUXF5eNowoAAAAAUoG83FhISIjmzZunAwcOqGDBgqpSpYpCQkIUGRmpK1euKDg42Nb36tWrGjhwoMLCwjKN8/jjj9t+LlKkiN2ywoUL37WGDh06qEyZMpo7d65Kliyp9PR0Va9e3fZgkzp16ig2NlYbNmzQpk2b1K1bN4WGhuqLL77Q1atX5evra3d5ZwZPT8/bbs9sNtvNzgEAAABAduVYcAsICNCXX34pq9Vqm3XbsWOH3N3dVbp0aUn/u8/tvffes4W0kJAQTZ8+XVeuXNGIESNs49WpU0dHjhxRhQoVslVHzZo1tXnz5ttennjp0iUdO3ZMc+fOVVBQkCTd9pJKi8Wi7t27q3v37urSpYtat26ty5cvq06dOjp37pwKFChgd98bAAAAAOSm+7pUMiEhQdHR0XavAQMGKC4uTkOGDNHRo0e1Zs0aTZo0ScOHD5eT083NFC1aVDVr1tSSJUtsDyFp1qyZ9u3bp+PHj9vNuI0ePVo//PCDBg8erOjoaJ04cUJr1qzJ9HCSvxszZox2796tQYMG6eDBgzp69Kg+/vhjXbx4UUWLFpWXl5fmzJmjX375RVu2bNHw4cPt1n/33Xf1+eef6+jRozp+/LhWrlwpHx8feXp6KjQ0VI0bN1anTp303Xff6dSpU/rhhx80btw47dmz534OJQAAAADc033NuEVGRqp27dp2bf369dP69es1cuRI1apVS8WKFVO/fv00fvx4u37BwcGKjo62BbdixYqpatWqOn/+vCpXrmzrV7NmTUVFRWncuHEKCgqS1WpV+fLl1b1797vWVqlSJX333XcaO3asGjRooMKFC6thw4bq0aOHnJyctGzZMoWFhal69eqqXLmyZs6caatFktzd3fXmm2/qxIkTcnZ2Vv369bV+/Xpb+Fy/fr3GjRunvn376o8//pCPj4+aNWsmb2/v+zmUAAAAAHBP2X6qJB5MxpNjeKokAACA8fFUSeQmhzxVEgAAAACQOwhuAAAAAGBwBDcAAAAAMDiCGwAAAAAYHMENAAAAAAyO4AYAAAAABkdwAwAAAACDI7gBAAAAgMER3AAAAADA4AhuAAAAAGBwBDcAAAAAMDiCGwAAAAAYXAFHF5Bf/Ty5lSwWi6PLAAAAAPAQYMYNAAAAAAyO4AYAAAAABkdwAwAAAACDI7gBAAAAgMER3AAAAADA4AhuAAAAAGBwBDcAAAAAMDiCGwAAAAAYHMENAAAAAAyO4AYAAAAABkdwAwAAAACDI7gBAAAAgMER3AAAAADA4AhuAAAAAGBwBDcAAAAAMDiCGwAAAAAYHMENAAAAAAyO4AYAAAAABkdwAwAAAACDI7gBAAAAgMER3AAAAADA4AhuAAAAAGBwBDcAAAAAMDiCGwAAAAAYHMENAAAAAAyO4AYAAAAABkdwAwAAAACDI7gBAAAAgMER3AAAAADA4AhuAAAAAGBwBDcAAAAAMDiCGwAAAAAYHMENAAAAAAyO4AYAAAAABkdwAwAAAACDI7gBAAAAgMER3AAAAADA4AhuAAAAAGBwBRxdQH5VfdK3cjK7OroMAACAbDs1vZ2jSwDyHWbcAAAAAMDgCG4AAAAAYHAENwAAAAAwOIIbAAAAABgcwQ0AAAAADI7gBgAAAAAGR3ADAAAAAIMjuAEAAACAwRHcAAAAAMDgCG4AAAAAYHAENwAAAAAwOIIbAAAAABgcwQ0AAAAADO6RDm4mk0kRERFZ7h8ZGSmTyaT4+PhcqwkAAAAAsuuhDG59+vSRyWSSyWRSwYIF5e3trSeffFLz5s1Tenq6rd/Zs2fVpk2bLI/bpEkTnT17Vh4eHpKkBQsWyNPTM6fLBwAAAIBseSiDmyS1bt1aZ8+e1alTp7RhwwY1b95cQ4cOVfv27XXjxg1Jko+Pj8xmc5bHLFSokHx8fGQymXKrbAAAAADItoc2uJnNZvn4+KhUqVKqU6eOxo4dqzVr1mjDhg1asGCBpMyXSv7www8KDAyUi4uL6tWrp4iICJlMJkVHR0uyv1QyMjJSffv2VUJCgm12Lzw8XJL00UcfqWLFinJxcZG3t7e6dOmStzsPAAAAIF8p4OgCclKLFi1Uq1YtrVq1Si+99JLdssTERHXo0EFt27bV0qVLdfr0aQ0bNuyOYzVp0kQzZszQxIkTdezYMUmSm5ub9uzZo7CwMC1evFhNmjTR5cuXtW3btjuOk5KSopSUFLs6AAAAACA7HqngJklVqlTRwYMHM7UvXbpUJpNJc+fOlYuLi6pWrarffvtN/fv3v+04hQoVkoeHh0wmk3x8fGztZ86cUZEiRdS+fXu5u7urTJkyql279h3reeONNzR58uQH3zEAAAAA+dZDe6nknVit1tveo3bs2DHVrFlTLi4utrYGDRpke/wnn3xSZcqUUbly5fTiiy9qyZIlSk5OvmP/MWPGKCEhwfaKi4vL9jYBAAAA5G+PXHCLiYlR2bJlc218d3d37du3T59//rl8fX01ceJE1apV645fIWA2m2WxWOxeAAAAAJAdj1Rw27Jliw4dOqRnn30207LKlSvr0KFDdveb7d69+67jFSpUSGlpaZnaCxQooNDQUL355ps6ePCgTp06pS1btjz4DgAAAADAbTy0wS0lJUXnzp3Tb7/9pn379un1119Xx44d1b59e/Xq1StT/+eff17p6ekaMGCAYmJi9O233+rtt9+WpDs+/t/f319Xr17V5s2bdfHiRSUnJ2vt2rWaOXOmoqOjdfr0aS1atEjp6emqXLlyru4vAAAAgPzroQ1u33zzjXx9feXv76/WrVtr69atmjlzptasWSNnZ+dM/S0Wi77++mtFR0crMDBQ48aN08SJEyXJ7r63WzVp0kQvv/yyunfvrhIlSujNN9+Up6enVq1apRYtWiggIECzZ8/W559/rmrVquXq/gIAAADIv0xWq9Xq6CIcZcmSJbbvaitcuHCebDMxMVEeHh7yG7ZCTmbXPNkmAABATjo1vZ2jSwAeCRnZICEh4Z7Pwnjkvg7gbhYtWqRy5cqpVKlSOnDggEaPHq1u3brlWWgDAAAAgPuRr4LbuXPnNHHiRJ07d06+vr7q2rWrpk2b5uiyAAAAAOCu8lVwGzVqlEaNGuXoMgAAAAAgWx7ah5MAAAAAQH5BcAMAAAAAgyO4AQAAAIDBEdwAAAAAwOAIbgAAAABgcAQ3AAAAADA4ghsAAAAAGBzBDQAAAAAMjuAGAAAAAAZXwNEF5Fc/T24li8Xi6DIAAAAAPASYcQMAAAAAgyO4AQAAAIDBEdwAAAAAwOAIbgAAAABgcAQ3AAAAADA4ghsAAAAAGBzBDQAAAAAMjuAGAAAAAAZHcAMAAAAAgyO4AQAAAIDBEdwAAAAAwOAIbgAAAABgcAQ3AAAAADA4ghsAAAAAGBzBDQAAAAAMjuAGAAAAAAZHcAMAAAAAgyO4AQAAAIDBEdwAAAAAwOAIbgAAAABgcAQ3AAAAADA4ghsAAAAAGBzBDQAAAAAMjuAGAAAAAAZHcAMAAAAAgyO4AQAAAIDBEdwAAAAAwOAIbgAAAABgcAQ3AAAAADA4ghsAAAAAGBzBDQAAAAAMjuAGAAAAAAZHcAMAAAAAgyO4AQAAAIDBEdwAAAAAwOAIbgAAAABgcAQ3AAAAADA4ghsAAAAAGFwBRxeQX1Wf9K2czK6OLgMAAIc4Nb2do0sAgIcKM24AAAAAYHAENwAAAAAwOIIbAAAAABgcwQ0AAAAADI7gBgAAAAAGR3ADAAAAAIMjuAEAAACAwRHcAAAAAMDgCG4AAAAAYHAENwAAAAAwOIIbAAAAABgcwQ0AAAAADI7gBgAAAAAGR3B7QOHh4QoMDHR0GQAAAAAeYQQ3AAAAADA4glsW/fXXX44uAQAAAEA+lePBLSQkRGFhYRo1apSKFSsmHx8fhYeH25afOXNGHTt2lJubmywWi7p166bz58/blmdcerh48WL5+/vLw8NDzz33nJKSku64zVmzZql69eq29xERETKZTJo9e7atLTQ0VOPHj7e9//jjj1W+fHkVKlRIlStX1uLFi+3GNJlM+vjjj/X000+rSJEimjZtmiRp+vTp8vb2lru7u/r166fr16/f97ECAAAAgKzIlRm3hQsXqkiRIvrpp5/05ptvasqUKdq4caPS09PVsWNHXb58WVFRUdq4caN+/fVXde/e3W79kydPKiIiQmvXrtXatWsVFRWl6dOn33F7wcHBOnLkiP744w9JUlRUlIoXL67IyEhJN2fLdu7cqZCQEEnS6tWrNXToUI0YMUI///yzBg4cqL59+2rr1q1244aHh6tz5846dOiQ/vGPf2jFihUKDw/X66+/rj179sjX11cfffTRXY9FSkqKEhMT7V4AAAAAkB0mq9VqzckBQ0JClJaWpm3bttnaGjRooBYtWqhly5Zq06aNYmNj5efnJ0k6cuSIqlWrpl27dql+/foKDw/XW2+9pXPnzsnd3V2SNGrUKH3//ff68ccfb7tNq9WqEiVKaPbs2erSpYtq166t7t276/3339fZs2e1Y8cONW/eXPHx8XJ1dVXTpk1VrVo1zZkzxzZGt27ddO3aNa1bt+7mgTGZNGzYML333nu2Pk2aNFHt2rX14Ycf2toaNWqk69evKzo6+ra1hYeHa/LkyZna/YatkJPZNYtHFQCAR8up6e0cXQIAOFxiYqI8PDyUkJAgi8Vy1765MuNWs2ZNu/e+vr66cOGCYmJi5OfnZwttklS1alV5enoqJibG1ubv728LbbeuL0lLliyRm5ub7bVt2zaZTCY1a9ZMkZGRio+P15EjRzRo0CClpKTo6NGjioqKUv369eXqejMoxcTEqGnTpnY1Nm3a1K4GSapXr57d+5iYGDVs2NCurXHjxnc9FmPGjFFCQoLtFRcXd9f+AAAAAPB3BXJj0IIFC9q9N5lMSk9Pz5H1n376abvwVKpUKUk3Z/rmzJmjbdu2qXbt2rJYLLYwFxUVpeDg4GzvR5EiRbK9zt+ZzWaZzeYHHgcAAABA/pWnT5UMCAhQXFyc3azTkSNHFB8fr6pVq2ZpDHd3d1WoUMH2Kly4sKT/3ee2cuVK271sISEh2rRpk3bs2GFry6hjx44dduPu2LHjnjUEBATop59+smu70+WbAAAAAJBTcmXG7U5CQ0NVo0YNvfDCC5oxY4Zu3LihQYMGKTg4ONNlidlVs2ZNFS1aVEuXLtXatWsl3Qxur7zyikwmk92lkSNHjlS3bt1Uu3ZthYaG6uuvv9aqVau0adOmu25j6NCh6tOnj+rVq6emTZtqyZIlOnz4sMqVK/dAtQMAAADA3eTpjJvJZNKaNWtUtGhRNWvWTKGhoSpXrpyWL1+eI2MHBQXJZDLpiSeekHQzzFksFtWrV8/ussdOnTrp/fff19tvv61q1arpk08+0fz58+1m5W6ne/fumjBhgkaNGqW6devq9OnT+uc///nAtQMAAADA3eT4UyVxdxlPjuGpkgCA/IynSgKAAZ4qCQAAAADIOQQ3AAAAADA4ghsAAAAAGBzBDQAAAAAMjuAGAAAAAAZHcAMAAAAAgyO4AQAAAIDBEdwAAAAAwOAIbgAAAABgcAQ3AAAAADA4ghsAAAAAGBzBDQAAAAAMroCjC8ivfp7cShaLxdFlAAAAAHgIMOMGAAAAAAZHcAMAAAAAgyO4AQAAAIDBEdwAAAAAwOAIbgAAAABgcAQ3AAAAADA4ghsAAAAAGBzBDQAAAAAMjuAGAAAAAAZHcAMAAAAAgyO4AQAAAIDBEdwAAAAAwOAIbgAAAABgcAQ3AAAAADA4ghsAAAAAGBzBDQAAAAAMjuAGAAAAAAZHcAMAAAAAgyO4AQAAAIDBEdwAAAAAwOAIbgAAAABgcAQ3AAAAADA4ghsAAAAAGBzBDQAAAAAMjuAGAAAAAAZHcAMAAAAAgyO4AQAAAIDBEdwAAAAAwOAIbgAAAABgcAQ3AAAAADA4ghsAAAAAGBzBDQAAAAAMjuAGAAAAAAZHcAMAAAAAgyO4AQAAAIDBEdwAAAAAwOAIbgAAAABgcAQ3AAAAADC4Ao4uIL+qPulbOZldHV2GJOnU9HaOLgEAAADAXTDjBgAAAAAGR3ADAAAAAIMjuAEAAACAwRHcAAAAAMDgCG4AAAAAYHAENwAAAAAwOIIbAAAAABgcwQ0AAAAADI7gBgAAAAAGR3ADAAAAAIMjuAEAAACAwRHcAAAAAMDgCG4AAAAAYHAENwAAAAAwuEcmuA0cOFDOzs5auXJlpmXJyckaM2aMypcvLxcXF5UoUULBwcFas2aNXb9ffvlFffv2VenSpWU2m1W2bFn16NFDe/bssfUxmUyKiIi443sAAAAAyGmPRHBLTk7WsmXLNGrUKM2bNy/T8pdfflmrVq3SBx98oKNHj+qbb75Rly5ddOnSJVufPXv2qG7dujp+/Lg++eQTHTlyRKtXr1aVKlU0YsSIvNwdAAAAALBTIKcGCgkJUc2aNeXi4qJPP/1UhQoV0ssvv6zw8HBJ0pkzZzRkyBBt3rxZTk5Oat26tT744AN5e3tLksLDwxUREaERI0ZowoQJunLlitq0aaO5c+fK3d39rtteuXKlqlatqldffVUlS5ZUXFyc/Pz8bMu/+uorvf/++2rbtq0kyd/fX3Xr1rUtt1qt6tOnjypWrKht27bJyel/eTYwMFBDhw7NqcMEAAAAANmWozNuCxcuVJEiRfTTTz/pzTff1JQpU7Rx40alp6erY8eOunz5sqKiorRx40b9+uuv6t69u936J0+eVEREhNauXau1a9cqKipK06dPv+d2P/vsM/Xs2VMeHh5q06aNFixYYLfcx8dH69evV1JS0m3Xj46O1uHDhzVixAi70JbB09Mzy8fg71JSUpSYmGj3AgAAAIDsyNHgVrNmTU2aNEkVK1ZUr169VK9ePW3evFmbN2/WoUOHtHTpUtWtW1cNGzbUokWLFBUVpd27d9vWT09P14IFC1S9enUFBQXpxRdf1ObNm++6zRMnTujHH3+0hcCePXtq/vz5slqttj5z5szRDz/8IC8vL9WvX1///ve/tWPHDrsxJKlKlSo5eTgkSW+88YY8PDxsr1tnAgEAAAAgK3I8uN3K19dXFy5cUExMjPz8/OxCS9WqVeXp6amYmBhbm7+/v91lkRnrS9KSJUvk5uZme23btk2SNG/ePLVq1UrFixeXJLVt21YJCQnasmWLbZxmzZrp119/1ebNm9WlSxcdPnxYQUFBmjp1qiTZhbycNmbMGCUkJNhecXFxubYtAAAAAI+mHLvHTZIKFixo995kMik9PT1H1n/66afVsGFD27JSpUopLS1NCxcu1Llz51SgwP92JS0tTfPmzVPLli3txg4KClJQUJBGjx6t1157TVOmTNHo0aNVqVIlSdLRo0dVu3btrO9wFpjNZpnN5hwdEwAAAED+kqPB7U4CAgIUFxdn99CQI0eOKD4+XlWrVs3SGO7u7pkeUvL1118rKSlJ+/fvl7Ozs639559/Vt++fRUfH3/H+9OqVq2qGzdu6Pr16woMDFTVqlX1zjvvqHv37pnuc7vbOAAAAACQ2/Lk6wBCQ0NVo0YNvfDCC9q3b5927dqlXr16KTg4WPXq1bvvcT/77DO1a9dOtWrVUvXq1W2vbt26ydPTU0uWLJF084mXn3zyifbu3atTp05p/fr1Gjt2rJo3by6LxSKTyaT58+fr+PHjCgoK0vr16/Xrr7/q4MGDmjZtmjp27JhThwIAAAAAsi1PgpvJZNKaNWtUtGhRNWvWTKGhoSpXrpyWL19+32OeP39e69at07PPPptpmZOTkzp37qzPPvtMktSqVSstXLhQTz31lAICAjRkyBC1atVKK1assK3ToEED7dmzRxUqVFD//v0VEBCgp59+WocPH9aMGTPuu04AAAAAeFAma24+mQOZJCYm3ny65LAVcjK7OrocSdKp6e0cXQIAAMAjzWq16saNG0pLS3N0Kchjzs7OKlCggEwmU6ZlGdkgISFBFovlruPkyT1uAAAAQH6Vmpqqs2fPKjk52dGlwEFcXV3l6+urQoUK3fcYBDcAAAAgl6Snpys2NlbOzs4qWbKkChUqdNuZFzyarFarUlNT9ccffyg2NlYVK1bM9CDErCK4AQAAALkkNTVV6enp8vPzk6urMW6TQd4qXLiwChYsqNOnTys1NVUuLi73NU6ePJwEAAAAyM/ud5YFj4ac+P3zCQIAAAAAgyO4AQAAAIDBcY8bAAAAkMf8X12Xp9vj658efsy4AQAAAMikT58+6tSpk6PLuK1Tp07JZDIpOjra0aXkGYIbAAAAgIdGamqqo0twCIIbAAAAgLsKCQnRkCFDNGzYMBUtWlTe3t6aO3eurl27pr59+8rd3V0VKlTQhg0bbOtERkbKZDJp3bp1qlmzplxcXNSoUSP9/PPPdmN/+eWXqlatmsxms/z9/fXOO+/YLff399fUqVPVq1cvWSwWDRgwQGXLlpUk1a5dWyaTSSEhIZKk3bt368knn1Tx4sXl4eGh4OBg7du3z248k8mkTz/9VJ07d5arq6sqVqyor776yq7P4cOH1b59e1ksFrm7uysoKEgnT560Lf/0008VEBAgFxcXValSRR999NEDH+N7IbgBAAAAuKeFCxeqePHi2rVrl4YMGaJ//vOf6tq1q5o0aaJ9+/bpqaee0osvvqjk5GS79UaOHKl33nlHu3fvVokSJdShQwf99ddfkqS9e/eqW7dueu6553To0CGFh4drwoQJWrBggd0Yb7/9tmrVqqX9+/drwoQJ2rVrlyRp06ZNOnv2rFatWiVJSkpKUu/evbV9+3b9+OOPqlixotq2baukpCS78SZPnqxu3brp4MGDatu2rV544QVdvnxZkvTbb7+pWbNmMpvN2rJli/bu3at//OMfunHjhiRpyZIlmjhxoqZNm6aYmBi9/vrrmjBhghYuXJjjx/xWJqvVas3VLcBOYmKiPDw85DdshZzMxvgSRm5WBQAAyB3Xr19XbGysypYta/fFyw/Dw0n69Omj+Ph4RUREKCQkRGlpadq2bZskKS0tTR4eHnrmmWe0aNEiSdK5c+fk6+urnTt3qlGjRoqMjFTz5s21bNkyde/eXZJ0+fJllS5dWgsWLFC3bt30wgsv6I8//tB3331n2+6oUaO0bt06HT58WNLNGbfatWtr9erV/9ufU6dUtmxZ7d+/X4GBgXfch/T0dHl6emrp0qVq3769pJszbuPHj9fUqVMlSdeuXZObm5s2bNig1q1ba+zYsVq2bJmOHTumggULZhqzQoUKmjp1qnr06GFre+2117R+/Xr98MMPt63jTp+DjGyQkJAgi8Vy51+GeKqkw/w8udU9fzkAAACAUdSsWdP2s7Ozs7y8vFSjRg1bm7e3tyTpwoULdus1btzY9nOxYsVUuXJlxcTESJJiYmLUsWNHu/5NmzbVjBkzlJaWJmdnZ0lSvXr1slTj+fPnNX78eEVGRurChQtKS0tTcnKyzpw5c8d9KVKkiCwWi63u6OhoBQUF3Ta0Xbt2TSdPnlS/fv3Uv39/W/uNGzfk4eGRpRrvF8ENAAAAwD39PciYTCa7NpPJJOnmLFdOK1KkSJb69e7dW5cuXdL777+vMmXKyGw2q3HjxpkeaHK7fcmou3Dhwncc/+rVq5KkuXPnqmHDhnbLMkJmbiG4AQAAAMg1P/74ox5//HFJ0pUrV3T8+HEFBARIkgICArRjxw67/jt27FClSpXuGoQKFSok6eYlm39f96OPPlLbtm0lSXFxcbp48WK26q1Zs6YWLlyov/76K1PA8/b2VsmSJfXrr7/qhRdeyNa4D4rgBgAAACDXTJkyRV5eXvL29ta4ceNUvHhx2/fDjRgxQvXr19fUqVPVvXt37dy5U7NmzbrnUxofe+wxFS5cWN98841Kly4tFxcXeXh4qGLFilq8eLHq1aunxMREjRw58q4zaLczePBgffDBB3ruuec0ZswYeXh46Mcff1SDBg1UuXJlTZ48WWFhYfLw8FDr1q2VkpKiPXv26MqVKxo+fPj9HqZ7IrgBAAAAeSw/PRxu+vTpGjp0qE6cOKHAwEB9/fXXthmzOnXqaMWKFZo4caKmTp0qX19fTZkyRX369LnrmAUKFNDMmTM1ZcoUTZw4UUFBQYqMjNRnn32mAQMGqE6dOvLz89Prr7+uV155JVv1enl5acuWLRo5cqSCg4Pl7OyswMBANW3aVJL00ksvydXVVW+99ZZGjhypIkWKqEaNGho2bNj9HJ4s46mSeSw7T44BAADAw+1OTxPMDzKeKnnlyhV5eno6uhyHyomnSvI9bgAAAABgcAQ3AAAAADA47nEDAAAAkONCQkLEXVk5hxk3AAAAADA4ghsAAACQy5h5yt9y4vdPcAMAAAByScYXOCcnJzu4EjhSxu//71/onR3c4wYAAADkEmdnZ3l6eurChQuSJFdXV5lMJgdXhbxitVqVnJysCxcuyNPTU87Ozvc9FsENAAAAyEU+Pj6SZAtvyH88PT1tn4P7RXADAAAAcpHJZJKvr68ee+wx/fXXX44uB3msYMGCDzTTloHgBgAAAOQBZ2fnHPkLPPInHk4CAAAAAAZHcAMAAAAAgyO4AQAAAIDBcY9bHsv48r3ExEQHVwIAAADAkTIyQVa+oJvglseSkpIkSX5+fg6uBAAAAIARJCUlycPD4659TNasxDvkmPT0dP3+++9yd3e/45cv1q9fX7t3777vbdzv+vezXlbXSUxMlJ+fn+Li4mSxWLJdW37xoL/7vOLIOnNz2zk5Nudx/sV57NhtG+k8fpAxsrse53HO4jx27Lbz03lstVqVlJSkkiVLysnp7nexMeOWx5ycnFS6dOm79nF2dn6g/5ne7/r3s15217FYLPxBcRcP+rvPK46sMze3nZNjcx7nX5zHjt22kc7jBxkju+txHucszmPHbju/ncf3mmnLwMNJDOhf//qXQ9a/n/UetFbYe1iOpyPrzM1t5+TYnMf518NyPDmP82asvDqXH5bP3cPiYTmenMd5M1Ze/pl8N1wqiTyRmJgoDw8PJSQkPBT/ggUgM85j4OHHeQw8vJhxQ54wm82aNGmSzGazo0sBcJ84j4GHH+cx8PBixg0AAAAADI4ZNwAAAAAwOIIbAAAAABgcwQ0AAAAADI7gBgAAAAAGR3ADAAAAAIMjuMFQ4uPjVa9ePQUGBqp69eqaO3euo0sCkE1xcXEKCQlR1apVVbNmTa1cudLRJQG4D507d1bRokXVpUsXR5cCQHwdAAwmLS1NKSkpcnV11bVr11S9enXt2bNHXl5eji4NQBadPXtW58+fV2BgoM6dO6e6devq+PHjKlKkiKNLA5ANkZGRSkpK0sKFC/XFF184uhwg32PGDYbi7OwsV1dXSVJKSoqsVqv4twXg4eLr66vAwEBJko+Pj4oXL67Lly87tigA2RYSEiJ3d3dHlwHg/yO4IVu+//57dejQQSVLlpTJZFJERESmPh9++KH8/f3l4uKihg0bateuXdnaRnx8vGrVqqXSpUtr5MiRKl68eA5VD0DKm/M4w969e5WWliY/P78HrBrArfLyPAZgDAQ3ZMu1a9dUq1Ytffjhh7ddvnz5cg0fPlyTJk3Svn37VKtWLbVq1UoXLlyw9cm4f+3vr99//12S5OnpqQMHDig2NlZLly7V+fPn82TfgPwiL85jSbp8+bJ69eqlOXPm5Po+AflNXp3HAIyDe9xw30wmk1avXq1OnTrZ2ho2bKj69etr1qxZkqT09HT5+flpyJAhevXVV7O9jUGDBqlFixbcGA3kktw6j1NSUvTkk0+qf//+evHFF3OjdAD/X27+eRwZGalZs2ZxjxtgAMy4IcekpqZq7969Cg0NtbU5OTkpNDRUO3fuzNIY58+fV1JSkiQpISFB33//vSpXrpwr9QLILCfOY6vVqj59+qhFixaENsABcuI8BmA8BDfkmIsXLyotLU3e3t527d7e3jp37lyWxjh9+rSCgoJUq1YtBQUFaciQIapRo0ZulAvgNnLiPN6xY4eWL1+uiIgIBQYGKjAwUIcOHcqNcgHcRk6cx5IUGhqqrl27av369SpdujShD3CwAo4uALhVgwYNFB0d7egyADyAJ554Qunp6Y4uA8AD2rRpk6NLAHALZtyQY4oXLy5nZ+dMDxM5f/68fHx8HFQVgOzgPAYefpzHwKOJ4IYcU6hQIdWtW1ebN2+2taWnp2vz5s1q3LixAysDkFWcx8DDj/MYeDRxqSSy5erVq/rll19s72NjYxUdHa1ixYrp8ccf1/Dhw9W7d2/Vq1dPDRo00IwZM3Tt2jX17dvXgVUDuBXnMfDw4zwG8h++DgDZEhkZqebNm2dq7927txYsWCBJmjVrlt566y2dO3dOgYGBmjlzpho2bJjHlQK4E85j4OHHeQzkPwQ3AAAAADA47nEDAAAAAIMjuAEAAACAwRHcAAAAAMDgCG4AAAAAYHAENwAAAAAwOIIbAAAAABgcwQ0AAAAADI7gBgAAAAAGR3ADAAAAAIMjuAEAAACAwRHcAAAAAMDgCG4AAAAAYHD/DzJUvjbyuqdUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Let's figure out distinctive features\n",
    "label_to_desc = {'fd':'Digits',\n",
    "                 'fu':'Uppercase',\n",
    "                 'fl':'Lowercase',\n",
    "                 'fx':'non-word',\n",
    "                 'fr': 'non-ASCII',\n",
    "                 }\n",
    "\n",
    "# create a dataframe with their (mean) importance\n",
    "f_importance = pd.DataFrame(data=cl.feature_importances_.flatten(),\n",
    "                            index=[label_to_desc[l] for l in features.columns],\n",
    "                            columns=[\"Importance\"],\n",
    "                            )\n",
    "# sort by importance and plot\n",
    "ax = f_importance.sort_values(\"Importance\").plot(kind=\"barh\",\n",
    "                                            title=\"Mean feature importance\",\n",
    "                                            figsize=(10, 5),\n",
    "                            )\n",
    "ax.set_xscale('log')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "id": "r1MwtF0-o3nI"
   },
   "outputs": [],
   "source": [
    "def check_ur_pwd(p_string, xgb_model):\n",
    "  \"\"\"predicts strength of a single password\"\"\"\n",
    "  assert isinstance(p_string, str), 'wrong input'\n",
    "  p_len = len(p_string)\n",
    "  # find all matches by patterns, then get frequencies and format for xgb input\n",
    "  patt = (\"\\d\",\"[A-Z]\", \"[a-z]\", \"[\\W_]\")\n",
    "  f = [len(tuple(re.finditer(pattern=p, string=p_string)))/p_len for p in patt]\n",
    "  f.append((p_len - len(f))/p_len)\n",
    "  df_single = pd.DataFrame([f], columns=features.columns)\n",
    "  return xgb_model.predict(df_single).item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "RmUVObmko32N",
    "outputId": "b80d2791-9899-46ca-8a05-36c4bcd0a568"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_ur_pwd(p_string='Arrt34t4eee_', xgb_model=cl)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CGMAkx_XD5MZ"
   },
   "source": [
    "I wish i had more time to implement PyTorch NN models i got used to and beat this baseline"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}