05_earthquake_undersampling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.preprocessing import (\n",
    "    LabelEncoder, MinMaxScaler\n",
    "    )\n",
    "from sklearn.metrics import (\n",
    "    precision_score, recall_score, roc_auc_score, f1_score, confusion_matrix, accuracy_score\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "quake_frame = pd.read_csv('data/consolidated_data.csv')\n",
    "\n",
    "quake_frame['simple_label'] = quake_frame['type'] != 'earthquake'\n",
    "\n",
    "quake_frame.drop(['id', 'Unnamed: 0', 'place', 'time', 'updated', 'type'], inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Undersampling, no imputation\n",
    "\n",
    "Keeping the Random Forest from before, we undersample by throwing away the NA values and undersample by various strategies to see the different results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "latitude           0\n",
       "longitude          0\n",
       "depth              0\n",
       "mag                0\n",
       "magType            0\n",
       "nst                0\n",
       "gap                0\n",
       "dmin               0\n",
       "rms                0\n",
       "net                0\n",
       "horizontalError    0\n",
       "depthError         0\n",
       "magError           0\n",
       "magNst             0\n",
       "status             0\n",
       "locationSource     0\n",
       "magSource          0\n",
       "simple_label       0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quake_frame.dropna(inplace=True)\n",
    "quake_frame.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1227408"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(quake_frame)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>latitude</th>\n",
       "      <td>1227408.0</td>\n",
       "      <td>37.361674</td>\n",
       "      <td>4.841731</td>\n",
       "      <td>0.000</td>\n",
       "      <td>35.964167</td>\n",
       "      <td>37.573000</td>\n",
       "      <td>38.817000</td>\n",
       "      <td>62.030667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>longitude</th>\n",
       "      <td>1227408.0</td>\n",
       "      <td>-119.557707</td>\n",
       "      <td>10.027502</td>\n",
       "      <td>-179.098</td>\n",
       "      <td>-122.701333</td>\n",
       "      <td>-120.558833</td>\n",
       "      <td>-118.150167</td>\n",
       "      <td>179.661500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>depth</th>\n",
       "      <td>1227408.0</td>\n",
       "      <td>6.016756</td>\n",
       "      <td>7.922880</td>\n",
       "      <td>-3.882</td>\n",
       "      <td>1.816000</td>\n",
       "      <td>4.413000</td>\n",
       "      <td>7.830000</td>\n",
       "      <td>211.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mag</th>\n",
       "      <td>1227408.0</td>\n",
       "      <td>1.258097</td>\n",
       "      <td>0.694405</td>\n",
       "      <td>-2.500</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>1.180000</td>\n",
       "      <td>1.670000</td>\n",
       "      <td>5.840000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>nst</th>\n",
       "      <td>1227408.0</td>\n",
       "      <td>17.010182</td>\n",
       "      <td>13.671235</td>\n",
       "      <td>0.000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>276.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gap</th>\n",
       "      <td>1227408.0</td>\n",
       "      <td>121.032150</td>\n",
       "      <td>65.767724</td>\n",
       "      <td>0.000</td>\n",
       "      <td>72.000000</td>\n",
       "      <td>105.000000</td>\n",
       "      <td>153.000000</td>\n",
       "      <td>360.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dmin</th>\n",
       "      <td>1227408.0</td>\n",
       "      <td>0.078264</td>\n",
       "      <td>0.342578</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.017120</td>\n",
       "      <td>0.037840</td>\n",
       "      <td>0.079990</td>\n",
       "      <td>141.160000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rms</th>\n",
       "      <td>1227408.0</td>\n",
       "      <td>0.097118</td>\n",
       "      <td>0.195847</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.030000</td>\n",
       "      <td>0.060000</td>\n",
       "      <td>0.130000</td>\n",
       "      <td>64.290000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>horizontalError</th>\n",
       "      <td>1227408.0</td>\n",
       "      <td>0.801039</td>\n",
       "      <td>2.296862</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.270000</td>\n",
       "      <td>0.410000</td>\n",
       "      <td>0.720000</td>\n",
       "      <td>194.584100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>depthError</th>\n",
       "      <td>1227408.0</td>\n",
       "      <td>2.773763</td>\n",
       "      <td>6.903563</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.490000</td>\n",
       "      <td>0.770000</td>\n",
       "      <td>1.460000</td>\n",
       "      <td>725.300000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>magError</th>\n",
       "      <td>1227408.0</td>\n",
       "      <td>0.158215</td>\n",
       "      <td>0.132854</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.090000</td>\n",
       "      <td>0.140000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>6.110000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>magNst</th>\n",
       "      <td>1227408.0</td>\n",
       "      <td>12.685281</td>\n",
       "      <td>17.846407</td>\n",
       "      <td>0.000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>430.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     count        mean        std      min         25%  \\\n",
       "latitude         1227408.0   37.361674   4.841731    0.000   35.964167   \n",
       "longitude        1227408.0 -119.557707  10.027502 -179.098 -122.701333   \n",
       "depth            1227408.0    6.016756   7.922880   -3.882    1.816000   \n",
       "mag              1227408.0    1.258097   0.694405   -2.500    0.800000   \n",
       "nst              1227408.0   17.010182  13.671235    0.000    8.000000   \n",
       "gap              1227408.0  121.032150  65.767724    0.000   72.000000   \n",
       "dmin             1227408.0    0.078264   0.342578    0.000    0.017120   \n",
       "rms              1227408.0    0.097118   0.195847    0.000    0.030000   \n",
       "horizontalError  1227408.0    0.801039   2.296862    0.000    0.270000   \n",
       "depthError       1227408.0    2.773763   6.903563    0.000    0.490000   \n",
       "magError         1227408.0    0.158215   0.132854    0.000    0.090000   \n",
       "magNst           1227408.0   12.685281  17.846407    0.000    4.000000   \n",
       "\n",
       "                        50%         75%         max  \n",
       "latitude          37.573000   38.817000   62.030667  \n",
       "longitude       -120.558833 -118.150167  179.661500  \n",
       "depth              4.413000    7.830000  211.000000  \n",
       "mag                1.180000    1.670000    5.840000  \n",
       "nst               13.000000   22.000000  276.000000  \n",
       "gap              105.000000  153.000000  360.000000  \n",
       "dmin               0.037840    0.079990  141.160000  \n",
       "rms                0.060000    0.130000   64.290000  \n",
       "horizontalError    0.410000    0.720000  194.584100  \n",
       "depthError         0.770000    1.460000  725.300000  \n",
       "magError           0.140000    0.200000    6.110000  \n",
       "magNst             8.000000   14.000000  430.000000  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "quake_frame.describe().transpose()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Alright, this changes the proportions slightly, but not too bad. If anything, one might suggest that at least the mild increase in proportion of non-earthquakes offsets the reduced dataset a little.  \n",
    "Okay, so the problematic values are no longer there, that's something.  \n",
    "Let's try this.  \n",
    "We'll start by mixing up the data frame, then encoding all the categories numerically and splitting it sklearn style."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "quake_frame = quake_frame.sample(frac=1, random_state=42).reset_index(drop=True)\n",
    "\n",
    "cat_columns = ['magType', 'net', 'status', 'locationSource', 'magSource']\n",
    "\n",
    "for cat in cat_columns:\n",
    "    quake_frame = pd.concat([quake_frame,\n",
    "                             pd.get_dummies(quake_frame[cat], prefix=cat)],\n",
    "                            axis=1)\n",
    "\n",
    "scale_cols = ['latitude', 'longitude', 'depth', 'mag', 'nst', 'gap', 'dmin', 'rms', 'horizontalError',\n",
    " 'depthError', 'magError', 'magNst']\n",
    "\n",
    "scaler = MinMaxScaler()\n",
    "\n",
    "quake_frame[scale_cols] = scaler.fit_transform(quake_frame[scale_cols])\n",
    "\n",
    "x_cols = ['latitude', 'longitude', 'depth', 'mag', 'nst', 'gap', 'dmin', 'rms', 'horizontalError', 'depthError',\n",
    " 'magError', 'magNst', 'magType_Mb', 'magType_Md', 'magType_Ml', 'magType_Unknown', 'magType_ma', 'magType_mb',\n",
    " 'magType_mc', 'magType_md', 'magType_me', 'magType_mh', 'magType_ml', 'magType_mlg', 'magType_mlr', 'magType_mw',\n",
    " 'net_av', 'net_ci', 'net_hv', 'net_ismpkansas', 'net_ld', 'net_mb', 'net_nc', 'net_nm', 'net_nn', 'net_pr',\n",
    " 'net_se', 'net_uu', 'net_uw', 'status_automatic', 'status_manual', 'status_reviewed', 'locationSource_av',\n",
    " 'locationSource_ci', 'locationSource_hv', 'locationSource_ismp', 'locationSource_ld', 'locationSource_mb',\n",
    " 'locationSource_nc', 'locationSource_nm', 'locationSource_nn', 'locationSource_pr', 'locationSource_se',\n",
    " 'locationSource_uu', 'locationSource_uw', 'magSource_av', 'magSource_ci', 'magSource_hv', 'magSource_ismp',\n",
    " 'magSource_ld', 'magSource_mb', 'magSource_nc', 'magSource_nm', 'magSource_nn', 'magSource_pr', 'magSource_se',\n",
    " 'magSource_uu', 'magSource_uw']\n",
    "\n",
    "y_col = ['simple_label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_length = int(np.round(len(quake_frame.index) * 0.8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = quake_frame.loc[:train_length, x_cols]\n",
    "train_y = quake_frame.loc[:train_length, y_col]\n",
    "\n",
    "valid_X = quake_frame.loc[train_length:, x_cols]\n",
    "valid_y = quake_frame.loc[train_length:, y_col]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try RandomUnderSampler (Controlled Undersampling)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import RandomUnderSampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "rus = RandomUnderSampler(random_state=42)\n",
    "\n",
    "train_X_resampled, train_y_resampled = rus.fit_resample(train_X, train_y)\n",
    "\n",
    "n_estim = 100\n",
    "\n",
    "rfc = RandomForestClassifier(n_estimators=n_estim,\n",
    "                             random_state=42)\n",
    "\n",
    "rfc.fit(train_X_resampled, np.ravel(train_y_resampled))\n",
    "\n",
    "preds = pd.DataFrame(rfc.predict(valid_X), columns=['predictions'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision:  0.6320670722659149\n",
      "Recall:  0.9780711825487944\n",
      "ROC score:  0.9785634915328738\n",
      "F1 score:  0.7678925545339823\n",
      "Accuracy score:  0.9790208650736103\n"
     ]
    }
   ],
   "source": [
    "prec = precision_score(valid_y, preds)\n",
    "reca = recall_score(valid_y, preds)\n",
    "roc = roc_auc_score(valid_y, preds)\n",
    "f1 = f1_score(valid_y, preds)\n",
    "acc = accuracy_score(valid_y, preds)\n",
    "conf_mat = confusion_matrix(valid_y, preds)\n",
    "\n",
    "print(\"Precision: \", prec)\n",
    "print(\"Recall: \", reca)\n",
    "print(\"ROC score: \", roc)\n",
    "print(\"F1 score: \", f1)\n",
    "print(\"Accuracy score: \", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try NearMiss Version 1, 2 and 3 (Controlled Undersampling)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import NearMiss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Version 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision:  0.08719435367390213\n",
      "Recall:  0.9772675086107921\n",
      "ROC score:  0.8004590545942816\n",
      "F1 score:  0.16010382672974016\n",
      "Accuracy score:  0.6361973586658085\n"
     ]
    }
   ],
   "source": [
    "nemi1 = NearMiss(version=1)\n",
    "\n",
    "train_X_resampled, train_y_resampled = nemi1.fit_resample(train_X, train_y)\n",
    "\n",
    "n_estim = 100\n",
    "\n",
    "rfc = RandomForestClassifier(n_estimators=n_estim,\n",
    "                             random_state=42)\n",
    "\n",
    "rfc.fit(train_X_resampled, np.ravel(train_y_resampled))\n",
    "\n",
    "preds = pd.DataFrame(rfc.predict(valid_X), columns=['predictions'])\n",
    "\n",
    "prec = precision_score(valid_y, preds)\n",
    "reca = recall_score(valid_y, preds)\n",
    "roc = roc_auc_score(valid_y, preds)\n",
    "f1 = f1_score(valid_y, preds)\n",
    "acc = accuracy_score(valid_y, preds)\n",
    "conf_mat = confusion_matrix(valid_y, preds)\n",
    "\n",
    "print(\"Precision: \", prec)\n",
    "print(\"Recall: \", reca)\n",
    "print(\"ROC score: \", roc)\n",
    "print(\"F1 score: \", f1)\n",
    "print(\"Accuracy score: \", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Version 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nemi2 = NearMiss(version=2)\n",
    "\n",
    "# train_X_resampled, train_y_resampled = nemi2.fit_resample(train_X, train_y)\n",
    "\n",
    "# n_estim = 100\n",
    "\n",
    "# rfc = RandomForestClassifier(n_estimators=n_estim,\n",
    "#                              random_state=42)\n",
    "\n",
    "# rfc.fit(train_X_resampled, np.ravel(train_y_resampled))\n",
    "\n",
    "# preds = pd.DataFrame(rfc.predict(valid_X), columns=['predictions'])\n",
    "\n",
    "# prec = precision_score(valid_y, preds)\n",
    "# reca = recall_score(valid_y, preds)\n",
    "# roc = roc_auc_score(valid_y, preds)\n",
    "# f1 = f1_score(valid_y, preds)\n",
    "# acc = accuracy_score(valid_y, preds)\n",
    "# conf_mat = confusion_matrix(valid_y, preds)\n",
    "\n",
    "# print(\"Precision: \", prec)\n",
    "# print(\"Recall: \", reca)\n",
    "# print(\"ROC score: \", roc)\n",
    "# print(\"F1 score: \", f1)\n",
    "# print(\"Accuracy score: \", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Very interesting. The kernel keeps dying for no apparent reason when running this."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Version 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision:  0.8737427776588915\n",
      "Recall:  0.9375430539609644\n",
      "ROC score:  0.9662796782821564\n",
      "F1 score:  0.9045192733717324\n",
      "Accuracy score:  0.9929770818226998\n"
     ]
    }
   ],
   "source": [
    "nemi3 = NearMiss(version=3)\n",
    "\n",
    "train_X_resampled, train_y_resampled = nemi3.fit_resample(train_X, train_y)\n",
    "\n",
    "n_estim = 100\n",
    "\n",
    "rfc = RandomForestClassifier(n_estimators=n_estim,\n",
    "                             random_state=42)\n",
    "\n",
    "rfc.fit(train_X_resampled, np.ravel(train_y_resampled))\n",
    "\n",
    "preds = pd.DataFrame(rfc.predict(valid_X), columns=['predictions'])\n",
    "\n",
    "prec = precision_score(valid_y, preds)\n",
    "reca = recall_score(valid_y, preds)\n",
    "roc = roc_auc_score(valid_y, preds)\n",
    "f1 = f1_score(valid_y, preds)\n",
    "acc = accuracy_score(valid_y, preds)\n",
    "conf_mat = confusion_matrix(valid_y, preds)\n",
    "\n",
    "print(\"Precision: \", prec)\n",
    "print(\"Recall: \", reca)\n",
    "print(\"ROC score: \", roc)\n",
    "print(\"F1 score: \", f1)\n",
    "print(\"Accuracy score: \", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try Tomek's Links (Cleaning Undersampling)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import TomekLinks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision:  0.962578222778473\n",
      "Recall:  0.88300803673938\n",
      "ROC score:  0.9408726092503685\n",
      "F1 score:  0.9210778443113773\n",
      "Accuracy score:  0.9946309709062171\n"
     ]
    }
   ],
   "source": [
    "toli = TomekLinks()\n",
    "\n",
    "train_X_resampled, train_y_resampled = toli.fit_resample(train_X, train_y)\n",
    "\n",
    "n_estim = 100\n",
    "\n",
    "rfc = RandomForestClassifier(n_estimators=n_estim,\n",
    "                             random_state=42)\n",
    "\n",
    "rfc.fit(train_X_resampled, np.ravel(train_y_resampled))\n",
    "\n",
    "preds = pd.DataFrame(rfc.predict(valid_X), columns=['predictions'])\n",
    "\n",
    "prec = precision_score(valid_y, preds)\n",
    "reca = recall_score(valid_y, preds)\n",
    "roc = roc_auc_score(valid_y, preds)\n",
    "f1 = f1_score(valid_y, preds)\n",
    "acc = accuracy_score(valid_y, preds)\n",
    "conf_mat = confusion_matrix(valid_y, preds)\n",
    "\n",
    "print(\"Precision: \", prec)\n",
    "print(\"Recall: \", reca)\n",
    "print(\"ROC score: \", roc)\n",
    "print(\"F1 score: \", f1)\n",
    "print(\"Accuracy score: \", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try EditedNearestNeighbours (Cleaning Undersampling)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import EditedNearestNeighbours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision:  0.9167342211928199\n",
      "Recall:  0.9088404133180252\n",
      "ROC score:  0.9529018683419819\n",
      "F1 score:  0.9127702507927355\n",
      "Accuracy score:  0.9938366153119169\n"
     ]
    }
   ],
   "source": [
    "enn = EditedNearestNeighbours()\n",
    "\n",
    "train_X_resampled, train_y_resampled = enn.fit_resample(train_X, train_y)\n",
    "\n",
    "n_estim = 100\n",
    "\n",
    "rfc = RandomForestClassifier(n_estimators=n_estim,\n",
    "                             random_state=42)\n",
    "\n",
    "rfc.fit(train_X_resampled, np.ravel(train_y_resampled))\n",
    "\n",
    "preds = pd.DataFrame(rfc.predict(valid_X), columns=['predictions'])\n",
    "\n",
    "prec = precision_score(valid_y, preds)\n",
    "reca = recall_score(valid_y, preds)\n",
    "roc = roc_auc_score(valid_y, preds)\n",
    "f1 = f1_score(valid_y, preds)\n",
    "acc = accuracy_score(valid_y, preds)\n",
    "conf_mat = confusion_matrix(valid_y, preds)\n",
    "\n",
    "print(\"Precision: \", prec)\n",
    "print(\"Recall: \", reca)\n",
    "print(\"ROC score: \", roc)\n",
    "print(\"F1 score: \", f1)\n",
    "print(\"Accuracy score: \", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try RepeatedEditedNearestNeighbours (Cleaning Undersampling)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import RepeatedEditedNearestNeighbours"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# renn = RepeatedEditedNearestNeighbours()\n",
    "\n",
    "# train_X_resampled, train_y_resampled = renn.fit_resample(train_X, train_y)\n",
    "\n",
    "# n_estim = 100\n",
    "\n",
    "# rfc = RandomForestClassifier(n_estimators=n_estim,\n",
    "#                              random_state=42)\n",
    "\n",
    "# rfc.fit(train_X_resampled, np.ravel(train_y_resampled))\n",
    "\n",
    "# preds = pd.DataFrame(rfc.predict(valid_X), columns=['predictions'])\n",
    "\n",
    "# prec = precision_score(valid_y, preds)\n",
    "# reca = recall_score(valid_y, preds)\n",
    "# roc = roc_auc_score(valid_y, preds)\n",
    "# f1 = f1_score(valid_y, preds)\n",
    "# acc = accuracy_score(valid_y, preds)\n",
    "# conf_mat = confusion_matrix(valid_y, preds)\n",
    "\n",
    "# print(\"Precision: \", prec)\n",
    "# print(\"Recall: \", reca)\n",
    "# print(\"ROC score: \", roc)\n",
    "# print(\"F1 score: \", f1)\n",
    "# print(\"Accuracy score: \", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This takes very, very long to run. I gave up eventually, I've got things to do and people to see."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try AllKNN (Cleaning Undersampling)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from imblearn.under_sampling import AllKNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "allknn = AllKNN()\n",
    "\n",
    "train_X_resampled, train_y_resampled = allknn.fit_resample(train_X, train_y)\n",
    "\n",
    "n_estim = 100\n",
    "\n",
    "rfc = RandomForestClassifier(n_estimators=n_estim,\n",
    "                             random_state=42)\n",
    "\n",
    "rfc.fit(train_X_resampled, np.ravel(train_y_resampled))\n",
    "\n",
    "preds = pd.DataFrame(rfc.predict(valid_X), columns=['predictions'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision:  0.9004637484447461\n",
      "Recall:  0.9140068886337543\n",
      "ROC score:  0.9551451164740579\n",
      "F1 score:  0.9071847757962509\n",
      "Accuracy score:  0.9933640755737692\n"
     ]
    }
   ],
   "source": [
    "prec = precision_score(valid_y, preds)\n",
    "reca = recall_score(valid_y, preds)\n",
    "roc = roc_auc_score(valid_y, preds)\n",
    "f1 = f1_score(valid_y, preds)\n",
    "acc = accuracy_score(valid_y, preds)\n",
    "conf_mat = confusion_matrix(valid_y, preds)\n",
    "\n",
    "print(\"Precision: \", prec)\n",
    "print(\"Recall: \", reca)\n",
    "print(\"ROC score: \", roc)\n",
    "print(\"F1 score: \", f1)\n",
    "print(\"Accuracy score: \", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try CondensedNearestNeighbour (Cleaning Undersampling)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import CondensedNearestNeighbour"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# connn = CondensedNearestNeighbour(random_state=42)\n",
    "\n",
    "# train_X_resampled, train_y_resampled = connn.fit_resample(train_X, train_y)\n",
    "\n",
    "# n_estim = 100\n",
    "\n",
    "# rfc = RandomForestClassifier(n_estimators=n_estim,\n",
    "#                              random_state=42)\n",
    "\n",
    "# rfc.fit(train_X_resampled, np.ravel(train_y_resampled))\n",
    "\n",
    "# preds = pd.DataFrame(rfc.predict(valid_X), columns=['predictions'])\n",
    "\n",
    "# prec = precision_score(valid_y, preds)\n",
    "# reca = recall_score(valid_y, preds)\n",
    "# roc = roc_auc_score(valid_y, preds)\n",
    "# f1 = f1_score(valid_y, preds)\n",
    "# conf_mat = confusion_matrix(valid_y, preds)\n",
    "\n",
    "# print(\"Precision: \", prec)\n",
    "# print(\"Recall: \", reca)\n",
    "# print(\"ROC score: \", roc)\n",
    "# print(\"F1 score: \", f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This takes very, very long to run. Yes, I ran out of patience eventually."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try OneSidedSelection (Cleaning Undersampling)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import OneSidedSelection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision:  0.9626343414146463\n",
      "Recall:  0.8843857634902411\n",
      "ROC score:  0.9415614726257991\n",
      "F1 score:  0.9218525610339875\n",
      "Accuracy score:  0.9946798543274049\n"
     ]
    }
   ],
   "source": [
    "oness = OneSidedSelection(random_state=42)\n",
    "\n",
    "train_X_resampled, train_y_resampled = oness.fit_resample(train_X, train_y)\n",
    "\n",
    "n_estim = 100\n",
    "\n",
    "rfc = RandomForestClassifier(n_estimators=n_estim,\n",
    "                             random_state=42)\n",
    "\n",
    "rfc.fit(train_X_resampled, np.ravel(train_y_resampled))\n",
    "\n",
    "preds = pd.DataFrame(rfc.predict(valid_X), columns=['predictions'])\n",
    "\n",
    "prec = precision_score(valid_y, preds)\n",
    "reca = recall_score(valid_y, preds)\n",
    "roc = roc_auc_score(valid_y, preds)\n",
    "f1 = f1_score(valid_y, preds)\n",
    "acc = accuracy_score(valid_y, preds)\n",
    "conf_mat = confusion_matrix(valid_y, preds)\n",
    "\n",
    "print(\"Precision: \", prec)\n",
    "print(\"Recall: \", reca)\n",
    "print(\"ROC score: \", roc)\n",
    "print(\"F1 score: \", f1)\n",
    "print(\"Accuracy score: \", acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try NeighbourhoodCleaningRule (Cleaning Undersampling)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from imblearn.under_sampling import NeighbourhoodCleaningRule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision:  0.9239588207767899\n",
      "Recall:  0.9067738231917336\n",
      "ROC score:  0.9520142830756027\n",
      "F1 score:  0.9152856646193069\n",
      "Accuracy score:  0.9940443698519648\n"
     ]
    }
   ],
   "source": [
    "ncr = NeighbourhoodCleaningRule()\n",
    "\n",
    "train_X_resampled, train_y_resampled = ncr.fit_resample(train_X, train_y)\n",
    "\n",
    "n_estim = 100\n",
    "\n",
    "rfc = RandomForestClassifier(n_estimators=n_estim,\n",
    "                             random_state=42)\n",
    "\n",
    "rfc.fit(train_X_resampled, np.ravel(train_y_resampled))\n",
    "\n",
    "preds = pd.DataFrame(rfc.predict(valid_X), columns=['predictions'])\n",
    "\n",
    "prec = precision_score(valid_y, preds)\n",
    "reca = recall_score(valid_y, preds)\n",
    "roc = roc_auc_score(valid_y, preds)\n",
    "f1 = f1_score(valid_y, preds)\n",
    "acc = accuracy_score(valid_y, preds)\n",
    "conf_mat = confusion_matrix(valid_y, preds)\n",
    "\n",
    "print(\"Precision: \", prec)\n",
    "print(\"Recall: \", reca)\n",
    "print(\"ROC score: \", roc)\n",
    "print(\"F1 score: \", f1)\n",
    "print(\"Accuracy score: \", acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}