binary_classifier-no-batchnorm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Binary Classifier implementation (model architecture, training, testing, etc.) derived from\n",
    "#     https://towardsdatascience.com/pytorch-tabular-binary-classification-a0368da5bb89"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import Dataset, DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loading subset of training data as per https://www.kaggle.com/sohier/competition-api-detailed-introduction/notebook\n",
    "train_df = pd.read_csv('data/train.csv', low_memory=False, nrows=10**5, \n",
    "                       dtype={'row_id': 'int64', 'timestamp': 'int64', 'user_id': 'int32', 'content_id': 'int16', 'content_type_id': 'int8',\n",
    "                              'task_container_id': 'int16', 'user_answer': 'int8', 'answered_correctly': 'int8', 'prior_question_elapsed_time': 'float32', \n",
    "                             'prior_question_had_explanation': 'boolean',\n",
    "                             }\n",
    "                      )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>user_id</th>\n",
       "      <th>content_id</th>\n",
       "      <th>content_type_id</th>\n",
       "      <th>task_container_id</th>\n",
       "      <th>user_answer</th>\n",
       "      <th>answered_correctly</th>\n",
       "      <th>prior_question_elapsed_time</th>\n",
       "      <th>prior_question_had_explanation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>115</td>\n",
       "      <td>5692</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>56943</td>\n",
       "      <td>115</td>\n",
       "      <td>5716</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>37000.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>118363</td>\n",
       "      <td>115</td>\n",
       "      <td>128</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>55000.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>131167</td>\n",
       "      <td>115</td>\n",
       "      <td>7860</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>19000.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>137965</td>\n",
       "      <td>115</td>\n",
       "      <td>7922</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>11000.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>99995</td>\n",
       "      <td>153647401</td>\n",
       "      <td>2078569</td>\n",
       "      <td>4334</td>\n",
       "      <td>0</td>\n",
       "      <td>275</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>6000.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>99996</td>\n",
       "      <td>153692472</td>\n",
       "      <td>2078569</td>\n",
       "      <td>6436</td>\n",
       "      <td>0</td>\n",
       "      <td>276</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>9000.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>99997</td>\n",
       "      <td>153722998</td>\n",
       "      <td>2078569</td>\n",
       "      <td>6446</td>\n",
       "      <td>0</td>\n",
       "      <td>277</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>21000.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>99998</td>\n",
       "      <td>153759775</td>\n",
       "      <td>2078569</td>\n",
       "      <td>3715</td>\n",
       "      <td>0</td>\n",
       "      <td>278</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>12000.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>99999</td>\n",
       "      <td>153842445</td>\n",
       "      <td>2078569</td>\n",
       "      <td>5624</td>\n",
       "      <td>0</td>\n",
       "      <td>279</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>16000.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100000 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       row_id  timestamp  user_id  content_id  content_type_id  \\\n",
       "0           0          0      115        5692                0   \n",
       "1           1      56943      115        5716                0   \n",
       "2           2     118363      115         128                0   \n",
       "3           3     131167      115        7860                0   \n",
       "4           4     137965      115        7922                0   \n",
       "...       ...        ...      ...         ...              ...   \n",
       "99995   99995  153647401  2078569        4334                0   \n",
       "99996   99996  153692472  2078569        6436                0   \n",
       "99997   99997  153722998  2078569        6446                0   \n",
       "99998   99998  153759775  2078569        3715                0   \n",
       "99999   99999  153842445  2078569        5624                0   \n",
       "\n",
       "       task_container_id  user_answer  answered_correctly  \\\n",
       "0                      1            3                   1   \n",
       "1                      2            2                   1   \n",
       "2                      0            0                   1   \n",
       "3                      3            0                   1   \n",
       "4                      4            1                   1   \n",
       "...                  ...          ...                 ...   \n",
       "99995                275            3                   0   \n",
       "99996                276            3                   0   \n",
       "99997                277            2                   1   \n",
       "99998                278            3                   0   \n",
       "99999                279            3                   1   \n",
       "\n",
       "       prior_question_elapsed_time  prior_question_had_explanation  \n",
       "0                              NaN                            <NA>  \n",
       "1                          37000.0                           False  \n",
       "2                          55000.0                           False  \n",
       "3                          19000.0                           False  \n",
       "4                          11000.0                           False  \n",
       "...                            ...                             ...  \n",
       "99995                       6000.0                            True  \n",
       "99996                       9000.0                            True  \n",
       "99997                      21000.0                            True  \n",
       "99998                      12000.0                            True  \n",
       "99999                      16000.0                            True  \n",
       "\n",
       "[100000 rows x 10 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get equal amount of correct and incorrect rows and shuffle data\n",
    "num_correct_rows = train_df[train_df['answered_correctly'] == 1].shape[0]\n",
    "num_incorrect_rows = train_df[train_df['answered_correctly'] == 0].shape[0]\n",
    "num_train_rows = min(num_correct_rows, num_incorrect_rows) * 2\n",
    "\n",
    "correct_df = train_df[train_df['answered_correctly'] == 1][:num_train_rows // 2]\n",
    "incorrect_df = train_df[train_df['answered_correctly'] == 0][:num_train_rows // 2]\n",
    "train_df = pd.concat([correct_df, incorrect_df]).dropna().sample(frac=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>row_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>user_id</th>\n",
       "      <th>content_id</th>\n",
       "      <th>content_type_id</th>\n",
       "      <th>task_container_id</th>\n",
       "      <th>user_answer</th>\n",
       "      <th>answered_correctly</th>\n",
       "      <th>prior_question_elapsed_time</th>\n",
       "      <th>prior_question_had_explanation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6840</th>\n",
       "      <td>6840</td>\n",
       "      <td>9936890006</td>\n",
       "      <td>24418</td>\n",
       "      <td>239</td>\n",
       "      <td>0</td>\n",
       "      <td>3792</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>17000.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20353</th>\n",
       "      <td>20353</td>\n",
       "      <td>82262699</td>\n",
       "      <td>260489</td>\n",
       "      <td>5250</td>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>24000.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13410</th>\n",
       "      <td>13410</td>\n",
       "      <td>2819981361</td>\n",
       "      <td>138650</td>\n",
       "      <td>8065</td>\n",
       "      <td>0</td>\n",
       "      <td>234</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>50800.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67423</th>\n",
       "      <td>67423</td>\n",
       "      <td>20875545441</td>\n",
       "      <td>1440282</td>\n",
       "      <td>9170</td>\n",
       "      <td>0</td>\n",
       "      <td>3751</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>24000.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28562</th>\n",
       "      <td>28562</td>\n",
       "      <td>610657</td>\n",
       "      <td>583550</td>\n",
       "      <td>8772</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38706</th>\n",
       "      <td>38706</td>\n",
       "      <td>8259930021</td>\n",
       "      <td>1084314</td>\n",
       "      <td>837</td>\n",
       "      <td>0</td>\n",
       "      <td>487</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>18000.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34544</th>\n",
       "      <td>34544</td>\n",
       "      <td>60769083</td>\n",
       "      <td>934487</td>\n",
       "      <td>6463</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>12000.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7219</th>\n",
       "      <td>7219</td>\n",
       "      <td>10878150882</td>\n",
       "      <td>24418</td>\n",
       "      <td>1111</td>\n",
       "      <td>0</td>\n",
       "      <td>4080</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>18000.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21340</th>\n",
       "      <td>21340</td>\n",
       "      <td>63446</td>\n",
       "      <td>327181</td>\n",
       "      <td>1278</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>21000.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37243</th>\n",
       "      <td>37243</td>\n",
       "      <td>5262307246</td>\n",
       "      <td>1024273</td>\n",
       "      <td>10104</td>\n",
       "      <td>0</td>\n",
       "      <td>287</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>73500.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>61458 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       row_id    timestamp  user_id  content_id  content_type_id  \\\n",
       "6840     6840   9936890006    24418         239                0   \n",
       "20353   20353     82262699   260489        5250                0   \n",
       "13410   13410   2819981361   138650        8065                0   \n",
       "67423   67423  20875545441  1440282        9170                0   \n",
       "28562   28562       610657   583550        8772                0   \n",
       "...       ...          ...      ...         ...              ...   \n",
       "38706   38706   8259930021  1084314         837                0   \n",
       "34544   34544     60769083   934487        6463                0   \n",
       "7219     7219  10878150882    24418        1111                0   \n",
       "21340   21340        63446   327181        1278                0   \n",
       "37243   37243   5262307246  1024273       10104                0   \n",
       "\n",
       "       task_container_id  user_answer  answered_correctly  \\\n",
       "6840                3792            1                   0   \n",
       "20353                 31            1                   0   \n",
       "13410                234            2                   1   \n",
       "67423               3751            0                   0   \n",
       "28562                 22            0                   0   \n",
       "...                  ...          ...                 ...   \n",
       "38706                487            3                   1   \n",
       "34544                 17            0                   1   \n",
       "7219                4080            3                   1   \n",
       "21340                  3            0                   0   \n",
       "37243                287            3                   1   \n",
       "\n",
       "       prior_question_elapsed_time  prior_question_had_explanation  \n",
       "6840                       17000.0                            True  \n",
       "20353                      24000.0                            True  \n",
       "13410                      50800.0                            True  \n",
       "67423                      24000.0                            True  \n",
       "28562                      10000.0                            True  \n",
       "...                            ...                             ...  \n",
       "38706                      18000.0                            True  \n",
       "34544                      12000.0                            True  \n",
       "7219                       18000.0                            True  \n",
       "21340                      21000.0                           False  \n",
       "37243                      73500.0                            True  \n",
       "\n",
       "[61458 rows x 10 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train_df.iloc[:, np.r_[2:7, 8:10]]\n",
    "y = train_df.iloc[:, 7]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPOCHS = 50\n",
    "BATCH_SIZE = 64\n",
    "LEARNING_RATE = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BinaryClassifier(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(BinaryClassifier, self).__init__()\n",
    "\n",
    "        self.layer_1 = nn.Linear(7, 64) \n",
    "        self.layer_2 = nn.Linear(64, 64)\n",
    "        self.layer_out = nn.Linear(64, 1) \n",
    "        \n",
    "        self.relu = nn.ReLU()\n",
    "        self.dropout = nn.Dropout(p=0.1)\n",
    "        \n",
    "    def forward(self, inputs):\n",
    "        x = self.relu(self.layer_1(inputs))\n",
    "        x = self.relu(self.layer_2(x))\n",
    "        x = self.dropout(x)\n",
    "        x = self.layer_out(x)\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BinaryClassifier(\n",
       "  (layer_1): Linear(in_features=7, out_features=64, bias=True)\n",
       "  (layer_2): Linear(in_features=64, out_features=64, bias=True)\n",
       "  (layer_out): Linear(in_features=64, out_features=1, bias=True)\n",
       "  (relu): ReLU()\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       ")"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "model = BinaryClassifier()\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.BCEWithLogitsLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TrainDataset(Dataset):\n",
    "    \n",
    "    def __init__(self, X_data, y_data):\n",
    "        self.X_data = X_data\n",
    "        self.y_data = y_data\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        return self.X_data[index], self.y_data[index]\n",
    "        \n",
    "    def __len__ (self):\n",
    "        return len(self.X_data)\n",
    "\n",
    "train_data = TrainDataset(torch.FloatTensor(np.array(X_train.values, dtype=np.float64)), \n",
    "                          torch.FloatTensor(y_train.values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TestDataset(Dataset):\n",
    "    \n",
    "    def __init__(self, X_data):\n",
    "        self.X_data = X_data\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        return self.X_data[index]\n",
    "        \n",
    "    def __len__ (self):\n",
    "        return len(self.X_data)\n",
    "    \n",
    "\n",
    "test_data = TestDataset(torch.FloatTensor(np.array(X_test.values, dtype=np.float64)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = DataLoader(dataset=train_data, batch_size=BATCH_SIZE, shuffle=True)\n",
    "test_loader = DataLoader(dataset=test_data, batch_size=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# calculates accuracy given predictions and labels\n",
    "def binary_acc(y_pred, y_test):\n",
    "    y_pred_tag = torch.round(torch.sigmoid(y_pred))\n",
    "\n",
    "    correct_results_sum = (y_pred_tag == y_test).sum().float()\n",
    "    acc = correct_results_sum/y_test.shape[0]\n",
    "    acc = torch.round(acc * 100)\n",
    "    \n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 001: | Loss: 318.71336 | Acc: 56.130\n",
      "Epoch 002: | Loss: 1.29016 | Acc: 61.101\n",
      "Epoch 003: | Loss: 0.62299 | Acc: 64.986\n",
      "Epoch 004: | Loss: 0.60057 | Acc: 66.312\n",
      "Epoch 005: | Loss: 0.59979 | Acc: 66.675\n",
      "Epoch 006: | Loss: 0.59623 | Acc: 67.379\n",
      "Epoch 007: | Loss: 0.59247 | Acc: 67.502\n",
      "Epoch 008: | Loss: 0.60418 | Acc: 66.829\n",
      "Epoch 009: | Loss: 0.61191 | Acc: 65.744\n",
      "Epoch 010: | Loss: 0.70269 | Acc: 50.863\n",
      "Epoch 011: | Loss: 0.68519 | Acc: 53.792\n",
      "Epoch 012: | Loss: 0.68092 | Acc: 57.918\n",
      "Epoch 013: | Loss: 0.68815 | Acc: 54.014\n",
      "Epoch 014: | Loss: 0.69056 | Acc: 53.480\n",
      "Epoch 015: | Loss: 0.69320 | Acc: 49.871\n",
      "Epoch 016: | Loss: 0.69314 | Acc: 50.202\n",
      "Epoch 017: | Loss: 0.69315 | Acc: 49.840\n",
      "Epoch 018: | Loss: 0.69318 | Acc: 50.016\n",
      "Epoch 019: | Loss: 0.69313 | Acc: 49.887\n",
      "Epoch 020: | Loss: 0.69313 | Acc: 49.651\n",
      "Epoch 021: | Loss: 0.69314 | Acc: 49.755\n",
      "Epoch 022: | Loss: 0.69313 | Acc: 50.045\n",
      "Epoch 023: | Loss: 0.69314 | Acc: 49.870\n",
      "Epoch 024: | Loss: 0.70306 | Acc: 49.991\n",
      "Epoch 025: | Loss: 0.69440 | Acc: 49.714\n",
      "Epoch 026: | Loss: 0.69315 | Acc: 49.828\n",
      "Epoch 027: | Loss: 0.69317 | Acc: 50.102\n",
      "Epoch 028: | Loss: 0.69317 | Acc: 50.202\n",
      "Epoch 029: | Loss: 0.69318 | Acc: 49.530\n",
      "Epoch 030: | Loss: 0.69315 | Acc: 50.113\n",
      "Epoch 031: | Loss: 0.69317 | Acc: 49.766\n",
      "Epoch 032: | Loss: 0.69317 | Acc: 50.112\n",
      "Epoch 033: | Loss: 0.69317 | Acc: 49.674\n",
      "Epoch 034: | Loss: 0.69317 | Acc: 50.006\n",
      "Epoch 035: | Loss: 0.69317 | Acc: 49.823\n",
      "Epoch 036: | Loss: 0.69316 | Acc: 50.048\n",
      "Epoch 037: | Loss: 0.69317 | Acc: 49.924\n",
      "Epoch 038: | Loss: 0.69317 | Acc: 49.758\n",
      "Epoch 039: | Loss: 0.69317 | Acc: 49.734\n",
      "Epoch 040: | Loss: 0.69317 | Acc: 49.865\n",
      "Epoch 041: | Loss: 0.69317 | Acc: 49.697\n",
      "Epoch 042: | Loss: 0.69318 | Acc: 49.797\n",
      "Epoch 043: | Loss: 0.69316 | Acc: 50.230\n",
      "Epoch 044: | Loss: 0.69317 | Acc: 50.073\n",
      "Epoch 045: | Loss: 0.69315 | Acc: 50.126\n",
      "Epoch 046: | Loss: 0.69317 | Acc: 49.924\n",
      "Epoch 047: | Loss: 0.69318 | Acc: 49.964\n",
      "Epoch 048: | Loss: 0.69316 | Acc: 49.807\n",
      "Epoch 049: | Loss: 0.69317 | Acc: 49.745\n",
      "Epoch 050: | Loss: 0.69318 | Acc: 49.803\n"
     ]
    }
   ],
   "source": [
    "model.train()\n",
    "epoch_losses = []\n",
    "for e in range(1, EPOCHS + 1):\n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    for X_batch, y_batch in train_loader:\n",
    "        X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        y_pred = model(X_batch)\n",
    "\n",
    "        loss = criterion(y_pred, y_batch.unsqueeze(1))\n",
    "        acc = binary_acc(y_pred, y_batch.unsqueeze(1))\n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_loss += loss.item()\n",
    "        epoch_acc += acc.item()\n",
    "    epoch_losses.append(epoch_loss / len(train_loader)) \n",
    "\n",
    "    print(f'Epoch {e+0:03}: | Loss: {epoch_loss/len(train_loader):.5f} | Acc: {epoch_acc/len(train_loader):.3f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f244a7e5610>]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAATy0lEQVR4nO3dbYid533n8e9vHjTKE61dj40qKStv0ULstlFg8Aa8LG4Sam+2rJwXLgps0YuA+sKBBLIsdt6kXRD0xTZdFjYFdWMiumm8giRrUUI3WjUlW9jGlVM3sayYiPhJlbAmSZM4TSzr4b8vzn1GZ0Yjz2hmjsbnOt8PDOc+17nvc65rxPx08b/vc1+pKiRJbZnY7A5Ikjae4S5JDTLcJalBhrskNchwl6QGTW12BwBuu+222rVr12Z3Q5JGylNPPfX9qppd7rU3Rbjv2rWLEydObHY3JGmkJHnxeq9ZlpGkBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUEjHe7nfvxzPv3V5/je/E83uyuS9KYy0uE+/+oF/utfnuZ78/+02V2RpDeVkQ73malJAC5curLJPZGkN5cRD/de9y9curzJPZGkN5eRDvet087cJWk5Ix3uCzP3i87cJWnQaIf7dL8s48xdkgaNdLhvmTTcJWk5Ix3uU5MTTE3EE6qStMRIhzv06u4XLjpzl6RBK4Z7kq1Jnkzy90lOJvn9rv3WJMeSfLd7vGXgmEeTnE7yXJL7hzmAmelJyzKStMRqZu4XgPdV1buBPcADSd4LPAIcr6rdwPHuOUnuAvYBdwMPAJ9JMjmEvgPdzN2yjCQtsmK4V0//5i3T3U8Be4HDXfth4MFuey/weFVdqKrngdPAPRvZ6UEzUxO8ZllGkhZZVc09yWSSp4HzwLGq+gZwR1WdA+geb+923w68PHD4ma5t6XseSHIiyYn5+fk1D2BmatKZuyQtsapwr6rLVbUH2AHck+RX32D3LPcWy7znoaqaq6q52dnZVXV2OTPTE9bcJWmJG7papqp+BPwVvVr6K0m2AXSP57vdzgA7Bw7bAZxdb0evx6tlJOlaq7laZjbJL3bbbwE+AHwHOArs73bbDzzRbR8F9iWZSXInsBt4coP7vcCyjCRda2oV+2wDDndXvEwAR6rqz5P8P+BIko8ALwEPAVTVySRHgGeBS8DDVTW09J2ZmuAff+bMXZIGrRjuVfUt4D3LtP8AeP91jjkIHFx371bBmrskXauBb6halpGkpUY+3LdOe0JVkpYa+XDvzdwNd0ka1EC4e/sBSVqqkXC/QtU135OSpLE1+uE+PUkVXLxsuEtS3+iHe38dVUszkrSgoXD3pKok9TUQ7r1bxRvuknTV6If7dG8Ir120LCNJfaMf7v2yjF9kkqQFDYR7vyzjzF2S+hoId0+oStJSox/u04a7JC01+uHeL8t4QlWSFjQQ7s7cJWmpBsLd69wlaamRD/et095+QJKWGvlwv1pzd+YuSX2jH+5eLSNJ1xj5cN8yaVlGkpYa+XCfmAhbJiecuUvSgBXDPcnOJF9LcirJySQf69p/L8k/JHm6+/ngwDGPJjmd5Lkk9w9zANCtxmTNXZIWTK1in0vAJ6rqm0neATyV5Fj32h9V1X8e3DnJXcA+4G7gl4H/k+RfVNXQ6iYz066jKkmDVpy5V9W5qvpmt/0qcArY/gaH7AUer6oLVfU8cBq4ZyM6ez0zU5O85sxdkhbcUM09yS7gPcA3uqaPJvlWkseS3NK1bQdeHjjsDMv8Z5DkQJITSU7Mz8/feM8H9BbJduYuSX2rDvckbwe+CHy8qn4C/DHwK8Ae4Bzwh/1dlzn8mtWrq+pQVc1V1dzs7OyN9nuRLVOeUJWkQasK9yTT9IL981X1JYCqeqWqLlfVFeBPuFp6OQPsHDh8B3B247p8rZnpScNdkgas5mqZAJ8FTlXVpwfatw3s9iHgmW77KLAvyUySO4HdwJMb1+Vr9a6WsSwjSX2ruVrmXuB3gG8nebpr+yTw4SR76JVcXgB+F6CqTiY5AjxL70qbh4d5pQz0wv3V1y4N8yMkaaSsGO5V9dcsX0f/yhsccxA4uI5+3ZCZqUm+f+n1m/VxkvSmN/LfUAWvc5ekpdoId7+hKkmLNBLuXi0jSYOaCPetlmUkaZEmwt2ZuyQt1ki4T/D6pStUXfNFWEkaS22Eu6sxSdIibYR7fx1Vw12SgGbC3aX2JGlQW+Hute6SBLQS7tP9sowzd0mCVsK9m7m7GpMk9TQV7p5QlaSeRsLdsowkDWoj3L3OXZIWaSPcvVpGkhZpJNwty0jSoEbC3bKMJA1qI9ytuUvSIk2E+9b+l5guWpaRJGgk3C3LSNJiTYT7lknDXZIGrRjuSXYm+VqSU0lOJvlY135rkmNJvts93jJwzKNJTid5Lsn9wxxA93m9RbK9WkaSgNXN3C8Bn6iqdwHvBR5OchfwCHC8qnYDx7vndK/tA+4GHgA+k2RyGJ0fNDM14XXuktRZMdyr6lxVfbPbfhU4BWwH9gKHu90OAw9223uBx6vqQlU9D5wG7tngfl9jZtp1VCWp74Zq7kl2Ae8BvgHcUVXnoPcfAHB7t9t24OWBw850bUvf60CSE0lOzM/Pr6Hri/Vm7pZlJAluINyTvB34IvDxqvrJG+26TNs1K1dX1aGqmququdnZ2dV247p6NXdn7pIEqwz3JNP0gv3zVfWlrvmVJNu617cB57v2M8DOgcN3AGc3prvXNzM16QlVSeqs5mqZAJ8FTlXVpwdeOgrs77b3A08MtO9LMpPkTmA38OTGdXl5M9PO3CWpb2oV+9wL/A7w7SRPd22fBP4AOJLkI8BLwEMAVXUyyRHgWXpX2jxcVUOfUnu1jCRdtWK4V9Vfs3wdHeD91znmIHBwHf26YTNTk/zoZ6/fzI+UpDetJr6hCp5QlaRB7YS717lL0oJ2wt3r3CVpQVvh7sxdkoCGwn2rZRlJWtBMuHtXSEm6qqFwn+Ti5eLylWvudCBJY6edcO/WUX3d0owkNRTuC0vtWZqRpIbCvVsk25m7JLUU7r2hvOa17pLUULhPu0i2JPW1E+79sox3hpSklsLdE6qS1NdguDtzl6R2wn26f7WMM3dJaifc+zN3a+6S1GC4W5aRpIbC3bKMJC1oJ9yduUvSgvbC3Zq7JLUT7lsty0jSghXDPcljSc4neWag7feS/EOSp7ufDw689miS00meS3L/sDq+1NREmIhlGUmC1c3cPwc8sEz7H1XVnu7nKwBJ7gL2AXd3x3wmyeRGdfaNJGFmyqX2JAlWEe5V9XXgh6t8v73A41V1oaqeB04D96yjfzdkZnqCC94VUpLWVXP/aJJvdWWbW7q27cDLA/uc6dqukeRAkhNJTszPz6+jG1f11lF15i5Jaw33PwZ+BdgDnAP+sGvPMvsuu6hpVR2qqrmqmpudnV1jNxabmZr0fu6SxBrDvapeqarLVXUF+BOull7OADsHdt0BnF1fF1fPmbsk9awp3JNsG3j6IaB/Jc1RYF+SmSR3AruBJ9fXxdWbmTbcJQlgaqUdknwBuA+4LckZ4FPAfUn20Cu5vAD8LkBVnUxyBHgWuAQ8XFU3rU7Su1rGsowkrRjuVfXhZZo/+wb7HwQOrqdTazUzNeE3VCWJhr6hCtbcJamvsXC3LCNJ0Fq4e0JVkoDWwt2auyQBzYW7ZRlJgubC3bKMJEFj4b512rtCShI0Fu4zUxNcvlJcumzASxpvbYX7tOuoShK0Fu5T/aX2DHdJ462xcO8Nx9v+Shp3bYW7ZRlJAloL94WyjDN3SeOtsXDvZu5+S1XSmGss3D2hKknQWrgv1Nwty0gab22Fu2UZSQKaC3fLMpIEzYW7ZRlJgtbC3evcJQloLdz7ZRm/oSppzDUW7s7cJQlWEe5JHktyPskzA223JjmW5Lvd4y0Drz2a5HSS55LcP6yOL8dwl6Se1czcPwc8sKTtEeB4Ve0GjnfPSXIXsA+4uzvmM0kmN6y3K5ianGBqIp5QlTT2Vgz3qvo68MMlzXuBw932YeDBgfbHq+pCVT0PnAbu2Ziuro6LZEvS2mvud1TVOYDu8faufTvw8sB+Z7q2ayQ5kOREkhPz8/Nr7Ma1ZlxqT5I2/IRqlmmr5XasqkNVNVdVc7OzsxvWgZmpCe/nLmnsrTXcX0myDaB7PN+1nwF2Duy3Azi79u7duJmpCWfuksbeWsP9KLC/294PPDHQvi/JTJI7gd3Ak+vr4o2ZmZr0hKqksTe10g5JvgDcB9yW5AzwKeAPgCNJPgK8BDwEUFUnkxwBngUuAQ9X1U1N2plpZ+6StGK4V9WHr/PS+6+z/0Hg4Ho6tR5eLSNJjX1DFSzLSBI0Ge6WZSSpvXC35i5JDYa7ZRlJajHcPaEqSW2Gu2UZSWOuvXCftiwjSe2Fezdzr1r2ljaSNBaaC/et05NUwcXLhruk8dVcuF9djcnSjKTx1XC4e1JV0vhqMNx7q/p5T3dJ46y9cJ925i5J7YV7vyzjF5kkjbEGw71XlvGEqqRx1mC4W5aRpPbC3Zq7JDUY7v2yjFfLSBpjDYa7M3dJajDc+ydUDXdJ46u9cJ/29gOS1F64e527JDG1noOTvAC8ClwGLlXVXJJbgf8J7AJeAH67qv5xfd1cPcsykrQxM/ffqKo9VTXXPX8EOF5Vu4Hj3fObxrtCStJwyjJ7gcPd9mHgwSF8xnVNTIQtky61J2m8rTfcC/hqkqeSHOja7qiqcwDd4+3LHZjkQJITSU7Mz8+vsxuLuUi2pHG3rpo7cG9VnU1yO3AsyXdWe2BVHQIOAczNzW3oskkz0xOWZSSNtXXN3KvqbPd4HvgycA/wSpJtAN3j+fV28kbNTE3ymjN3SWNszeGe5G1J3tHfBn4TeAY4CuzvdtsPPLHeTt6o3iLZztwlja/1lGXuAL6cpP8+f1ZVf5Hkb4EjST4CvAQ8tP5u3pgtU55QlTTe1hzuVfU94N3LtP8AeP96OrVeM9OThruksdbcN1Shf7WMZRlJ46vdcHfmLmmMNRrulmUkjbc2w93r3CWNuTbD3W+oShpzjYa7ZRlJ463RcLcsI2m8tRnu014tI2m8tRnuU5O8fukKVRt6PzJJGhmNhnt/wQ5n75LGU5PhvnXapfYkjbcmw/3qItmeVJU0ntoOd2fuksZUm+G+UJZx5i5pPLUZ7t3M3dWYJI2rpsPdsoykcdVouFuWkTTe2gz3aWfuksZbm+G+cCmk4S5pPDUa7pZlJI23RsPdsoyk8dZmuFtzlzTmhhbuSR5I8lyS00keGdbnLGehLOPtBySNqaGEe5JJ4L8B/wa4C/hwkruG8VnL6ZdlXvzBzzh9/qfMv3qB153FSxojU0N633uA01X1PYAkjwN7gWeH9HmLzExN8LYtk/zp37zIn/7Niwvtb90yyS+8ZXrhrpEAWbIRbkzyxkcsvad8XedJAVequHylqOpt934GPmvhM/vPV/hsvJ89QC35Pa/G9X6zK/xzSzfsA++6g4Mf+rUNf99hhft24OWB52eAfzm4Q5IDwAGAd77znRv64Un4i4//a07P/5Sf/PwiP/75RX78s+7x5xcXavH9P/R+AN9wFF7ngKIWB++SQBh8Ovifw2RgImFiIkx0272Xs/Bh/aC63jokSz/bMOpZ/HtY6Zey/C/XtV80DHf/8i8M5X2HFe7L/fUsnrRWHQIOAczNzW34n83OW9/KzlvfutFvK0kjYVgnVM8AOwee7wDODumzJElLDCvc/xbYneTOJFuAfcDRIX2WJGmJoZRlqupSko8C/xuYBB6rqpPD+CxJ0rWGVXOnqr4CfGVY7y9Jur4mv6EqSePOcJekBhnuktQgw12SGpSlX4/flE4k88CLK+54fbcB39+g7owSxz1eHPd4Wc24/1lVzS73wpsi3NcryYmqmtvsftxsjnu8OO7xst5xW5aRpAYZ7pLUoFbC/dBmd2CTOO7x4rjHy7rG3UTNXZK0WCszd0nSAMNdkho00uG+mYtw30xJHktyPskzA223JjmW5Lvd4y2b2cdhSLIzydeSnEpyMsnHuvamx55ka5Ink/x9N+7f79qbHndfkskkf5fkz7vn4zLuF5J8O8nTSU50bWse+8iG+2Yvwn2TfQ54YEnbI8DxqtoNHO+et+YS8ImqehfwXuDh7t+49bFfAN5XVe8G9gAPJHkv7Y+772PAqYHn4zJugN+oqj0D17eveewjG+4MLMJdVa8D/UW4m1NVXwd+uKR5L3C42z4MPHgz+3QzVNW5qvpmt/0qvT/47TQ+9ur5afd0uvspGh83QJIdwL8F/vtAc/PjfgNrHvsoh/tyi3Bv36S+bIY7quoc9EIQuH2T+zNUSXYB7wG+wRiMvStNPA2cB45V1ViMG/gvwH8Ergy0jcO4ofcf+FeTPJXkQNe25rEPbbGOm2DFRbjVhiRvB74IfLyqfpIs90/flqq6DOxJ8ovAl5P86iZ3aeiS/BZwvqqeSnLfJndnM9xbVWeT3A4cS/Kd9bzZKM/cx30R7leSbAPoHs9vcn+GIsk0vWD/fFV9qWsei7EDVNWPgL+id86l9XHfC/y7JC/QK7O+L8n/oP1xA1BVZ7vH88CX6ZWe1zz2UQ73cV+E+yiwv9veDzyxiX0ZivSm6J8FTlXVpwdeanrsSWa7GTtJ3gJ8APgOjY+7qh6tqh1VtYve3/NfVtW/p/FxAyR5W5J39LeB3wSeYR1jH+lvqCb5IL0aXX8R7oOb26PhSPIF4D56twB9BfgU8L+AI8A7gZeAh6pq6UnXkZbkXwH/F/g2V2uwn6RXd2927El+nd7Js0l6E7AjVfWfkvwSDY97UFeW+Q9V9VvjMO4k/5zebB165fI/q6qD6xn7SIe7JGl5o1yWkSRdh+EuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGvT/AX/o/36iaPENAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(epoch_losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get predictions for test data\n",
    "y_pred_list = []\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    for X_batch in test_loader:\n",
    "        X_batch = X_batch.to(device)\n",
    "        y_test_pred = model(X_batch)\n",
    "        y_test_pred = torch.sigmoid(y_test_pred)\n",
    "        y_pred_tag = torch.round(y_test_pred)\n",
    "        y_pred_list.append(y_pred_tag.cpu().numpy())\n",
    "\n",
    "y_pred_list = [a.squeeze().tolist() for a in y_pred_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[    0 10187]\n",
      " [    0 10095]]\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.00      0.00      0.00     10187\n",
      "           1       0.50      1.00      0.66     10095\n",
      "\n",
      "    accuracy                           0.50     20282\n",
      "   macro avg       0.25      0.50      0.33     20282\n",
      "weighted avg       0.25      0.50      0.33     20282\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rishi/anaconda3/envs/dl/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "# get performance on test data\n",
    "print(confusion_matrix(y_test, y_pred_list))\n",
    "print(classification_report(y_test, y_pred_list))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "riiid",
   "language": "python",
   "name": "riiid"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}