bare-minimum-nb-27-Oct-2020_v2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T21:48:11.675597Z",
     "start_time": "2020-10-22T21:48:11.672440Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import mean_squared_log_error\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from xgboost import XGBRegressor\n",
    "import math"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "The evaluation criteria for this Kaggle Competition is RMSLE - \"Submissions are evaluated on Root-Mean-Squared-Error (RMSE) between the logarithm of the predicted value and the logarithm of the observed sales price. (Taking logs means that errors in predicting expensive houses and cheap houses will affect the result equally.)\"\n",
    "\n",
    "The Root Mean Squared Log Error (RMSLE) can be defined using a slight modification on sklearn's mean_squared_log_error function, which itself a modification on the familiar Mean Squared Error (MSE) metric. Dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def root_mean_squared_log_error(y_validations, y_predicted):\n",
    "    if len(y_predicted) != len(y_validations):\n",
    "        return 'error: mismatch in number of data points between x and y'\n",
    "    y_predict_modified = [math.log(i) for i in y_predicted]\n",
    "    y_validations_modified = [math.log(i) for i in y_validations]\n",
    "\n",
    "    return mean_squared_error(y_validations_modified, y_predict_modified, squared=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T21:48:11.714502Z",
     "start_time": "2020-10-22T21:48:11.680226Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('./input/house-prices-advanced-regression-techniques/train.csv')\n",
    "df_test = pd.read_csv('./input/house-prices-advanced-regression-techniques/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>Street</th>\n",
       "      <th>Alley</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>Utilities</th>\n",
       "      <th>...</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>208500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>RL</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2007</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>181500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>223500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>70</td>\n",
       "      <td>RL</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2006</td>\n",
       "      <td>WD</td>\n",
       "      <td>Abnorml</td>\n",
       "      <td>140000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>84.0</td>\n",
       "      <td>14260</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 81 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \\\n",
       "0   1          60       RL         65.0     8450   Pave   NaN      Reg   \n",
       "1   2          20       RL         80.0     9600   Pave   NaN      Reg   \n",
       "2   3          60       RL         68.0    11250   Pave   NaN      IR1   \n",
       "3   4          70       RL         60.0     9550   Pave   NaN      IR1   \n",
       "4   5          60       RL         84.0    14260   Pave   NaN      IR1   \n",
       "\n",
       "  LandContour Utilities  ... PoolArea PoolQC Fence MiscFeature MiscVal MoSold  \\\n",
       "0         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      2   \n",
       "1         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      5   \n",
       "2         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      9   \n",
       "3         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      2   \n",
       "4         Lvl    AllPub  ...        0    NaN   NaN         NaN       0     12   \n",
       "\n",
       "  YrSold  SaleType  SaleCondition  SalePrice  \n",
       "0   2008        WD         Normal     208500  \n",
       "1   2007        WD         Normal     181500  \n",
       "2   2008        WD         Normal     223500  \n",
       "3   2006        WD        Abnorml     140000  \n",
       "4   2008        WD         Normal     250000  \n",
       "\n",
       "[5 rows x 81 columns]"
      ]
     },
     "execution_count": 341,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>Street</th>\n",
       "      <th>Alley</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>Utilities</th>\n",
       "      <th>...</th>\n",
       "      <th>ScreenPorch</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1461</td>\n",
       "      <td>20</td>\n",
       "      <td>RH</td>\n",
       "      <td>80.0</td>\n",
       "      <td>11622</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>120</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MnPrv</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2010</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1462</td>\n",
       "      <td>20</td>\n",
       "      <td>RL</td>\n",
       "      <td>81.0</td>\n",
       "      <td>14267</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Gar2</td>\n",
       "      <td>12500</td>\n",
       "      <td>6</td>\n",
       "      <td>2010</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1463</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>74.0</td>\n",
       "      <td>13830</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MnPrv</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2010</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1464</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>78.0</td>\n",
       "      <td>9978</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2010</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1465</td>\n",
       "      <td>120</td>\n",
       "      <td>RL</td>\n",
       "      <td>43.0</td>\n",
       "      <td>5005</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>HLS</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>144</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2010</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 80 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Id  MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \\\n",
       "0  1461          20       RH         80.0    11622   Pave   NaN      Reg   \n",
       "1  1462          20       RL         81.0    14267   Pave   NaN      IR1   \n",
       "2  1463          60       RL         74.0    13830   Pave   NaN      IR1   \n",
       "3  1464          60       RL         78.0     9978   Pave   NaN      IR1   \n",
       "4  1465         120       RL         43.0     5005   Pave   NaN      IR1   \n",
       "\n",
       "  LandContour Utilities  ... ScreenPorch PoolArea PoolQC  Fence MiscFeature  \\\n",
       "0         Lvl    AllPub  ...         120        0    NaN  MnPrv         NaN   \n",
       "1         Lvl    AllPub  ...           0        0    NaN    NaN        Gar2   \n",
       "2         Lvl    AllPub  ...           0        0    NaN  MnPrv         NaN   \n",
       "3         Lvl    AllPub  ...           0        0    NaN    NaN         NaN   \n",
       "4         HLS    AllPub  ...         144        0    NaN    NaN         NaN   \n",
       "\n",
       "  MiscVal MoSold  YrSold  SaleType  SaleCondition  \n",
       "0       0      6    2010        WD         Normal  \n",
       "1   12500      6    2010        WD         Normal  \n",
       "2       0      3    2010        WD         Normal  \n",
       "3       0      6    2010        WD         Normal  \n",
       "4       0      1    2010        WD         Normal  \n",
       "\n",
       "[5 rows x 80 columns]"
      ]
     },
     "execution_count": 342,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T21:48:11.802006Z",
     "start_time": "2020-10-22T21:48:11.715829Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>OverallQual</th>\n",
       "      <th>OverallCond</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>YearRemodAdd</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>BsmtFinSF1</th>\n",
       "      <th>...</th>\n",
       "      <th>WoodDeckSF</th>\n",
       "      <th>OpenPorchSF</th>\n",
       "      <th>EnclosedPorch</th>\n",
       "      <th>3SsnPorch</th>\n",
       "      <th>ScreenPorch</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1201.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1452.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "      <td>1460.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>730.500000</td>\n",
       "      <td>56.897260</td>\n",
       "      <td>70.049958</td>\n",
       "      <td>10516.828082</td>\n",
       "      <td>6.099315</td>\n",
       "      <td>5.575342</td>\n",
       "      <td>1971.267808</td>\n",
       "      <td>1984.865753</td>\n",
       "      <td>103.685262</td>\n",
       "      <td>443.639726</td>\n",
       "      <td>...</td>\n",
       "      <td>94.244521</td>\n",
       "      <td>46.660274</td>\n",
       "      <td>21.954110</td>\n",
       "      <td>3.409589</td>\n",
       "      <td>15.060959</td>\n",
       "      <td>2.758904</td>\n",
       "      <td>43.489041</td>\n",
       "      <td>6.321918</td>\n",
       "      <td>2007.815753</td>\n",
       "      <td>180921.195890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>421.610009</td>\n",
       "      <td>42.300571</td>\n",
       "      <td>24.284752</td>\n",
       "      <td>9981.264932</td>\n",
       "      <td>1.382997</td>\n",
       "      <td>1.112799</td>\n",
       "      <td>30.202904</td>\n",
       "      <td>20.645407</td>\n",
       "      <td>181.066207</td>\n",
       "      <td>456.098091</td>\n",
       "      <td>...</td>\n",
       "      <td>125.338794</td>\n",
       "      <td>66.256028</td>\n",
       "      <td>61.119149</td>\n",
       "      <td>29.317331</td>\n",
       "      <td>55.757415</td>\n",
       "      <td>40.177307</td>\n",
       "      <td>496.123024</td>\n",
       "      <td>2.703626</td>\n",
       "      <td>1.328095</td>\n",
       "      <td>79442.502883</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>1300.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1872.000000</td>\n",
       "      <td>1950.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2006.000000</td>\n",
       "      <td>34900.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>365.750000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>59.000000</td>\n",
       "      <td>7553.500000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1954.000000</td>\n",
       "      <td>1967.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>2007.000000</td>\n",
       "      <td>129975.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>730.500000</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>69.000000</td>\n",
       "      <td>9478.500000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1973.000000</td>\n",
       "      <td>1994.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>383.500000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>2008.000000</td>\n",
       "      <td>163000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1095.250000</td>\n",
       "      <td>70.000000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>11601.500000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>2000.000000</td>\n",
       "      <td>2004.000000</td>\n",
       "      <td>166.000000</td>\n",
       "      <td>712.250000</td>\n",
       "      <td>...</td>\n",
       "      <td>168.000000</td>\n",
       "      <td>68.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>2009.000000</td>\n",
       "      <td>214000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1460.000000</td>\n",
       "      <td>190.000000</td>\n",
       "      <td>313.000000</td>\n",
       "      <td>215245.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>2010.000000</td>\n",
       "      <td>2010.000000</td>\n",
       "      <td>1600.000000</td>\n",
       "      <td>5644.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>857.000000</td>\n",
       "      <td>547.000000</td>\n",
       "      <td>552.000000</td>\n",
       "      <td>508.000000</td>\n",
       "      <td>480.000000</td>\n",
       "      <td>738.000000</td>\n",
       "      <td>15500.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>2010.000000</td>\n",
       "      <td>755000.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 38 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                Id   MSSubClass  LotFrontage        LotArea  OverallQual  \\\n",
       "count  1460.000000  1460.000000  1201.000000    1460.000000  1460.000000   \n",
       "mean    730.500000    56.897260    70.049958   10516.828082     6.099315   \n",
       "std     421.610009    42.300571    24.284752    9981.264932     1.382997   \n",
       "min       1.000000    20.000000    21.000000    1300.000000     1.000000   \n",
       "25%     365.750000    20.000000    59.000000    7553.500000     5.000000   \n",
       "50%     730.500000    50.000000    69.000000    9478.500000     6.000000   \n",
       "75%    1095.250000    70.000000    80.000000   11601.500000     7.000000   \n",
       "max    1460.000000   190.000000   313.000000  215245.000000    10.000000   \n",
       "\n",
       "       OverallCond    YearBuilt  YearRemodAdd   MasVnrArea   BsmtFinSF1  ...  \\\n",
       "count  1460.000000  1460.000000   1460.000000  1452.000000  1460.000000  ...   \n",
       "mean      5.575342  1971.267808   1984.865753   103.685262   443.639726  ...   \n",
       "std       1.112799    30.202904     20.645407   181.066207   456.098091  ...   \n",
       "min       1.000000  1872.000000   1950.000000     0.000000     0.000000  ...   \n",
       "25%       5.000000  1954.000000   1967.000000     0.000000     0.000000  ...   \n",
       "50%       5.000000  1973.000000   1994.000000     0.000000   383.500000  ...   \n",
       "75%       6.000000  2000.000000   2004.000000   166.000000   712.250000  ...   \n",
       "max       9.000000  2010.000000   2010.000000  1600.000000  5644.000000  ...   \n",
       "\n",
       "        WoodDeckSF  OpenPorchSF  EnclosedPorch    3SsnPorch  ScreenPorch  \\\n",
       "count  1460.000000  1460.000000    1460.000000  1460.000000  1460.000000   \n",
       "mean     94.244521    46.660274      21.954110     3.409589    15.060959   \n",
       "std     125.338794    66.256028      61.119149    29.317331    55.757415   \n",
       "min       0.000000     0.000000       0.000000     0.000000     0.000000   \n",
       "25%       0.000000     0.000000       0.000000     0.000000     0.000000   \n",
       "50%       0.000000    25.000000       0.000000     0.000000     0.000000   \n",
       "75%     168.000000    68.000000       0.000000     0.000000     0.000000   \n",
       "max     857.000000   547.000000     552.000000   508.000000   480.000000   \n",
       "\n",
       "          PoolArea       MiscVal       MoSold       YrSold      SalePrice  \n",
       "count  1460.000000   1460.000000  1460.000000  1460.000000    1460.000000  \n",
       "mean      2.758904     43.489041     6.321918  2007.815753  180921.195890  \n",
       "std      40.177307    496.123024     2.703626     1.328095   79442.502883  \n",
       "min       0.000000      0.000000     1.000000  2006.000000   34900.000000  \n",
       "25%       0.000000      0.000000     5.000000  2007.000000  129975.000000  \n",
       "50%       0.000000      0.000000     6.000000  2008.000000  163000.000000  \n",
       "75%       0.000000      0.000000     8.000000  2009.000000  214000.000000  \n",
       "max     738.000000  15500.000000    12.000000  2010.000000  755000.000000  \n",
       "\n",
       "[8 rows x 38 columns]"
      ]
     },
     "execution_count": 343,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T21:48:11.876652Z",
     "start_time": "2020-10-22T21:48:11.849629Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1460 entries, 0 to 1459\n",
      "Data columns (total 81 columns):\n",
      " #   Column         Non-Null Count  Dtype  \n",
      "---  ------         --------------  -----  \n",
      " 0   Id             1460 non-null   int64  \n",
      " 1   MSSubClass     1460 non-null   int64  \n",
      " 2   MSZoning       1460 non-null   object \n",
      " 3   LotFrontage    1201 non-null   float64\n",
      " 4   LotArea        1460 non-null   int64  \n",
      " 5   Street         1460 non-null   object \n",
      " 6   Alley          91 non-null     object \n",
      " 7   LotShape       1460 non-null   object \n",
      " 8   LandContour    1460 non-null   object \n",
      " 9   Utilities      1460 non-null   object \n",
      " 10  LotConfig      1460 non-null   object \n",
      " 11  LandSlope      1460 non-null   object \n",
      " 12  Neighborhood   1460 non-null   object \n",
      " 13  Condition1     1460 non-null   object \n",
      " 14  Condition2     1460 non-null   object \n",
      " 15  BldgType       1460 non-null   object \n",
      " 16  HouseStyle     1460 non-null   object \n",
      " 17  OverallQual    1460 non-null   int64  \n",
      " 18  OverallCond    1460 non-null   int64  \n",
      " 19  YearBuilt      1460 non-null   int64  \n",
      " 20  YearRemodAdd   1460 non-null   int64  \n",
      " 21  RoofStyle      1460 non-null   object \n",
      " 22  RoofMatl       1460 non-null   object \n",
      " 23  Exterior1st    1460 non-null   object \n",
      " 24  Exterior2nd    1460 non-null   object \n",
      " 25  MasVnrType     1452 non-null   object \n",
      " 26  MasVnrArea     1452 non-null   float64\n",
      " 27  ExterQual      1460 non-null   object \n",
      " 28  ExterCond      1460 non-null   object \n",
      " 29  Foundation     1460 non-null   object \n",
      " 30  BsmtQual       1423 non-null   object \n",
      " 31  BsmtCond       1423 non-null   object \n",
      " 32  BsmtExposure   1422 non-null   object \n",
      " 33  BsmtFinType1   1423 non-null   object \n",
      " 34  BsmtFinSF1     1460 non-null   int64  \n",
      " 35  BsmtFinType2   1422 non-null   object \n",
      " 36  BsmtFinSF2     1460 non-null   int64  \n",
      " 37  BsmtUnfSF      1460 non-null   int64  \n",
      " 38  TotalBsmtSF    1460 non-null   int64  \n",
      " 39  Heating        1460 non-null   object \n",
      " 40  HeatingQC      1460 non-null   object \n",
      " 41  CentralAir     1460 non-null   object \n",
      " 42  Electrical     1459 non-null   object \n",
      " 43  1stFlrSF       1460 non-null   int64  \n",
      " 44  2ndFlrSF       1460 non-null   int64  \n",
      " 45  LowQualFinSF   1460 non-null   int64  \n",
      " 46  GrLivArea      1460 non-null   int64  \n",
      " 47  BsmtFullBath   1460 non-null   int64  \n",
      " 48  BsmtHalfBath   1460 non-null   int64  \n",
      " 49  FullBath       1460 non-null   int64  \n",
      " 50  HalfBath       1460 non-null   int64  \n",
      " 51  BedroomAbvGr   1460 non-null   int64  \n",
      " 52  KitchenAbvGr   1460 non-null   int64  \n",
      " 53  KitchenQual    1460 non-null   object \n",
      " 54  TotRmsAbvGrd   1460 non-null   int64  \n",
      " 55  Functional     1460 non-null   object \n",
      " 56  Fireplaces     1460 non-null   int64  \n",
      " 57  FireplaceQu    770 non-null    object \n",
      " 58  GarageType     1379 non-null   object \n",
      " 59  GarageYrBlt    1379 non-null   float64\n",
      " 60  GarageFinish   1379 non-null   object \n",
      " 61  GarageCars     1460 non-null   int64  \n",
      " 62  GarageArea     1460 non-null   int64  \n",
      " 63  GarageQual     1379 non-null   object \n",
      " 64  GarageCond     1379 non-null   object \n",
      " 65  PavedDrive     1460 non-null   object \n",
      " 66  WoodDeckSF     1460 non-null   int64  \n",
      " 67  OpenPorchSF    1460 non-null   int64  \n",
      " 68  EnclosedPorch  1460 non-null   int64  \n",
      " 69  3SsnPorch      1460 non-null   int64  \n",
      " 70  ScreenPorch    1460 non-null   int64  \n",
      " 71  PoolArea       1460 non-null   int64  \n",
      " 72  PoolQC         7 non-null      object \n",
      " 73  Fence          281 non-null    object \n",
      " 74  MiscFeature    54 non-null     object \n",
      " 75  MiscVal        1460 non-null   int64  \n",
      " 76  MoSold         1460 non-null   int64  \n",
      " 77  YrSold         1460 non-null   int64  \n",
      " 78  SaleType       1460 non-null   object \n",
      " 79  SaleCondition  1460 non-null   object \n",
      " 80  SalePrice      1460 non-null   int64  \n",
      "dtypes: float64(3), int64(35), object(43)\n",
      "memory usage: 924.0+ KB\n"
     ]
    }
   ],
   "source": [
    "# Get the overall concise summary of the DataFrame\n",
    "df_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T21:48:11.882978Z",
     "start_time": "2020-10-22T21:48:11.877846Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Id', 'MSSubClass', 'LotFrontage', 'LotArea', 'OverallQual',\n",
       "       'OverallCond', 'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1',\n",
       "       'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', '1stFlrSF', '2ndFlrSF',\n",
       "       'LowQualFinSF', 'GrLivArea', 'BsmtFullBath', 'BsmtHalfBath', 'FullBath',\n",
       "       'HalfBath', 'BedroomAbvGr', 'KitchenAbvGr', 'TotRmsAbvGrd',\n",
       "       'Fireplaces', 'GarageYrBlt', 'GarageCars', 'GarageArea', 'WoodDeckSF',\n",
       "       'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea',\n",
       "       'MiscVal', 'MoSold', 'YrSold', 'SalePrice'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 345,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Lets check what are the numerical features in the df_train dataset\n",
    "numeric_features = df_train.select_dtypes(include=[np.number])\n",
    "numeric_features.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T21:48:11.905145Z",
     "start_time": "2020-10-22T21:48:11.884698Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>OverallQual</th>\n",
       "      <th>OverallCond</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>YearRemodAdd</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>BsmtFinSF1</th>\n",
       "      <th>...</th>\n",
       "      <th>WoodDeckSF</th>\n",
       "      <th>OpenPorchSF</th>\n",
       "      <th>EnclosedPorch</th>\n",
       "      <th>3SsnPorch</th>\n",
       "      <th>ScreenPorch</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>2003</td>\n",
       "      <td>2003</td>\n",
       "      <td>196.0</td>\n",
       "      <td>706</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>61</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2008</td>\n",
       "      <td>208500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>1976</td>\n",
       "      <td>1976</td>\n",
       "      <td>0.0</td>\n",
       "      <td>978</td>\n",
       "      <td>...</td>\n",
       "      <td>298</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2007</td>\n",
       "      <td>181500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>60</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>2001</td>\n",
       "      <td>2002</td>\n",
       "      <td>162.0</td>\n",
       "      <td>486</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>42</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>223500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>70</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>1915</td>\n",
       "      <td>1970</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>272</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2006</td>\n",
       "      <td>140000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>60</td>\n",
       "      <td>84.0</td>\n",
       "      <td>14260</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>2000</td>\n",
       "      <td>2000</td>\n",
       "      <td>350.0</td>\n",
       "      <td>655</td>\n",
       "      <td>...</td>\n",
       "      <td>192</td>\n",
       "      <td>84</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>2008</td>\n",
       "      <td>250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 38 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  MSSubClass  LotFrontage  LotArea  OverallQual  OverallCond  YearBuilt  \\\n",
       "0   1          60         65.0     8450            7            5       2003   \n",
       "1   2          20         80.0     9600            6            8       1976   \n",
       "2   3          60         68.0    11250            7            5       2001   \n",
       "3   4          70         60.0     9550            7            5       1915   \n",
       "4   5          60         84.0    14260            8            5       2000   \n",
       "\n",
       "   YearRemodAdd  MasVnrArea  BsmtFinSF1  ...  WoodDeckSF  OpenPorchSF  \\\n",
       "0          2003       196.0         706  ...           0           61   \n",
       "1          1976         0.0         978  ...         298            0   \n",
       "2          2002       162.0         486  ...           0           42   \n",
       "3          1970         0.0         216  ...           0           35   \n",
       "4          2000       350.0         655  ...         192           84   \n",
       "\n",
       "   EnclosedPorch  3SsnPorch  ScreenPorch  PoolArea  MiscVal  MoSold  YrSold  \\\n",
       "0              0          0            0         0        0       2    2008   \n",
       "1              0          0            0         0        0       5    2007   \n",
       "2              0          0            0         0        0       9    2008   \n",
       "3            272          0            0         0        0       2    2006   \n",
       "4              0          0            0         0        0      12    2008   \n",
       "\n",
       "   SalePrice  \n",
       "0     208500  \n",
       "1     181500  \n",
       "2     223500  \n",
       "3     140000  \n",
       "4     250000  \n",
       "\n",
       "[5 rows x 38 columns]"
      ]
     },
     "execution_count": 346,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# numeric features head\n",
    "numeric_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T21:48:11.917732Z",
     "start_time": "2020-10-22T21:48:11.906955Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count      1460.000000\n",
       "mean     180921.195890\n",
       "std       79442.502883\n",
       "min       34900.000000\n",
       "25%      129975.000000\n",
       "50%      163000.000000\n",
       "75%      214000.000000\n",
       "max      755000.000000\n",
       "Name: SalePrice, dtype: float64"
      ]
     },
     "execution_count": 347,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now a general description of SalePrice\n",
    "# Now, as we will be predicting 'SalePrice' lets see description of that column\n",
    "df_train['SalePrice'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['LotFrontage', 'LotArea', 'OverallQual', 'YearBuilt',\n",
       "       'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1', 'BsmtUnfSF',\n",
       "       'TotalBsmtSF', '1stFlrSF', '2ndFlrSF', 'GrLivArea', 'BsmtFullBath',\n",
       "       'FullBath', 'HalfBath', 'TotRmsAbvGrd', 'Fireplaces',\n",
       "       'GarageYrBlt', 'GarageCars', 'GarageArea', 'WoodDeckSF',\n",
       "       'OpenPorchSF', 'SalePrice'], dtype=object)"
      ]
     },
     "execution_count": 348,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_correlations = df_train.corr()\n",
    "top_feature_columns = top_correlations['SalePrice'][top_correlations['SalePrice'].values > 0.2].index.values\n",
    "top_feature_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T21:48:15.569999Z",
     "start_time": "2020-10-22T21:48:14.769510Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Handling Missing Values for 19 features which have missing values mentioned above\n",
    "df_train['GarageYrBlt'] = df_train['GarageYrBlt'].fillna(0)\n",
    "# filling in missing GarageYrBuilt values with zeros.  \n",
    "# But this may not be the most logical approach - refer to this discussion below for mor perspective\n",
    "# https://www.kaggle.com/c/house-prices-advanced-regression-techniques/discussion/60143\n",
    "\n",
    "# similary fillingup na valuse for couple of other features\n",
    "df_train['LotFrontage'] = df_train.groupby('Neighborhood')['LotFrontage'].transform(lambda x: x.fillna(x.median()))\n",
    "df_train['MasVnrArea'] = df_train['MasVnrArea'].fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T21:48:32.098258Z",
     "start_time": "2020-10-22T21:48:32.079678Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Total</th>\n",
       "      <th>Percent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>PoolQC</th>\n",
       "      <td>1453</td>\n",
       "      <td>0.995205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MiscFeature</th>\n",
       "      <td>1406</td>\n",
       "      <td>0.963014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Alley</th>\n",
       "      <td>1369</td>\n",
       "      <td>0.937671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fence</th>\n",
       "      <td>1179</td>\n",
       "      <td>0.807534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FireplaceQu</th>\n",
       "      <td>690</td>\n",
       "      <td>0.472603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GarageType</th>\n",
       "      <td>81</td>\n",
       "      <td>0.055479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GarageCond</th>\n",
       "      <td>81</td>\n",
       "      <td>0.055479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GarageFinish</th>\n",
       "      <td>81</td>\n",
       "      <td>0.055479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GarageQual</th>\n",
       "      <td>81</td>\n",
       "      <td>0.055479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BsmtExposure</th>\n",
       "      <td>38</td>\n",
       "      <td>0.026027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BsmtFinType2</th>\n",
       "      <td>38</td>\n",
       "      <td>0.026027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BsmtFinType1</th>\n",
       "      <td>37</td>\n",
       "      <td>0.025342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BsmtCond</th>\n",
       "      <td>37</td>\n",
       "      <td>0.025342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BsmtQual</th>\n",
       "      <td>37</td>\n",
       "      <td>0.025342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MasVnrType</th>\n",
       "      <td>8</td>\n",
       "      <td>0.005479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Electrical</th>\n",
       "      <td>1</td>\n",
       "      <td>0.000685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>YearRemodAdd</th>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RoofStyle</th>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RoofMatl</th>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ExterQual</th>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Total   Percent\n",
       "PoolQC         1453  0.995205\n",
       "MiscFeature    1406  0.963014\n",
       "Alley          1369  0.937671\n",
       "Fence          1179  0.807534\n",
       "FireplaceQu     690  0.472603\n",
       "GarageType       81  0.055479\n",
       "GarageCond       81  0.055479\n",
       "GarageFinish     81  0.055479\n",
       "GarageQual       81  0.055479\n",
       "BsmtExposure     38  0.026027\n",
       "BsmtFinType2     38  0.026027\n",
       "BsmtFinType1     37  0.025342\n",
       "BsmtCond         37  0.025342\n",
       "BsmtQual         37  0.025342\n",
       "MasVnrType        8  0.005479\n",
       "Electrical        1  0.000685\n",
       "YearRemodAdd      0  0.000000\n",
       "RoofStyle         0  0.000000\n",
       "RoofMatl          0  0.000000\n",
       "ExterQual         0  0.000000"
      ]
     },
     "execution_count": 350,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Checking out missing data\n",
    "total = df_train.isnull().sum().sort_values(ascending=False)\n",
    "percent = (df_train.isnull().sum() / df_train.isnull().count()).sort_values(ascending=False)\n",
    "missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n",
    "missing_data.head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### And we will use the famous LinearRegression() function of scikit-learn to implement this in a while.\n",
    "First some data pre-procession.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T21:48:36.683800Z",
     "start_time": "2020-10-22T21:48:36.674703Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OverallQual</th>\n",
       "      <th>GrLivArea</th>\n",
       "      <th>GarageCars</th>\n",
       "      <th>TotalBsmtSF</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>FullBath</th>\n",
       "      <th>OpenPorchSF</th>\n",
       "      <th>WoodDeckSF</th>\n",
       "      <th>2ndFlrSF</th>\n",
       "      <th>YearRemodAdd</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7</td>\n",
       "      <td>1710</td>\n",
       "      <td>2</td>\n",
       "      <td>856</td>\n",
       "      <td>2003</td>\n",
       "      <td>2</td>\n",
       "      <td>61</td>\n",
       "      <td>0</td>\n",
       "      <td>854</td>\n",
       "      <td>2003</td>\n",
       "      <td>196.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>1262</td>\n",
       "      <td>2</td>\n",
       "      <td>1262</td>\n",
       "      <td>1976</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>298</td>\n",
       "      <td>0</td>\n",
       "      <td>1976</td>\n",
       "      <td>0.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>1786</td>\n",
       "      <td>2</td>\n",
       "      <td>920</td>\n",
       "      <td>2001</td>\n",
       "      <td>2</td>\n",
       "      <td>42</td>\n",
       "      <td>0</td>\n",
       "      <td>866</td>\n",
       "      <td>2002</td>\n",
       "      <td>162.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7</td>\n",
       "      <td>1717</td>\n",
       "      <td>3</td>\n",
       "      <td>756</td>\n",
       "      <td>1915</td>\n",
       "      <td>1</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>756</td>\n",
       "      <td>1970</td>\n",
       "      <td>0.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>2198</td>\n",
       "      <td>3</td>\n",
       "      <td>1145</td>\n",
       "      <td>2000</td>\n",
       "      <td>2</td>\n",
       "      <td>84</td>\n",
       "      <td>192</td>\n",
       "      <td>1053</td>\n",
       "      <td>2000</td>\n",
       "      <td>350.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>14260</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   OverallQual  GrLivArea  GarageCars  TotalBsmtSF  YearBuilt  FullBath  \\\n",
       "0            7       1710           2          856       2003         2   \n",
       "1            6       1262           2         1262       1976         2   \n",
       "2            7       1786           2          920       2001         2   \n",
       "3            7       1717           3          756       1915         1   \n",
       "4            8       2198           3         1145       2000         2   \n",
       "\n",
       "   OpenPorchSF  WoodDeckSF  2ndFlrSF  YearRemodAdd  MasVnrArea  LotFrontage  \\\n",
       "0           61           0       854          2003       196.0         65.0   \n",
       "1            0         298         0          1976         0.0         80.0   \n",
       "2           42           0       866          2002       162.0         68.0   \n",
       "3           35           0       756          1970         0.0         60.0   \n",
       "4           84         192      1053          2000       350.0         84.0   \n",
       "\n",
       "   LotArea  \n",
       "0     8450  \n",
       "1     9600  \n",
       "2    11250  \n",
       "3     9550  \n",
       "4    14260  "
      ]
     },
     "execution_count": 351,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "train_target_label = df_train['SalePrice']\n",
    "# labels are dependent variables whose values are to be predicted. \n",
    "top_feature_columns_modified = ['OverallQual', 'GrLivArea', 'GarageCars', 'TotalBsmtSF', 'YearBuilt', 'FullBath', 'OpenPorchSF', \n",
    "        'WoodDeckSF', '2ndFlrSF', 'YearRemodAdd', 'MasVnrArea', 'LotFrontage', 'LotArea']\n",
    "\n",
    "training_sample_df = df_train[top_feature_columns_modified]\n",
    "test_sample_df = df_test[top_feature_columns_modified]\n",
    "\n",
    "training_sample_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T21:48:36.702627Z",
     "start_time": "2020-10-22T21:48:36.685027Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OverallQual</th>\n",
       "      <th>GrLivArea</th>\n",
       "      <th>GarageCars</th>\n",
       "      <th>TotalBsmtSF</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>FullBath</th>\n",
       "      <th>OpenPorchSF</th>\n",
       "      <th>WoodDeckSF</th>\n",
       "      <th>2ndFlrSF</th>\n",
       "      <th>YearRemodAdd</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>896</td>\n",
       "      <td>1.0</td>\n",
       "      <td>882.0</td>\n",
       "      <td>1961</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>140</td>\n",
       "      <td>0</td>\n",
       "      <td>1961</td>\n",
       "      <td>0.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>11622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>1329</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1329.0</td>\n",
       "      <td>1958</td>\n",
       "      <td>1</td>\n",
       "      <td>36</td>\n",
       "      <td>393</td>\n",
       "      <td>0</td>\n",
       "      <td>1958</td>\n",
       "      <td>108.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>14267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>1629</td>\n",
       "      <td>2.0</td>\n",
       "      <td>928.0</td>\n",
       "      <td>1997</td>\n",
       "      <td>2</td>\n",
       "      <td>34</td>\n",
       "      <td>212</td>\n",
       "      <td>701</td>\n",
       "      <td>1998</td>\n",
       "      <td>0.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>13830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>1604</td>\n",
       "      <td>2.0</td>\n",
       "      <td>926.0</td>\n",
       "      <td>1998</td>\n",
       "      <td>2</td>\n",
       "      <td>36</td>\n",
       "      <td>360</td>\n",
       "      <td>678</td>\n",
       "      <td>1998</td>\n",
       "      <td>20.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>9978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>1280</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1280.0</td>\n",
       "      <td>1992</td>\n",
       "      <td>2</td>\n",
       "      <td>82</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1992</td>\n",
       "      <td>0.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>5005</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   OverallQual  GrLivArea  GarageCars  TotalBsmtSF  YearBuilt  FullBath  \\\n",
       "0            5        896         1.0        882.0       1961         1   \n",
       "1            6       1329         1.0       1329.0       1958         1   \n",
       "2            5       1629         2.0        928.0       1997         2   \n",
       "3            6       1604         2.0        926.0       1998         2   \n",
       "4            8       1280         2.0       1280.0       1992         2   \n",
       "\n",
       "   OpenPorchSF  WoodDeckSF  2ndFlrSF  YearRemodAdd  MasVnrArea  LotFrontage  \\\n",
       "0            0         140         0          1961         0.0         80.0   \n",
       "1           36         393         0          1958       108.0         81.0   \n",
       "2           34         212       701          1998         0.0         74.0   \n",
       "3           36         360       678          1998        20.0         78.0   \n",
       "4           82           0         0          1992         0.0         43.0   \n",
       "\n",
       "   LotArea  \n",
       "0    11622  \n",
       "1    14267  \n",
       "2    13830  \n",
       "3     9978  \n",
       "4     5005  "
      ]
     },
     "execution_count": 352,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_sample_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Do some data pre-processing by replacing all null values with median\n",
    "using scikit-learn's Imputer class.\n",
    "\n",
    "The strategy parameter is optional and takes a string, (default=\"mean\")\n",
    "\n",
    "If \"mean\", then replace missing values using the mean along the axis.\n",
    "If \"median\", then replace missing values using the median along the axis.\n",
    "If \"most_frequent\", then replace missing using the most frequent value along the axis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T21:48:36.724447Z",
     "start_time": "2020-10-22T21:48:36.704603Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "imputer = SimpleImputer(strategy = 'median')\n",
    "\n",
    "# During fit() the imputer learns about the mean, median etc of the data,\n",
    "# which is then applied to the missing values during transform().\n",
    "imputer.fit(training_sample_df)\n",
    "imputer.fit(test_sample_df)\n",
    "\n",
    "# Note - sklearn.preprocessing.Imputer.fit_transform returns a new array, it doesn't alter the argument array.\n",
    "training_sample_df = imputer.transform(training_sample_df)\n",
    "test_sample_df = imputer.transform(test_sample_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "Previously we standardized the data with manual plain-python implement\n",
    "#### Now we will use sklearn function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T21:48:36.731759Z",
     "start_time": "2020-10-22T21:48:36.726348Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "\n",
    "# Again, during fit() the imputer learns about the mean, median etc of the data,\n",
    "# which is then applied to the missing values during transform().\n",
    "scaler.fit(training_sample_df)\n",
    "scaler.fit(test_sample_df)\n",
    "\n",
    "training_sample_df = scaler.transform(training_sample_df)\n",
    "test_sample_df = scaler.transform(test_sample_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T21:48:36.742280Z",
     "start_time": "2020-10-22T21:48:36.732864Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1168, 13)"
      ]
     },
     "execution_count": 355,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Syntax of train_test_split\n",
    "# X_train_split, X_test_split, y_train, y_test = train_test_split(X, y, train_size=0.33, random_state=42)\n",
    "X_train_split, X_test_split, y_train, y_test = train_test_split(training_sample_df, train_target_label, random_state=42, train_size = 0.8 )\n",
    "\n",
    "X_train_split.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T21:48:36.752121Z",
     "start_time": "2020-10-22T21:48:36.743302Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(292, 13)"
      ]
     },
     "execution_count": 356,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_split.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1460, 13)"
      ]
     },
     "execution_count": 357,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_sample_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1459, 13)"
      ]
     },
     "execution_count": 358,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_sample_df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Run a scikit-learn Linear Regression model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T21:48:36.763301Z",
     "start_time": "2020-10-22T21:48:36.753114Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, normalize=False)"
      ]
     },
     "execution_count": 359,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = LinearRegression()\n",
    "# we have to fit this model to our data, in other words, we have to make it “learn” using our training data.\n",
    "# The syntax of the fit function is just plain model.fit(X_train_split, Y_train)\n",
    "model.fit(training_sample_df, train_target_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Intercept is  177921.01851212673\n",
      "Slope i.e. coefficient of x is  [28196.91947993 26349.76648831  9645.4413169   7658.2055073\n",
      "  6637.47089276 -2965.55939821   768.02475758  4437.72171596\n",
      " -4227.6251946   6888.80707764  5992.62389513   579.10402749\n",
      "  2935.94808718]\n"
     ]
    }
   ],
   "source": [
    "print('Intercept is ', model.intercept_)\n",
    "\n",
    "# For retrieving the slope (coefficient of x). This will be an array of values.\n",
    "print(\"Slope i.e. coefficient of x is \", model.coef_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Regression score is 0.7858973705366121\n",
      "train_target_label is  0       208500\n",
      "1       181500\n",
      "2       223500\n",
      "3       140000\n",
      "4       250000\n",
      "         ...  \n",
      "1455    175000\n",
      "1456    210000\n",
      "1457    266500\n",
      "1458    142125\n",
      "1459    147500\n",
      "Name: SalePrice, Length: 1460, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# regression = model.fit(training_sample_df, train_target_label)\n",
    "print(\"Regression score is\", model.score(training_sample_df, train_target_label))\n",
    "print('train_target_label is ', train_target_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that my model is trained with the training data set, I can now start testing the model with the testing dataset to Predict Sales Price from Test data set.\n",
    "The y_predict is a numpy array that contains all the predicted values for the input values in the X_test_split series.\n",
    "\n",
    "The general syntax is for making predictions on the test data, I have to execute the following form of:\n",
    "\n",
    "```python\n",
    "regressor = LinearRegression()\n",
    "y_predition_from_model = regressor.predict(X_test_split)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-10-22T21:48:36.771805Z",
     "start_time": "2020-10-22T21:48:36.764499Z"
    },
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "y_predict = model.predict(X_test_split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Root Mean Square Error is  36992.39012928462\n",
      "Root Mean Square Log Error is  0.26703294966342594\n"
     ]
    }
   ],
   "source": [
    "rmse_simple_linear = np.sqrt(metrics.mean_squared_error(y_test, y_predict))\n",
    "print('Root Mean Square Error is ', rmse_simple_linear)\n",
    "\n",
    "# MSE_Log\n",
    "rmse_log_simple_linear = np.sqrt(mean_squared_log_error(y_test, y_predict))\n",
    "print('Root Mean Square Log Error is ', rmse_log_simple_linear)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1461</td>\n",
       "      <td>108125.218262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1462</td>\n",
       "      <td>171809.395377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1463</td>\n",
       "      <td>172711.837537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1464</td>\n",
       "      <td>195073.734546</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1465</td>\n",
       "      <td>209815.191496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1454</th>\n",
       "      <td>2915</td>\n",
       "      <td>68025.514633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1455</th>\n",
       "      <td>2916</td>\n",
       "      <td>80707.019360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1456</th>\n",
       "      <td>2917</td>\n",
       "      <td>174307.976512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1457</th>\n",
       "      <td>2918</td>\n",
       "      <td>114170.511870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1458</th>\n",
       "      <td>2919</td>\n",
       "      <td>240588.774556</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1459 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        ID      SalePrice\n",
       "0     1461  108125.218262\n",
       "1     1462  171809.395377\n",
       "2     1463  172711.837537\n",
       "3     1464  195073.734546\n",
       "4     1465  209815.191496\n",
       "...    ...            ...\n",
       "1454  2915   68025.514633\n",
       "1455  2916   80707.019360\n",
       "1456  2917  174307.976512\n",
       "1457  2918  114170.511870\n",
       "1458  2919  240588.774556\n",
       "\n",
       "[1459 rows x 2 columns]"
      ]
     },
     "execution_count": 364,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction_on_test_data = model.predict(test_sample_df)\n",
    "# print(\"Regression score on test sample data is\", model.score(test_sample_df, train_target_label ))\n",
    "testID = df_test['Id']\n",
    "predict_submission = pd.DataFrame()\n",
    "predict_submission['ID'] = testID\n",
    "predict_submission['SalePrice'] = prediction_on_test_data\n",
    "predict_submission"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ridge Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE of Linear Ridge is  36992.54832288\n",
      "MSE Log of Linear Ridge is  0.26682113387130174\n"
     ]
    }
   ],
   "source": [
    "model_ridge = Ridge(alpha=0.5)\n",
    "model_ridge.fit(training_sample_df, train_target_label)\n",
    "y_predict_ridge = model_ridge.predict(X_test_split)\n",
    "\n",
    "mse_linear_ridge = np.sqrt(metrics.mean_squared_error(y_test, y_predict_ridge))\n",
    "print('MSE of Linear Ridge is ', mse_linear_ridge)\n",
    "\n",
    "# Log Error\n",
    "mse_log_linear_ridge = np.sqrt(mean_squared_log_error(y_test, y_predict_ridge))\n",
    "print('MSE Log of Linear Ridge is ', mse_log_linear_ridge)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### And now MSE Calculation for Lasso Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE of Lasso Regression is  37002.420402536394\n",
      "MSE Log of Lasso Regression is  0.2665108348893785\n"
     ]
    }
   ],
   "source": [
    "model_lasso = Lasso(alpha=33)\n",
    "model_lasso.fit(training_sample_df, train_target_label)\n",
    "y_predict_lasso = model_lasso.predict(X_test_split)\n",
    "\n",
    "mse_linear_lasso = np.sqrt(metrics.mean_squared_error(y_test, y_predict_lasso))\n",
    "print('MSE of Lasso Regression is ', mse_linear_lasso)\n",
    "\n",
    "# Log Error\n",
    "mse_log_linear_lasso = np.sqrt(mean_squared_log_error(y_test, y_predict_lasso))\n",
    "print('MSE Log of Lasso Regression is ', mse_log_linear_lasso)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "##### Now we will be using Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE Random Forest is  10885.041435808504\n",
      "MSE Log Random Forest is  0.06787969532802059\n"
     ]
    }
   ],
   "source": [
    "RFR = RandomForestRegressor(max_depth=50)\n",
    "RFR.fit(training_sample_df, train_target_label)\n",
    "\n",
    "y_predict_random_forest = RFR.predict(X_test_split)\n",
    "\n",
    "mse_random_forest = np.sqrt(metrics.mean_squared_error(y_test, y_predict_random_forest))\n",
    "print('MSE Random Forest is ', mse_random_forest)\n",
    "\n",
    "# Log Error\n",
    "mse_log_random_forest = np.sqrt(mean_squared_log_error(y_test, y_predict_random_forest))\n",
    "print('MSE Log Random Forest is ', mse_log_random_forest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now final predictions for submissions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indexex  [1461 1462 1463 ... 2917 2918 2919]\n"
     ]
    }
   ],
   "source": [
    "# Y_test_predicted_for_submission = RFR.predict(df_test)\n",
    "# Y_test_predicted_for_submission = RFR.predict(X_test_split)\n",
    "Y_test_predicted_for_submission = RFR.predict(test_sample_df)\n",
    "\n",
    "indexes = np.arange(df_test.shape[0]+2, 2*df_test.shape[0]+2)\n",
    "print('Indexex ', indexes)\n",
    "\n",
    "# output_for_submission = pd.DataFrame({'Id': test_sample_df.Id,\n",
    "#                        'SalePrice': Y_test_predicted_for_submission})\n",
    "\n",
    "# output_for_submission = pd.DataFrame({'Id': indexes,\n",
    "#                        'SalePrice': Y_test_predicted_for_submission})\n",
    "\n",
    "# output_for_submission = pd.DataFrame({'Id': df_test.Id,\n",
    "#                        'SalePrice': Y_test_predicted_for_submission})\n",
    "\n",
    "# print(output_for_submission)\n",
    "\n",
    "# *****************\n",
    "testID = df_test['Id']\n",
    "output_for_submission = pd.DataFrame()\n",
    "output_for_submission['ID'] = testID\n",
    "output_for_submission['SalePrice'] = Y_test_predicted_for_submission\n",
    "output_for_submission.to_csv('submission.csv', index=False)\n",
    "# The above gave me a rank of 3000+ in the Leaderboard which is 68% from Top and a score of 0.15\n",
    "# So thats not very encouraging, so lets get back to improving the model\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## More Feature Engineering\n",
    "\n",
    "After relatively large MSE from the above simple Linear Regression lets fine-tune a little bit more our feature engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSLE from custom-defined function 0.12255286983099915\n",
      "RMSLE using scikit-learn function 0.015019012287913868\n"
     ]
    }
   ],
   "source": [
    "all_features_except_target = [i for i in df_train.columns if i not in ['SalePrice'] ]\n",
    "features_X = df_train[all_features_except_target]\n",
    "y = df_train['SalePrice']\n",
    "\n",
    "\n",
    "X_feature_engineering = features_X.copy()\n",
    "\n",
    "# To measure how recently the House was re-modified - calculate a column by subtracting YearBuilt from YearRemodAdd.\n",
    "# The notes on the data says - YearRemodAdd: Remodel date (same as construction date if no remodeling or additions)\n",
    "X_feature_engineering['years_since_update'] = X_feature_engineering['YearRemodAdd'] - X_feature_engineering['YearBuilt']\n",
    "X_feature_engineering['garage_value'] = X_feature_engineering['YearBuilt'] * X_feature_engineering['GarageCars']\n",
    "\n",
    "X_feature_engineering = X_feature_engineering.drop(columns=['GarageCars'])\n",
    "\n",
    "feature_numerical_columns = [col_name for col_name in X_feature_engineering.columns if\n",
    "                X_feature_engineering[col_name].dtype in ['int64', 'float64']]\n",
    "\n",
    "feature_categorical_cols = [col_name for col_name in X_feature_engineering.columns if\n",
    "                    X_feature_engineering[col_name].nunique() < 50 and\n",
    "                    X_feature_engineering[col_name].dtype in ['object', 'bool']]\n",
    "\n",
    "\n",
    "feature_numerical_transformer = SimpleImputer(strategy='constant')\n",
    "\n",
    "feature_categorical_transformer = Pipeline(steps=[\n",
    "    ('imputer', SimpleImputer(strategy='most_frequent')),\n",
    "    ('onehot', OneHotEncoder(handle_unknown='ignore'))\n",
    "])\n",
    "\n",
    "feature_preprocessor = ColumnTransformer(\n",
    "    transformers=[\n",
    "        ('num', feature_numerical_transformer, feature_numerical_columns),\n",
    "        ('cat', feature_categorical_transformer, feature_categorical_cols)\n",
    "])\n",
    "\n",
    "feature_model = XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
    "             colsample_bynode=1, colsample_bytree=0.6, gamma=0.0, gpu_id=-1,\n",
    "             importance_type='gain', interaction_constraints='',\n",
    "             learning_rate=0.02, max_delta_step=0, max_depth=4,\n",
    "             min_child_weight=0.0, monotone_constraints='()',\n",
    "             n_estimators=1250, n_jobs=0, num_parallel_tree=1, random_state=0,\n",
    "             reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=0.8,\n",
    "             tree_method='exact', validate_parameters=1, verbosity=None)\n",
    "\n",
    "feature_classifier = Pipeline(steps=[('feature_preprocessor', feature_preprocessor),\n",
    "                      ('feature_model', feature_model)\n",
    "                     ])\n",
    "\n",
    "feature_X_train, feature_X_valid, feature_y_train, feature_y_test = train_test_split(X_feature_engineering, y, random_state=0)\n",
    "\n",
    "feature_classifier.fit(feature_X_train, feature_y_train, feature_model__verbose=False)\n",
    "feature_predictions = feature_classifier.predict(feature_X_valid)\n",
    "\n",
    "print('RMSLE from custom-defined function', root_mean_squared_log_error(feature_y_test, feature_predictions))\n",
    "\n",
    "print('RMSLE using scikit-learn function', mean_squared_log_error(feature_y_test, feature_predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "X_test_original = pd.read_csv('./input/house-prices-advanced-regression-techniques/test.csv')\n",
    "\n",
    "X_test_original['years_since_update'] = X_test_original['YearRemodAdd'] - X_feature_engineering['YearBuilt']\n",
    "X_test_original['garage_value'] = X_test_original['YearBuilt'] * X_test_original['GarageCars']\n",
    "\n",
    "X_test_original = X_test_original.drop(columns=['GarageCars'])\n",
    "\n",
    "feature_classifier.fit(X_feature_engineering, y, feature_model__verbose=False)\n",
    "\n",
    "model_predictions = feature_classifier.predict(X_test_original)\n",
    "output = pd.DataFrame({'Id': X_test_original.Id,\n",
    "                       'SalePrice': model_predictions})\n",
    "output.to_csv('submission.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "838px",
    "left": "404px",
    "right": "20px",
    "top": "141px",
    "width": "800px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}