ember2018-HyperparameterOpt- AHBO-TPE RF, KNN, SGD, LR, .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import ember\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "#import altair as alt\n",
    "import lightgbm as lgb\n",
    "import matplotlib.pylab as plt\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "#_ = alt.renderers.enable('notebook')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir =r\"C:\\Users\\fahad\\Ember\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 2.61 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "emberdf = ember.read_metadata(data_dir)\n",
    "X_train, y_train, X_test, y_test = ember.read_vectorized_features(data_dir)\n",
    "lgbm_model = lgb.Booster(model_file=os.path.join(data_dir, \"ember_model_2018.txt\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reduce_mem_usage(df, verbose=True):\n",
    "    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n",
    "    start_mem = df.memory_usage().sum() / 1024**2    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtypes\n",
    "        if col_type in numerics:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)    \n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mem. usage decreased to 39.10 Mb (14.6% reduction)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sha256</th>\n",
       "      <th>appeared</th>\n",
       "      <th>subset</th>\n",
       "      <th>label</th>\n",
       "      <th>avclass</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0abb4fda7d5b13801d63bee53e5e256be43e141faa077a...</td>\n",
       "      <td>2006-12</td>\n",
       "      <td>train</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c9cafff8a596ba8a80bafb4ba8ae6f2ef3329d95b85f15...</td>\n",
       "      <td>2007-01</td>\n",
       "      <td>train</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>eac8ddb4970f8af985742973d6f0e06902d42a3684d791...</td>\n",
       "      <td>2007-02</td>\n",
       "      <td>train</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7f513818bcc276c531af2e641c597744da807e21cc1160...</td>\n",
       "      <td>2007-02</td>\n",
       "      <td>train</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ca65e1c387a4cc9e7d8a8ce12bf1bcf9f534c9032b9d95...</td>\n",
       "      <td>2007-02</td>\n",
       "      <td>train</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999995</th>\n",
       "      <td>e033bc4967ce64bbb5cafdb234372099395185a6e0280c...</td>\n",
       "      <td>2018-12</td>\n",
       "      <td>test</td>\n",
       "      <td>1</td>\n",
       "      <td>zbot</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999996</th>\n",
       "      <td>c7d16736fd905f5fbe4530670b1fe787eb12ee86536380...</td>\n",
       "      <td>2018-12</td>\n",
       "      <td>test</td>\n",
       "      <td>1</td>\n",
       "      <td>flystudio</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999997</th>\n",
       "      <td>0020077cb673729209d88b603bddf56b925b18e682892a...</td>\n",
       "      <td>2018-12</td>\n",
       "      <td>test</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999998</th>\n",
       "      <td>1b7e7c8febabf70d1c17fe3c7abf80f33003581c380f28...</td>\n",
       "      <td>2018-12</td>\n",
       "      <td>test</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999999</th>\n",
       "      <td>836063f2312b597632bca1f738e68e4d23f672d587a7fc...</td>\n",
       "      <td>2018-12</td>\n",
       "      <td>test</td>\n",
       "      <td>1</td>\n",
       "      <td>emotet</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000000 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   sha256 appeared subset  \\\n",
       "0       0abb4fda7d5b13801d63bee53e5e256be43e141faa077a...  2006-12  train   \n",
       "1       c9cafff8a596ba8a80bafb4ba8ae6f2ef3329d95b85f15...  2007-01  train   \n",
       "2       eac8ddb4970f8af985742973d6f0e06902d42a3684d791...  2007-02  train   \n",
       "3       7f513818bcc276c531af2e641c597744da807e21cc1160...  2007-02  train   \n",
       "4       ca65e1c387a4cc9e7d8a8ce12bf1bcf9f534c9032b9d95...  2007-02  train   \n",
       "...                                                   ...      ...    ...   \n",
       "999995  e033bc4967ce64bbb5cafdb234372099395185a6e0280c...  2018-12   test   \n",
       "999996  c7d16736fd905f5fbe4530670b1fe787eb12ee86536380...  2018-12   test   \n",
       "999997  0020077cb673729209d88b603bddf56b925b18e682892a...  2018-12   test   \n",
       "999998  1b7e7c8febabf70d1c17fe3c7abf80f33003581c380f28...  2018-12   test   \n",
       "999999  836063f2312b597632bca1f738e68e4d23f672d587a7fc...  2018-12   test   \n",
       "\n",
       "        label    avclass  \n",
       "0           0        NaN  \n",
       "1           0        NaN  \n",
       "2           0        NaN  \n",
       "3           0        NaN  \n",
       "4           0        NaN  \n",
       "...       ...        ...  \n",
       "999995      1       zbot  \n",
       "999996      1  flystudio  \n",
       "999997      0        NaN  \n",
       "999998      0        NaN  \n",
       "999999      1     emotet  \n",
       "\n",
       "[1000000 rows x 5 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reduce_mem_usage(emberdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1000000 entries, 0 to 999999\n",
      "Data columns (total 5 columns):\n",
      " #   Column    Non-Null Count    Dtype \n",
      "---  ------    --------------    ----- \n",
      " 0   sha256    1000000 non-null  object\n",
      " 1   appeared  1000000 non-null  object\n",
      " 2   subset    1000000 non-null  object\n",
      " 3   label     1000000 non-null  int8  \n",
      " 4   avclass   485000 non-null   object\n",
      "dtypes: int8(1), object(4)\n",
      "memory usage: 39.1+ MB\n"
     ]
    }
   ],
   "source": [
    "emberdf.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [3:56:16<00:00, 1417.65s/trial, best loss: -0.67486373888022] \n",
      "Random Forest: Hyperopt estimated optimum {'criterion': 0, 'max_depth': 15.0, 'max_features': 0, 'min_samples_leaf': 10.0, 'min_samples_split': 5.0, 'n_estimators': 64.0}\n",
      "CPU times: user 3h 54min 9s, sys: 59.1 s, total: 3h 55min 8s\n",
      "Wall time: 3h 56min 23s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#Random Forest with AHBO-TPE \n",
    "from hyperopt import hp, fmin, tpe, STATUS_OK, Trials\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "# Define the objective function\n",
    "def objective(params):\n",
    "    params = {\n",
    "        'n_estimators': int(params['n_estimators']), \n",
    "        'max_depth': int(params['max_depth']),\n",
    "        'max_features': (params['max_features']),\n",
    "        \"min_samples_split\":int(params['min_samples_split']),\n",
    "        \"min_samples_leaf\":int(params['min_samples_leaf']),\n",
    "        \"criterion\":str(params['criterion'])\n",
    "    }\n",
    "    clf = RandomForestClassifier( **params)\n",
    "    score = cross_val_score(clf, X_train, y_train, scoring='accuracy', cv=3).mean()\n",
    "    #print(\"ROC-AUC {:.3f} params {}\".format(score, params))\n",
    "\n",
    "    return {'loss':-score, 'status': STATUS_OK }\n",
    "# Define the hyperparameter configuration space\n",
    "space = {\n",
    "    'n_estimators': hp.quniform('n_estimators', 10, 100, 1),\n",
    "    'max_depth': hp.quniform('max_depth', 2, 60, 1),\n",
    "    \"max_features\":hp.choice('max_features', ['auto', 'sqrt']),\n",
    "    \"min_samples_split\":hp.quniform('min_samples_split',2,10,1),\n",
    "    \"min_samples_leaf\":hp.quniform('min_samples_leaf',1,10,1),\n",
    "    \"criterion\":hp.choice('criterion',['gini'])\n",
    "}\n",
    "\n",
    "best = fmin(fn=objective,\n",
    "            space=space,\n",
    "            algo=tpe.suggest,\n",
    "            max_evals=10)\n",
    "print(\"Random Forest: Hyperopt estimated optimum {}\".format(best))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RF with AHBO-TPE Accuracy: 0.935595\n",
      "RF with AHBO-TPE ROC AUC score: 0.9358213734999999\n",
      "RF with AHBO-TPE Classification Report :               precision    recall  f1-score   support\n",
      "\n",
      "        -1.0       0.00      0.00      0.00        67\n",
      "         0.0       0.93      0.94      0.94     99824\n",
      "         1.0       0.94      0.94      0.94    100109\n",
      "\n",
      "    accuracy                           0.94    200000\n",
      "   macro avg       0.62      0.62      0.62    200000\n",
      "weighted avg       0.94      0.94      0.94    200000\n",
      "\n",
      "CPU times: user 29min 43s, sys: 7.64 s, total: 29min 51s\n",
      "Wall time: 31min 14s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/acp18fta/.conda/envs/my-python-env/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#1-4,Random Forest with Hyperparameters (original defautl accuracy is 89% after manual tweaking accuracy changed from 92.5-94.0%)\n",
    "#importing module (4~5% increase in model performance and training time was reduced by 26 mins)\n",
    "from sklearn.ensemble import RandomForestClassifier \n",
    "# Hyperparmeters \n",
    "params = {\n",
    "        'n_estimators': 100, # tweaked from 64.0 to 100.0 \n",
    "        'max_depth': 30, # tweaked from 15.0 to 30.0 \n",
    "        'max_features': 'auto', # same\n",
    "        \"min_samples_split\":10, # tweaked from 5.0 to 10.0\n",
    "        \"min_samples_leaf\": 30, # tweaked from 10.0 to 30.0\n",
    "        \"criterion\": 'gini' # same\n",
    "    }\n",
    "#making the instance\n",
    "model = RandomForestClassifier( **params)\n",
    "#learning\n",
    "model.fit(X_train,y_train)\n",
    "#Prediction\n",
    "prediction=model.predict(X_test)\n",
    "#importing the metrics module\n",
    "from sklearn import metrics\n",
    "#evaluation(Accuracy)\n",
    "print(\"RF with AHBO-TPE Accuracy:\",metrics.accuracy_score(prediction,y_test))\n",
    "#evaluation of AUC ROC SCORE \n",
    "print(\"RF with AHBO-TPE ROC AUC score:\",metrics.roc_auc_score(y_test,prediction))\n",
    "#evaluation(Classification_report)\n",
    "from sklearn.metrics import classification_report\n",
    "print(\"RF with AHBO-TPE Classification Report :\",metrics.classification_report(prediction, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [53:03:46<00:00, 38205.40s/trial, best loss: -0.5211737201390912]   \n",
      "KNN: Hyperopt estimated optimum {'n_neighbors': 15.0}\n",
      "Wall time: 2d 5h 3min 47s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#KNN with AHBO-TPE\n",
    "from hyperopt import hp, fmin, tpe, STATUS_OK, Trials\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "def objective(params):\n",
    "    params = {\n",
    "        'n_neighbors': abs(int(params['n_neighbors'])),\n",
    "    }\n",
    "    clf = KNeighborsClassifier( **params)\n",
    "    score = cross_val_score(clf, X_train, y_train, scoring='accuracy', cv=3).mean()\n",
    "\n",
    "    return {'loss':-score, 'status': STATUS_OK }\n",
    "\n",
    "space = {\n",
    "    'n_neighbors': hp.quniform('n_neighbors', 1,31, 1),\n",
    "}\n",
    "\n",
    "best = fmin(fn=objective,\n",
    "            space=space,\n",
    "            algo=tpe.suggest,\n",
    "            max_evals=5)\n",
    "print(\"KNN: Hyperopt estimated optimum {}\".format(best))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNN with AHBO-TPE Accuracy: 0.74861\n",
      "KNN with AHBO-TPE ROC AUC score: 0.77424499905\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNN with AHBO-TPE Classification Report :               precision    recall  f1-score   support\n",
      "\n",
      "        -1.0       0.00      0.00      0.00     13490\n",
      "         0.0       0.75      0.79      0.77     95349\n",
      "         1.0       0.74      0.82      0.78     91161\n",
      "\n",
      "    accuracy                           0.75    200000\n",
      "   macro avg       0.50      0.54      0.52    200000\n",
      "weighted avg       0.70      0.75      0.72    200000\n",
      "\n",
      "Wall time: 4h 37min 39s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#2-4,kNearestNeighbors (originally default accuracy 69%) \n",
    "#importing module (4-5% increase in model performance)\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "#making the instance\n",
    "params = {\n",
    "       'n_neighbors': 15\n",
    "    }\n",
    "model = KNeighborsClassifier(** params)\n",
    "#learning\n",
    "model.fit(X_train,y_train)\n",
    "#Prediction\n",
    "prediction=model.predict(X_test)\n",
    "#importing the metrics module\n",
    "from sklearn import metrics\n",
    "#evaluation(Accuracy)\n",
    "print(\"KNN with AHBO-TPE Accuracy:\",metrics.accuracy_score(prediction,y_test))\n",
    "#evaluation of AUC ROC SCORE \n",
    "print(\"KNN with AHBO-TPE ROC AUC score:\",metrics.roc_auc_score(y_test,prediction))\n",
    "#evaluation(Classification_report)\n",
    "from sklearn.metrics import classification_report\n",
    "print(\"KNN with AHBO-TPE Classification Report :\",metrics.classification_report(prediction, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/5 [00:00<?, ?trial/s, best loss=?]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:573: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "\n",
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:573: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "\n",
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:573: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|██        | 1/5 [30:50<2:03:20, 1850.19s/trial, best loss: -0.30424619597119573]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:573: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "\n",
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:573: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "\n",
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:573: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|████      | 2/5 [1:08:08<1:38:19, 1966.52s/trial, best loss: -0.3607611737635321]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:573: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "\n",
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:573: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "\n",
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:573: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      " 60%|██████    | 3/5 [1:52:04<1:12:15, 2167.53s/trial, best loss: -0.37640627318558023]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:573: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "\n",
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:573: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "\n",
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:573: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      " 80%|████████  | 4/5 [2:07:07<29:48, 1788.08s/trial, best loss: -0.37640627318558023]  "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:573: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "\n",
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:573: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "\n",
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:573: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [2:09:59<00:00, 1559.98s/trial, best loss: -0.3831375043840153] \n",
      "SGD: Hyperopt estimated optimum {'l1_ratio': 1, 'loss': 1, 'max_iter': 8.0}\n",
      "Wall time: 2h 9min 59s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#SGD Classifier with AHBO-TPE\n",
    "from scipy.stats import loguniform\n",
    "from sklearn.linear_model import SGDClassifier \n",
    "from hyperopt import hp, fmin, tpe, STATUS_OK, Trials\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "def objective(params):\n",
    "    params = {\n",
    "        'penalty':(params['penalty'])\n",
    "        'loss': (params['loss']),\n",
    "        'max_iter': abs(int(params['max_iter'])),\n",
    "        'l1_ratio': (params['l1_ratio'])\n",
    "    }\n",
    "    clf = SGDClassifier( **params)\n",
    "    score = cross_val_score(clf, X_train, y_train, scoring='accuracy', cv=3).mean()\n",
    "\n",
    "    return {'loss':-score, 'status': STATUS_OK }\n",
    "\n",
    "space = {\n",
    "    'penalty': hp.choice('penalty',['l2']),\n",
    "    'loss': hp.choice('loss', ['hinge', 'squared_hinge']),\n",
    "    'max_iter': hp.quniform('max_iter',1,200,1),\n",
    "    'l1_ratio': hp.choice('l1_ratio',(0, 1)),\n",
    "}\n",
    "       \n",
    "                       \n",
    "best = fmin(fn=objective,\n",
    "            space=space,\n",
    "            algo=tpe.suggest,\n",
    "            max_evals=5)\n",
    "print(\"SGD: Hyperopt estimated optimum {}\".format(best))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:573: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SGD with AHBO-TPE  Accuracy: 0.58661\n",
      "SGD with AHBO-TPE   ROC AUC score: 0.5968822598000001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\fahad\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SGD with AHBO-TPE  Classification Report :               precision    recall  f1-score   support\n",
      "\n",
      "        -1.0       0.00      0.00      0.00      1510\n",
      "         0.0       0.96      0.55      0.70    174856\n",
      "         1.0       0.21      0.89      0.34     23634\n",
      "\n",
      "    accuracy                           0.59    200000\n",
      "   macro avg       0.39      0.48      0.35    200000\n",
      "weighted avg       0.87      0.59      0.65    200000\n",
      "\n",
      "Wall time: 1min 39s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#3-4,SGS Classifier with Hyperparameters(originally default accuracy 56%)\n",
    "#importing module (~4% increase in model performance)\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "#making the instance\n",
    "params = {\n",
    "       'penalty':'l2', # same penalty\n",
    "       'loss': 'hinge', # squared hinge results were fast but accuracy went down , changed from squared hinge \n",
    "       'max_iter': 10 # tweaked from 8 to 10\n",
    "    }\n",
    "model = SGDClassifier( **params)\n",
    "#learning\n",
    "model.fit(X_train,y_train)\n",
    "#Prediction\n",
    "prediction=model.predict(X_test)\n",
    "#importing the metrics module\n",
    "from sklearn import metrics\n",
    "#evaluation(Accuracy)\n",
    "print(\"SGD with AHBO-TPE  Accuracy:\",metrics.accuracy_score(prediction,y_test))\n",
    "#evaluation of AUC ROC SCORE \n",
    "print(\"SGD with AHBO-TPE   ROC AUC score:\",metrics.roc_auc_score(y_test,prediction))\n",
    "#evaluation(Classification_report)\n",
    "from sklearn.metrics import classification_report\n",
    "print(\"SGD with AHBO-TPE  Classification Report :\",metrics.classification_report(prediction, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/5 [00:00<?, ?trial/s, best loss=?]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/acp18fta/.conda/envs/my-python-env/lib/python3.6/site-packages/sklearn/linear_model/_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "\n",
      "/home/acp18fta/.conda/envs/my-python-env/lib/python3.6/site-packages/sklearn/linear_model/_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "\n",
      "/home/acp18fta/.conda/envs/my-python-env/lib/python3.6/site-packages/sklearn/linear_model/_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|██        | 1/5 [38:33<2:34:15, 2313.89s/trial, best loss: -0.38019006213876577]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/acp18fta/.conda/envs/my-python-env/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "\n",
      "/home/acp18fta/.conda/envs/my-python-env/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "\n",
      "/home/acp18fta/.conda/envs/my-python-env/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|████      | 2/5 [40:59<1:23:10, 1663.42s/trial, best loss: -0.3808050273527867] "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/acp18fta/.conda/envs/my-python-env/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "\n",
      "/home/acp18fta/.conda/envs/my-python-env/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "\n",
      "/home/acp18fta/.conda/envs/my-python-env/lib/python3.6/site-packages/sklearn/linear_model/_logistic.py:764: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      " 60%|██████    | 3/5 [44:21<40:49, 1224.95s/trial, best loss: -0.3808050273527867]  "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/acp18fta/.conda/envs/my-python-env/lib/python3.6/site-packages/sklearn/svm/_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "\n",
      "/home/acp18fta/.conda/envs/my-python-env/lib/python3.6/site-packages/sklearn/svm/_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "\n",
      "/home/acp18fta/.conda/envs/my-python-env/lib/python3.6/site-packages/sklearn/svm/_base.py:977: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  \"the number of iterations.\", ConvergenceWarning)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      " 80%|████████  | 4/5 [47:19<15:10, 910.90s/trial, best loss: -0.3808050273527867] "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/acp18fta/.conda/envs/my-python-env/lib/python3.6/site-packages/sklearn/linear_model/_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "\n",
      "/home/acp18fta/.conda/envs/my-python-env/lib/python3.6/site-packages/sklearn/linear_model/_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "\n",
      "/home/acp18fta/.conda/envs/my-python-env/lib/python3.6/site-packages/sklearn/linear_model/_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
      "  \"the coef_ did not converge\", ConvergenceWarning)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [1:23:15<00:00, 999.19s/trial, best loss: -0.3808050273527867] \n",
      "LR: Hyperopt estimated optimum {'C': 8.0, 'max_iter': 32.0, 'solver': 1}\n",
      "CPU times: user 3h 6min 44s, sys: 19min 14s, total: 3h 25min 59s\n",
      "Wall time: 1h 23min 17s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#Logistic Regression Classifier with AHBO-TPE\n",
    "from scipy.stats import loguniform\n",
    "from sklearn.linear_model import LogisticRegression \n",
    "from hyperopt import hp, fmin, tpe, STATUS_OK, Trials\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "def objective(params):\n",
    "    params = {\n",
    "         'max_iter': abs(int(params['max_iter'])),\n",
    "         'C': abs(float(params['C'])),\n",
    "          'solver': str(params['solver']) \n",
    "    }\n",
    "    clf = LogisticRegression( **params)\n",
    "    score = cross_val_score(clf, X_train, y_train, scoring='accuracy', cv=3).mean()\n",
    "\n",
    "    return {'loss':-score, 'status': STATUS_OK }\n",
    "\n",
    "space = {\n",
    "    'max_iter': hp.quniform('max_iter', 1,200,1),\n",
    "     'C': hp.quniform('C',1.0,20.0,1.0),\n",
    "      'solver': hp.choice('solver',['liblinear','lbfgs', 'sag', 'saga']) #  Saga supposed to be faster with large dataset\n",
    "}                      \n",
    "\n",
    "best = fmin(fn=objective,\n",
    "            space=space,\n",
    "            algo=tpe.suggest,\n",
    "            max_evals=5)\n",
    "print(\"LR: Hyperopt estimated optimum {}\".format(best))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR with AHBO-TPE Accuracy: 0.615865\n",
      "LR with AHBO-TPE ROC AUC score: 0.6185117450000001\n",
      "LR with AHBO-TPE Classification Report :               precision    recall  f1-score   support\n",
      "\n",
      "        -1.0       0.00      0.00      0.00       528\n",
      "         0.0       0.33      0.77      0.46     42628\n",
      "         1.0       0.90      0.58      0.70    156844\n",
      "\n",
      "    accuracy                           0.62    200000\n",
      "   macro avg       0.41      0.45      0.39    200000\n",
      "weighted avg       0.78      0.62      0.65    200000\n",
      "\n",
      "CPU times: user 1h 5min 39s, sys: 3.9 s, total: 1h 5min 43s\n",
      "Wall time: 1h 5min 37s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/acp18fta/.conda/envs/my-python-env/lib/python3.6/site-packages/sklearn/metrics/_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#4-4,Logistic Regression Classifier with Hyperparameters (originally default accuracy 59%)\n",
    "#importing module (3% increase in model performance and training time was reduced by approximate 38 mins)\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "#making the instance\n",
    "\n",
    "params = {\n",
    "       'max_iter': 200, # the more iteration you supply the more it can increase the accuracy.\n",
    "       'C': 8,\n",
    "       'solver': 'sag' # changed from lbfgs to check how fast can this solver converges \n",
    "    }\n",
    "model = LogisticRegression( **params)\n",
    "#learning\n",
    "model.fit(X_train,y_train)\n",
    "#Prediction\n",
    "prediction=model.predict(X_test)\n",
    "#importing the metrics module\n",
    "from sklearn import metrics\n",
    "#evaluation(Accuracy)\n",
    "print(\"LR with AHBO-TPE Accuracy:\",metrics.accuracy_score(prediction,y_test))\n",
    "#evaluation of AUC ROC SCORE \n",
    "print(\"LR with AHBO-TPE ROC AUC score:\",metrics.roc_auc_score(y_test,prediction))\n",
    "#evaluation(Classification_report)\n",
    "from sklearn.metrics import classification_report\n",
    "print(\"LR with AHBO-TPE Classification Report :\",metrics.classification_report(prediction, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}