Classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df1=pd.read_csv(\"bert_processed_titles.csv\")\n",
    "df1=df1.drop(columns=['Unnamed: 0'])\n",
    "df2=pd.read_csv(\"bert_processed_post.csv\")\n",
    "df2=df2.drop(columns=['Unnamed: 0'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories=pd.read_csv(\"Final data.csv\").Category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==============================\n",
      "LogisticRegression\n",
      "****Results****\n",
      "Accuracy: 75.2294%\n",
      "Log Loss: 0.6376905031261763\n",
      "==============================\n",
      "KNeighborsClassifier\n",
      "****Results****\n",
      "Accuracy: 64.0367%\n",
      "Log Loss: 5.507343608750054\n",
      "==============================\n",
      "SVC\n",
      "****Results****\n",
      "Accuracy: 49.1743%\n",
      "Log Loss: 0.7601729371195928\n",
      "==============================\n",
      "NuSVC\n",
      "****Results****\n",
      "Accuracy: 71.0092%\n",
      "Log Loss: 0.6826310526604662\n",
      "==============================\n",
      "DecisionTreeClassifier\n",
      "****Results****\n",
      "Accuracy: 58.3486%\n",
      "Log Loss: 14.387147502431716\n",
      "==============================\n",
      "RandomForestClassifier\n",
      "****Results****\n",
      "Accuracy: 67.3394%\n",
      "Log Loss: 0.7785145479134848\n",
      "==============================\n",
      "GaussianNB\n",
      "****Results****\n",
      "Accuracy: 64.9541%\n",
      "Log Loss: 7.456319760182133\n",
      "==============================\n",
      "LinearDiscriminantAnalysis\n",
      "****Results****\n",
      "Accuracy: 65.1376%\n",
      "Log Loss: 4.594095259520855\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\python38-32\\lib\\site-packages\\sklearn\\discriminant_analysis.py:691: UserWarning: Variables are collinear\n",
      "  warnings.warn(\"Variables are collinear\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==============================\n",
      "QuadraticDiscriminantAnalysis\n",
      "****Results****\n",
      "Accuracy: 30.4587%\n",
      "Log Loss: 24.018708722332384\n",
      "==============================\n"
     ]
    }
   ],
   "source": [
    "train_features, test_features, train_labels, test_labels = train_test_split(df1, categories,test_size=0.3,random_state=40)\n",
    "from sklearn.metrics import accuracy_score, log_loss\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC, LinearSVC, NuSVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\n",
    "\n",
    "classifiers = [\n",
    "    LogisticRegression(max_iter=2000),\n",
    "    KNeighborsClassifier(3),\n",
    "    SVC(kernel=\"rbf\", C=0.025, probability=True),\n",
    "    NuSVC(probability=True),\n",
    "    DecisionTreeClassifier(),\n",
    "    RandomForestClassifier(),\n",
    "    GaussianNB(),\n",
    "    LinearDiscriminantAnalysis(),\n",
    "    QuadraticDiscriminantAnalysis()]\n",
    "\n",
    "# Logging for Visual Comparison\n",
    "log_cols=[\"Classifier\", \"Accuracy\", \"Log Loss\"]\n",
    "log = pd.DataFrame(columns=log_cols)\n",
    "\n",
    "for clf in classifiers:\n",
    "    clf.fit(train_features, train_labels)\n",
    "    name = clf.__class__.__name__\n",
    "    \n",
    "    print(\"=\"*30)\n",
    "    print(name)\n",
    "    \n",
    "    print('****Results****')\n",
    "    train_predictions = clf.predict(test_features)\n",
    "    acc = accuracy_score(test_labels, train_predictions)\n",
    "    print(\"Accuracy: {:.4%}\".format(acc))\n",
    "    \n",
    "    train_predictions = clf.predict_proba(test_features)\n",
    "    ll = log_loss(test_labels, train_predictions)\n",
    "    print(\"Log Loss: {}\".format(ll))\n",
    "    \n",
    "    log_entry = pd.DataFrame([[name, acc*100, ll]], columns=log_cols)\n",
    "    log = log.append(log_entry)\n",
    "    \n",
    "print(\"=\"*30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==============================\n",
      "LogisticRegression\n",
      "****Results****\n",
      "Accuracy: 83.4862%\n",
      "Log Loss: 0.44251210659482404\n",
      "==============================\n",
      "KNeighborsClassifier\n",
      "****Results****\n",
      "Accuracy: 74.3119%\n",
      "Log Loss: 4.320485606165597\n",
      "==============================\n",
      "SVC\n",
      "****Results****\n",
      "Accuracy: 49.1743%\n",
      "Log Loss: 0.6100318067992297\n",
      "==============================\n",
      "NuSVC\n",
      "****Results****\n",
      "Accuracy: 77.9817%\n",
      "Log Loss: 0.5195934169323997\n",
      "==============================\n",
      "DecisionTreeClassifier\n",
      "****Results****\n",
      "Accuracy: 66.2385%\n",
      "Log Loss: 11.660797902134984\n",
      "==============================\n",
      "RandomForestClassifier\n",
      "****Results****\n",
      "Accuracy: 76.8807%\n",
      "Log Loss: 0.6553286763849236\n",
      "==============================\n",
      "GradientBoostingClassifier\n",
      "****Results****\n",
      "Accuracy: 79.6330%\n",
      "Log Loss: 0.5088714910654801\n",
      "==============================\n",
      "GaussianNB\n",
      "****Results****\n",
      "Accuracy: 74.8624%\n",
      "Log Loss: 6.571678710666426\n",
      "==============================\n",
      "LinearDiscriminantAnalysis\n",
      "****Results****\n",
      "Accuracy: 72.4771%\n",
      "Log Loss: 2.6573351422497073\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\python38-32\\lib\\site-packages\\sklearn\\discriminant_analysis.py:691: UserWarning: Variables are collinear\n",
      "  warnings.warn(\"Variables are collinear\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==============================\n",
      "QuadraticDiscriminantAnalysis\n",
      "****Results****\n",
      "Accuracy: 27.3394%\n",
      "Log Loss: 25.096065050247027\n",
      "==============================\n"
     ]
    }
   ],
   "source": [
    "train_features, test_features, train_labels, test_labels = train_test_split(df2, categories,test_size=0.3,random_state=40)\n",
    "from sklearn.metrics import accuracy_score, log_loss\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC, LinearSVC, NuSVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\n",
    "\n",
    "classifiers = [\n",
    "    LogisticRegression(max_iter=2000),\n",
    "    KNeighborsClassifier(3),\n",
    "    SVC(kernel=\"rbf\", C=0.025, probability=True),\n",
    "    NuSVC(probability=True),\n",
    "    DecisionTreeClassifier(),\n",
    "    RandomForestClassifier(),\n",
    "    GradientBoostingClassifier(),\n",
    "    GaussianNB(),\n",
    "    LinearDiscriminantAnalysis(),\n",
    "    QuadraticDiscriminantAnalysis()]\n",
    "\n",
    "# Logging for Visual Comparison\n",
    "log_cols=[\"Classifier\", \"Accuracy\", \"Log Loss\"]\n",
    "log = pd.DataFrame(columns=log_cols)\n",
    "\n",
    "for clf in classifiers:\n",
    "    clf.fit(train_features, train_labels)\n",
    "    name = clf.__class__.__name__\n",
    "    \n",
    "    print(\"=\"*30)\n",
    "    print(name)\n",
    "    \n",
    "    print('****Results****')\n",
    "    train_predictions = clf.predict(test_features)\n",
    "    acc = accuracy_score(test_labels, train_predictions)\n",
    "    print(\"Accuracy: {:.4%}\".format(acc))\n",
    "    \n",
    "    train_predictions = clf.predict_proba(test_features)\n",
    "    ll = log_loss(test_labels, train_predictions)\n",
    "    print(\"Log Loss: {}\".format(ll))\n",
    "    \n",
    "    log_entry = pd.DataFrame([[name, acc*100, ll]], columns=log_cols)\n",
    "    log = log.append(log_entry)\n",
    "    \n",
    "print(\"=\"*30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>1526</th>\n",
       "      <th>1527</th>\n",
       "      <th>1528</th>\n",
       "      <th>1529</th>\n",
       "      <th>1530</th>\n",
       "      <th>1531</th>\n",
       "      <th>1532</th>\n",
       "      <th>1533</th>\n",
       "      <th>1534</th>\n",
       "      <th>1535</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.323475</td>\n",
       "      <td>0.018886</td>\n",
       "      <td>-0.114901</td>\n",
       "      <td>-0.304749</td>\n",
       "      <td>-0.361601</td>\n",
       "      <td>-0.144184</td>\n",
       "      <td>0.323100</td>\n",
       "      <td>0.121138</td>\n",
       "      <td>0.055903</td>\n",
       "      <td>-1.002128</td>\n",
       "      <td>...</td>\n",
       "      <td>0.325367</td>\n",
       "      <td>0.221319</td>\n",
       "      <td>-0.083464</td>\n",
       "      <td>-0.038477</td>\n",
       "      <td>0.185880</td>\n",
       "      <td>0.206531</td>\n",
       "      <td>0.129274</td>\n",
       "      <td>-0.204942</td>\n",
       "      <td>0.066266</td>\n",
       "      <td>0.193529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.432648</td>\n",
       "      <td>0.054735</td>\n",
       "      <td>-0.225384</td>\n",
       "      <td>-0.349018</td>\n",
       "      <td>-0.244678</td>\n",
       "      <td>-0.092731</td>\n",
       "      <td>0.270218</td>\n",
       "      <td>0.020490</td>\n",
       "      <td>-0.001303</td>\n",
       "      <td>-0.998625</td>\n",
       "      <td>...</td>\n",
       "      <td>0.291863</td>\n",
       "      <td>0.247866</td>\n",
       "      <td>-0.103916</td>\n",
       "      <td>0.037328</td>\n",
       "      <td>0.151021</td>\n",
       "      <td>0.196845</td>\n",
       "      <td>0.106454</td>\n",
       "      <td>-0.243563</td>\n",
       "      <td>0.087740</td>\n",
       "      <td>0.062133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.384477</td>\n",
       "      <td>0.203552</td>\n",
       "      <td>-0.151845</td>\n",
       "      <td>-0.395367</td>\n",
       "      <td>-0.558783</td>\n",
       "      <td>-0.108828</td>\n",
       "      <td>0.128819</td>\n",
       "      <td>0.007210</td>\n",
       "      <td>0.199363</td>\n",
       "      <td>-1.020770</td>\n",
       "      <td>...</td>\n",
       "      <td>0.272460</td>\n",
       "      <td>0.272832</td>\n",
       "      <td>-0.081840</td>\n",
       "      <td>-0.030869</td>\n",
       "      <td>0.164903</td>\n",
       "      <td>0.005209</td>\n",
       "      <td>-0.013996</td>\n",
       "      <td>-0.237227</td>\n",
       "      <td>0.142631</td>\n",
       "      <td>0.179154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.224948</td>\n",
       "      <td>0.043741</td>\n",
       "      <td>0.049741</td>\n",
       "      <td>-0.461003</td>\n",
       "      <td>-0.422239</td>\n",
       "      <td>-0.065275</td>\n",
       "      <td>0.293739</td>\n",
       "      <td>0.000257</td>\n",
       "      <td>0.026831</td>\n",
       "      <td>-1.066734</td>\n",
       "      <td>...</td>\n",
       "      <td>0.284596</td>\n",
       "      <td>0.212406</td>\n",
       "      <td>-0.079739</td>\n",
       "      <td>0.006585</td>\n",
       "      <td>0.114842</td>\n",
       "      <td>0.072750</td>\n",
       "      <td>0.117432</td>\n",
       "      <td>-0.308233</td>\n",
       "      <td>0.089964</td>\n",
       "      <td>0.182412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.419161</td>\n",
       "      <td>0.068550</td>\n",
       "      <td>-0.023623</td>\n",
       "      <td>-0.467691</td>\n",
       "      <td>-0.446928</td>\n",
       "      <td>-0.070011</td>\n",
       "      <td>0.275403</td>\n",
       "      <td>-0.038472</td>\n",
       "      <td>0.104754</td>\n",
       "      <td>-1.049468</td>\n",
       "      <td>...</td>\n",
       "      <td>0.302620</td>\n",
       "      <td>0.264415</td>\n",
       "      <td>-0.168423</td>\n",
       "      <td>-0.046455</td>\n",
       "      <td>0.259645</td>\n",
       "      <td>-0.070813</td>\n",
       "      <td>0.002642</td>\n",
       "      <td>-0.226117</td>\n",
       "      <td>0.049908</td>\n",
       "      <td>0.233653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1810</th>\n",
       "      <td>0.328831</td>\n",
       "      <td>0.081033</td>\n",
       "      <td>-0.232647</td>\n",
       "      <td>-0.248958</td>\n",
       "      <td>-0.167015</td>\n",
       "      <td>-0.003451</td>\n",
       "      <td>0.349201</td>\n",
       "      <td>-0.144981</td>\n",
       "      <td>-0.029696</td>\n",
       "      <td>-1.016797</td>\n",
       "      <td>...</td>\n",
       "      <td>0.310365</td>\n",
       "      <td>0.177154</td>\n",
       "      <td>-0.148924</td>\n",
       "      <td>-0.058485</td>\n",
       "      <td>0.132223</td>\n",
       "      <td>0.316460</td>\n",
       "      <td>-0.081747</td>\n",
       "      <td>-0.029593</td>\n",
       "      <td>0.006634</td>\n",
       "      <td>0.045405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1811</th>\n",
       "      <td>0.332319</td>\n",
       "      <td>0.034011</td>\n",
       "      <td>-0.097480</td>\n",
       "      <td>-0.413252</td>\n",
       "      <td>-0.429220</td>\n",
       "      <td>-0.099983</td>\n",
       "      <td>0.262789</td>\n",
       "      <td>0.022917</td>\n",
       "      <td>0.109021</td>\n",
       "      <td>-1.091354</td>\n",
       "      <td>...</td>\n",
       "      <td>0.258830</td>\n",
       "      <td>0.269137</td>\n",
       "      <td>-0.174335</td>\n",
       "      <td>0.028305</td>\n",
       "      <td>0.096787</td>\n",
       "      <td>0.100269</td>\n",
       "      <td>0.100474</td>\n",
       "      <td>-0.240648</td>\n",
       "      <td>0.219197</td>\n",
       "      <td>0.160523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1812</th>\n",
       "      <td>0.235371</td>\n",
       "      <td>0.025480</td>\n",
       "      <td>-0.092196</td>\n",
       "      <td>-0.469083</td>\n",
       "      <td>-0.350185</td>\n",
       "      <td>-0.064664</td>\n",
       "      <td>0.385660</td>\n",
       "      <td>0.032605</td>\n",
       "      <td>0.108762</td>\n",
       "      <td>-1.024851</td>\n",
       "      <td>...</td>\n",
       "      <td>0.309784</td>\n",
       "      <td>0.287435</td>\n",
       "      <td>-0.096790</td>\n",
       "      <td>0.031133</td>\n",
       "      <td>0.050241</td>\n",
       "      <td>0.159451</td>\n",
       "      <td>0.132936</td>\n",
       "      <td>-0.134567</td>\n",
       "      <td>0.127866</td>\n",
       "      <td>0.171213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1813</th>\n",
       "      <td>0.284350</td>\n",
       "      <td>-0.029581</td>\n",
       "      <td>-0.191624</td>\n",
       "      <td>-0.310858</td>\n",
       "      <td>-0.369617</td>\n",
       "      <td>0.051746</td>\n",
       "      <td>0.361012</td>\n",
       "      <td>-0.076742</td>\n",
       "      <td>0.019964</td>\n",
       "      <td>-0.939237</td>\n",
       "      <td>...</td>\n",
       "      <td>0.313144</td>\n",
       "      <td>0.167466</td>\n",
       "      <td>-0.124767</td>\n",
       "      <td>-0.010094</td>\n",
       "      <td>0.225876</td>\n",
       "      <td>0.128111</td>\n",
       "      <td>0.108890</td>\n",
       "      <td>-0.268145</td>\n",
       "      <td>0.099276</td>\n",
       "      <td>0.097705</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1814</th>\n",
       "      <td>0.387905</td>\n",
       "      <td>0.018739</td>\n",
       "      <td>-0.104244</td>\n",
       "      <td>-0.318491</td>\n",
       "      <td>-0.188115</td>\n",
       "      <td>0.021120</td>\n",
       "      <td>0.384614</td>\n",
       "      <td>-0.155113</td>\n",
       "      <td>0.109627</td>\n",
       "      <td>-1.137548</td>\n",
       "      <td>...</td>\n",
       "      <td>0.245538</td>\n",
       "      <td>0.221405</td>\n",
       "      <td>-0.188702</td>\n",
       "      <td>0.079381</td>\n",
       "      <td>0.167522</td>\n",
       "      <td>0.039953</td>\n",
       "      <td>-0.027000</td>\n",
       "      <td>-0.095687</td>\n",
       "      <td>0.054833</td>\n",
       "      <td>0.071289</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1815 rows × 1536 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6     \\\n",
       "0     0.323475  0.018886 -0.114901 -0.304749 -0.361601 -0.144184  0.323100   \n",
       "1     0.432648  0.054735 -0.225384 -0.349018 -0.244678 -0.092731  0.270218   \n",
       "2     0.384477  0.203552 -0.151845 -0.395367 -0.558783 -0.108828  0.128819   \n",
       "3     0.224948  0.043741  0.049741 -0.461003 -0.422239 -0.065275  0.293739   \n",
       "4     0.419161  0.068550 -0.023623 -0.467691 -0.446928 -0.070011  0.275403   \n",
       "...        ...       ...       ...       ...       ...       ...       ...   \n",
       "1810  0.328831  0.081033 -0.232647 -0.248958 -0.167015 -0.003451  0.349201   \n",
       "1811  0.332319  0.034011 -0.097480 -0.413252 -0.429220 -0.099983  0.262789   \n",
       "1812  0.235371  0.025480 -0.092196 -0.469083 -0.350185 -0.064664  0.385660   \n",
       "1813  0.284350 -0.029581 -0.191624 -0.310858 -0.369617  0.051746  0.361012   \n",
       "1814  0.387905  0.018739 -0.104244 -0.318491 -0.188115  0.021120  0.384614   \n",
       "\n",
       "          7         8         9     ...      1526      1527      1528  \\\n",
       "0     0.121138  0.055903 -1.002128  ...  0.325367  0.221319 -0.083464   \n",
       "1     0.020490 -0.001303 -0.998625  ...  0.291863  0.247866 -0.103916   \n",
       "2     0.007210  0.199363 -1.020770  ...  0.272460  0.272832 -0.081840   \n",
       "3     0.000257  0.026831 -1.066734  ...  0.284596  0.212406 -0.079739   \n",
       "4    -0.038472  0.104754 -1.049468  ...  0.302620  0.264415 -0.168423   \n",
       "...        ...       ...       ...  ...       ...       ...       ...   \n",
       "1810 -0.144981 -0.029696 -1.016797  ...  0.310365  0.177154 -0.148924   \n",
       "1811  0.022917  0.109021 -1.091354  ...  0.258830  0.269137 -0.174335   \n",
       "1812  0.032605  0.108762 -1.024851  ...  0.309784  0.287435 -0.096790   \n",
       "1813 -0.076742  0.019964 -0.939237  ...  0.313144  0.167466 -0.124767   \n",
       "1814 -0.155113  0.109627 -1.137548  ...  0.245538  0.221405 -0.188702   \n",
       "\n",
       "          1529      1530      1531      1532      1533      1534      1535  \n",
       "0    -0.038477  0.185880  0.206531  0.129274 -0.204942  0.066266  0.193529  \n",
       "1     0.037328  0.151021  0.196845  0.106454 -0.243563  0.087740  0.062133  \n",
       "2    -0.030869  0.164903  0.005209 -0.013996 -0.237227  0.142631  0.179154  \n",
       "3     0.006585  0.114842  0.072750  0.117432 -0.308233  0.089964  0.182412  \n",
       "4    -0.046455  0.259645 -0.070813  0.002642 -0.226117  0.049908  0.233653  \n",
       "...        ...       ...       ...       ...       ...       ...       ...  \n",
       "1810 -0.058485  0.132223  0.316460 -0.081747 -0.029593  0.006634  0.045405  \n",
       "1811  0.028305  0.096787  0.100269  0.100474 -0.240648  0.219197  0.160523  \n",
       "1812  0.031133  0.050241  0.159451  0.132936 -0.134567  0.127866  0.171213  \n",
       "1813 -0.010094  0.225876  0.128111  0.108890 -0.268145  0.099276  0.097705  \n",
       "1814  0.079381  0.167522  0.039953 -0.027000 -0.095687  0.054833  0.071289  \n",
       "\n",
       "[1815 rows x 1536 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3=pd.concat([df1,df2],axis=1)\n",
    "df3.columns=range(1536)\n",
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==============================\n",
      "LogisticRegression\n",
      "****Results****\n",
      "Accuracy: 83.3028%\n",
      "Log Loss: 0.45036675010636196\n",
      "==============================\n",
      "KNeighborsClassifier\n",
      "****Results****\n",
      "Accuracy: 73.2110%\n",
      "Log Loss: 4.161249283692677\n"
     ]
    }
   ],
   "source": [
    "train_features, test_features, train_labels, test_labels = train_test_split(df3, categories,test_size=0.3,random_state=40)\n",
    "from sklearn.metrics import accuracy_score, log_loss\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC, LinearSVC, NuSVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\n",
    "\n",
    "classifiers = [\n",
    "    LogisticRegression(max_iter=2000),\n",
    "    KNeighborsClassifier(3),\n",
    "    SVC(kernel=\"rbf\", C=0.025, probability=True),\n",
    "    NuSVC(probability=True),\n",
    "    DecisionTreeClassifier(),\n",
    "    RandomForestClassifier(),\n",
    "    GradientBoostingClassifier(),\n",
    "    GaussianNB(),\n",
    "    LinearDiscriminantAnalysis(),\n",
    "    QuadraticDiscriminantAnalysis()]\n",
    "\n",
    "# Logging for Visual Comparison\n",
    "log_cols=[\"Classifier\", \"Accuracy\", \"Log Loss\"]\n",
    "log = pd.DataFrame(columns=log_cols)\n",
    "\n",
    "for clf in classifiers:\n",
    "    clf.fit(train_features, train_labels)\n",
    "    name = clf.__class__.__name__\n",
    "    \n",
    "    print(\"=\"*30)\n",
    "    print(name)\n",
    "    \n",
    "    print('****Results****')\n",
    "    train_predictions = clf.predict(test_features)\n",
    "    acc = accuracy_score(test_labels, train_predictions)\n",
    "    print(\"Accuracy: {:.4%}\".format(acc))\n",
    "    \n",
    "    train_predictions = clf.predict_proba(test_features)\n",
    "    ll = log_loss(test_labels, train_predictions)\n",
    "    print(\"Log Loss: {}\".format(ll))\n",
    "    \n",
    "    log_entry = pd.DataFrame([[name, acc*100, ll]], columns=log_cols)\n",
    "    log = log.append(log_entry)\n",
    "    \n",
    "print(\"=\"*30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifiers[0].predict(test_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}