Hyperparameter_Tuning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "language": "python",
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.7.9",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "colab": {
      "name": "Hyperparameter-Tuning.ipynb",
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "JkopauX8tied",
        "trusted": true
      },
      "source": [
        "from numpy import array\n",
        "from tensorflow.keras.preprocessing.text import one_hot\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Activation, Dropout, Dense\n",
        "from tensorflow.keras.layers import Flatten, LSTM\n",
        "from tensorflow.keras.layers import GlobalMaxPooling1D\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.layers import Embedding\n",
        "from sklearn.model_selection import train_test_split\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.layers import Input\n",
        "# from tensorflow.keras.layers.merge import Concatenate\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import re\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nm03N-iFtieu",
        "trusted": true
      },
      "source": [
        "toxic_comments = pd.read_csv(\"../input/comment/1613373921-5e748a2d5fc288e9f69c5f86.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CJKPrKbktie7",
        "trusted": true,
        "outputId": "52e28f0e-76dd-420e-aa3c-b1bf1338254d"
      },
      "source": [
        "toxic_comments.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                 id                                       comment_text  toxic  \\\n",
              "0  0000997932d777bf  Explanation\\nWhy the edits made under my usern...      0   \n",
              "1  000103f0d9cfb60f  D'aww! He matches this background colour I'm s...      0   \n",
              "2  000113f07ec002fd  Hey man, I'm really not trying to edit war. It...      0   \n",
              "3  0001b41b1c6bb37e  \"\\nMore\\nI can't make any real suggestions on ...      0   \n",
              "4  0001d958c54c6e35  You, sir, are my hero. Any chance you remember...      0   \n",
              "\n",
              "   severe_toxic  obscene  threat  insult  identity_hate  \n",
              "0             0        0       0       0              0  \n",
              "1             0        0       0       0              0  \n",
              "2             0        0       0       0              0  \n",
              "3             0        0       0       0              0  \n",
              "4             0        0       0       0              0  "
            ],
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>comment_text</th>\n",
              "      <th>toxic</th>\n",
              "      <th>severe_toxic</th>\n",
              "      <th>obscene</th>\n",
              "      <th>threat</th>\n",
              "      <th>insult</th>\n",
              "      <th>identity_hate</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0000997932d777bf</td>\n",
              "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>000103f0d9cfb60f</td>\n",
              "      <td>D'aww! He matches this background colour I'm s...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>000113f07ec002fd</td>\n",
              "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0001b41b1c6bb37e</td>\n",
              "      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0001d958c54c6e35</td>\n",
              "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J35_wYyRtie9",
        "trusted": true,
        "outputId": "4d0e38db-775e-4e10-de11-19eb9d1a1a0a"
      },
      "source": [
        "print(toxic_comments.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(159571, 8)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ntPuM7_etie9",
        "trusted": true
      },
      "source": [
        "filter = toxic_comments[\"comment_text\"] != \"\"\n",
        "toxic_comments = toxic_comments[filter]\n",
        "toxic_comments = toxic_comments.dropna()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_VjZ4F9dtie-",
        "trusted": true
      },
      "source": [
        "def preprocess_text(sen):\n",
        "    # Remove punctuations and numbers\n",
        "    sentence = re.sub('[^a-zA-Z]', ' ', sen)\n",
        "\n",
        "    # Single character removal\n",
        "    sentence = re.sub(r\"\\s+[a-zA-Z]\\s+\", ' ', sentence)\n",
        "\n",
        "    # Removing multiple spaces\n",
        "    sentence = re.sub(r'\\s+', ' ', sentence)\n",
        "\n",
        "    return sentence"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z6DMrt14tie_",
        "trusted": true
      },
      "source": [
        "X = []\n",
        "sentences = list(toxic_comments[\"comment_text\"])\n",
        "for sen in sentences:\n",
        "    X.append(preprocess_text(sen))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xCinPfXYtie_",
        "trusted": true
      },
      "source": [
        "toxic = np.sum(toxic_comments[[\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\", \"identity_hate\"]],axis = 1) > 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rJLxiCnDtifA",
        "trusted": true
      },
      "source": [
        "from nltk.corpus import stopwords"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_2StET_ktifB",
        "trusted": true
      },
      "source": [
        "my_stopwords = stopwords.words('english')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f4v58IqMtifB",
        "trusted": true
      },
      "source": [
        "from numpy import array\n",
        "from numpy import asarray\n",
        "from numpy import zeros\n",
        "\n",
        "embeddings_dictionary = dict()\n",
        "\n",
        "glove_file = open('../input/glove-global-vectors-for-word-representation/glove.6B.100d.txt', encoding=\"utf8\")\n",
        "\n",
        "for line in glove_file:\n",
        "    records = line.split()\n",
        "    word = records[0]\n",
        "    vector_dimensions = asarray(records[1:], dtype='float32')\n",
        "    embeddings_dictionary[word] = vector_dimensions\n",
        "glove_file.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SyeMCccftifC"
      },
      "source": [
        "### Vectorize the comments"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w3ugja8otifC",
        "trusted": true
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "vectorizer = TfidfVectorizer(stop_words = my_stopwords)\n",
        "bag_of_words = vectorizer.fit_transform(X)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q53Kk3bltifD",
        "trusted": true,
        "outputId": "44ef6bef-0d12-4f03-dd2e-7b3c95bc9bbf"
      },
      "source": [
        "print(\"Sparsity of bag_of_words:\",\n",
        "      len(bag_of_words.data)/(bag_of_words.shape[0]*bag_of_words.shape[1])*100)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sparsity of bag_of_words: 0.016165423205353118\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cAI3zNpBtifE",
        "trusted": true,
        "outputId": "2d366f2a-c51e-4a14-cc5b-22f4bfa2bddf"
      },
      "source": [
        "bag_of_words"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<159571x168645 sparse matrix of type '<class 'numpy.float64'>'\n",
              "\twith 4350253 stored elements in Compressed Sparse Row format>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Antn34zmtifE",
        "trusted": true,
        "outputId": "7ad7a5ae-900c-48cf-a1c6-e018add9923c"
      },
      "source": [
        "list(vectorizer.vocabulary_.keys())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['explanation',\n",
              " 'edits',\n",
              " 'made',\n",
              " 'username',\n",
              " 'hardcore',\n",
              " 'metallica',\n",
              " 'fan',\n",
              " 'reverted',\n",
              " 'vandalisms',\n",
              " 'closure',\n",
              " 'gas',\n",
              " 'voted',\n",
              " 'new',\n",
              " 'york',\n",
              " 'dolls',\n",
              " 'fac',\n",
              " 'please',\n",
              " 'remove',\n",
              " 'template',\n",
              " 'talk',\n",
              " 'page',\n",
              " 'since',\n",
              " 'retired',\n",
              " 'aww',\n",
              " 'matches',\n",
              " 'background',\n",
              " 'colour',\n",
              " 'seemingly',\n",
              " 'stuck',\n",
              " 'thanks',\n",
              " 'january',\n",
              " 'utc',\n",
              " 'hey',\n",
              " 'man',\n",
              " 'really',\n",
              " 'trying',\n",
              " 'edit',\n",
              " 'war',\n",
              " 'guy',\n",
              " 'constantly',\n",
              " 'removing',\n",
              " 'relevant',\n",
              " 'information',\n",
              " 'talking',\n",
              " 'instead',\n",
              " 'seems',\n",
              " 'care',\n",
              " 'formatting',\n",
              " 'actual',\n",
              " 'info',\n",
              " 'make',\n",
              " 'real',\n",
              " 'suggestions',\n",
              " 'improvement',\n",
              " 'wondered',\n",
              " 'section',\n",
              " 'statistics',\n",
              " 'later',\n",
              " 'subsection',\n",
              " 'types',\n",
              " 'accidents',\n",
              " 'think',\n",
              " 'references',\n",
              " 'may',\n",
              " 'need',\n",
              " 'tidying',\n",
              " 'exact',\n",
              " 'format',\n",
              " 'ie',\n",
              " 'date',\n",
              " 'etc',\n",
              " 'one',\n",
              " 'else',\n",
              " 'first',\n",
              " 'preferences',\n",
              " 'style',\n",
              " 'want',\n",
              " 'let',\n",
              " 'know',\n",
              " 'appears',\n",
              " 'backlog',\n",
              " 'articles',\n",
              " 'review',\n",
              " 'guess',\n",
              " 'delay',\n",
              " 'reviewer',\n",
              " 'turns',\n",
              " 'listed',\n",
              " 'form',\n",
              " 'eg',\n",
              " 'wikipedia',\n",
              " 'good',\n",
              " 'article',\n",
              " 'nominations',\n",
              " 'transport',\n",
              " 'sir',\n",
              " 'hero',\n",
              " 'chance',\n",
              " 'remember',\n",
              " 'congratulations',\n",
              " 'well',\n",
              " 'use',\n",
              " 'tools',\n",
              " 'cocksucker',\n",
              " 'piss',\n",
              " 'around',\n",
              " 'work',\n",
              " 'vandalism',\n",
              " 'matt',\n",
              " 'shirvington',\n",
              " 'banned',\n",
              " 'sorry',\n",
              " 'word',\n",
              " 'nonsense',\n",
              " 'offensive',\n",
              " 'anyway',\n",
              " 'intending',\n",
              " 'write',\n",
              " 'anything',\n",
              " 'wow',\n",
              " 'would',\n",
              " 'jump',\n",
              " 'merely',\n",
              " 'requesting',\n",
              " 'encyclopedic',\n",
              " 'school',\n",
              " 'reference',\n",
              " 'selective',\n",
              " 'breeding',\n",
              " 'almost',\n",
              " 'stub',\n",
              " 'points',\n",
              " 'animal',\n",
              " 'short',\n",
              " 'messy',\n",
              " 'gives',\n",
              " 'must',\n",
              " 'someone',\n",
              " 'expertise',\n",
              " 'eugenics',\n",
              " 'alignment',\n",
              " 'subject',\n",
              " 'contrary',\n",
              " 'dulithgow',\n",
              " 'fair',\n",
              " 'rationale',\n",
              " 'image',\n",
              " 'wonju',\n",
              " 'jpg',\n",
              " 'uploading',\n",
              " 'notice',\n",
              " 'specifies',\n",
              " 'used',\n",
              " 'constitutes',\n",
              " 'addition',\n",
              " 'boilerplate',\n",
              " 'also',\n",
              " 'description',\n",
              " 'specific',\n",
              " 'using',\n",
              " 'consistent',\n",
              " 'go',\n",
              " 'include',\n",
              " 'uploaded',\n",
              " 'media',\n",
              " 'consider',\n",
              " 'checking',\n",
              " 'specified',\n",
              " 'pages',\n",
              " 'find',\n",
              " 'list',\n",
              " 'edited',\n",
              " 'clicking',\n",
              " 'contributions',\n",
              " 'link',\n",
              " 'located',\n",
              " 'top',\n",
              " 'logged',\n",
              " 'selecting',\n",
              " 'dropdown',\n",
              " 'box',\n",
              " 'note',\n",
              " 'images',\n",
              " 'lacking',\n",
              " 'deleted',\n",
              " 'week',\n",
              " 'described',\n",
              " 'criteria',\n",
              " 'speedy',\n",
              " 'deletion',\n",
              " 'questions',\n",
              " 'ask',\n",
              " 'copyright',\n",
              " 'thank',\n",
              " 'contribs',\n",
              " 'unspecified',\n",
              " 'source',\n",
              " 'noticed',\n",
              " 'file',\n",
              " 'currently',\n",
              " 'specify',\n",
              " 'created',\n",
              " 'content',\n",
              " 'status',\n",
              " 'unclear',\n",
              " 'create',\n",
              " 'owner',\n",
              " 'obtained',\n",
              " 'website',\n",
              " 'taken',\n",
              " 'together',\n",
              " 'restatement',\n",
              " 'terms',\n",
              " 'usually',\n",
              " 'sufficient',\n",
              " 'however',\n",
              " 'holder',\n",
              " 'different',\n",
              " 'publisher',\n",
              " 'acknowledged',\n",
              " 'adding',\n",
              " 'add',\n",
              " 'proper',\n",
              " 'licensing',\n",
              " 'tag',\n",
              " 'already',\n",
              " 'took',\n",
              " 'picture',\n",
              " 'audio',\n",
              " 'video',\n",
              " 'release',\n",
              " 'gfdl',\n",
              " 'believe',\n",
              " 'meets',\n",
              " 'tags',\n",
              " 'see',\n",
              " 'full',\n",
              " 'files',\n",
              " 'tagged',\n",
              " 'following',\n",
              " 'unsourced',\n",
              " 'untagged',\n",
              " 'copyrighted',\n",
              " 'non',\n",
              " 'free',\n",
              " 'license',\n",
              " 'per',\n",
              " 'hours',\n",
              " 'bbq',\n",
              " 'lets',\n",
              " 'discuss',\n",
              " 'maybe',\n",
              " 'phone',\n",
              " 'exclusive',\n",
              " 'group',\n",
              " 'wp',\n",
              " 'talibans',\n",
              " 'destroying',\n",
              " 'self',\n",
              " 'appointed',\n",
              " 'purist',\n",
              " 'gang',\n",
              " 'asks',\n",
              " 'abt',\n",
              " 'anti',\n",
              " 'social',\n",
              " 'destructive',\n",
              " 'contribution',\n",
              " 'sityush',\n",
              " 'clean',\n",
              " 'behavior',\n",
              " 'issue',\n",
              " 'nonsensical',\n",
              " 'warnings',\n",
              " 'start',\n",
              " 'throwing',\n",
              " 'accusations',\n",
              " 'making',\n",
              " 'ad',\n",
              " 'hominem',\n",
              " 'attacks',\n",
              " 'going',\n",
              " 'strengthen',\n",
              " 'argument',\n",
              " 'look',\n",
              " 'like',\n",
              " 'abusing',\n",
              " 'power',\n",
              " 'admin',\n",
              " 'probably',\n",
              " 'single',\n",
              " 'talked',\n",
              " 'event',\n",
              " 'int',\n",
              " 'news',\n",
              " 'late',\n",
              " 'absence',\n",
              " 'notable',\n",
              " 'living',\n",
              " 'ex',\n",
              " 'president',\n",
              " 'attend',\n",
              " 'certainly',\n",
              " 'dedicating',\n",
              " 'aircracft',\n",
              " 'carrier',\n",
              " 'intend',\n",
              " 'revert',\n",
              " 'hopes',\n",
              " 'attracting',\n",
              " 'attention',\n",
              " 'willing',\n",
              " 'throw',\n",
              " 'quite',\n",
              " 'liberally',\n",
              " 'perhaps',\n",
              " 'achieve',\n",
              " 'level',\n",
              " 'civility',\n",
              " 'rational',\n",
              " 'discussion',\n",
              " 'topic',\n",
              " 'resolve',\n",
              " 'matter',\n",
              " 'peacefully',\n",
              " 'oh',\n",
              " 'girl',\n",
              " 'started',\n",
              " 'arguments',\n",
              " 'nose',\n",
              " 'belong',\n",
              " 'yvesnimmo',\n",
              " 'said',\n",
              " 'situation',\n",
              " 'settled',\n",
              " 'apologized',\n",
              " 'juelz',\n",
              " 'santanas',\n",
              " 'age',\n",
              " 'santana',\n",
              " 'years',\n",
              " 'old',\n",
              " 'came',\n",
              " 'february',\n",
              " 'th',\n",
              " 'makes',\n",
              " 'turn',\n",
              " 'songs',\n",
              " 'diplomats',\n",
              " 'third',\n",
              " 'neff',\n",
              " 'signed',\n",
              " 'cam',\n",
              " 'label',\n",
              " 'roc',\n",
              " 'fella',\n",
              " 'coming',\n",
              " 'singles',\n",
              " 'town',\n",
              " 'yes',\n",
              " 'born',\n",
              " 'could',\n",
              " 'older',\n",
              " 'lloyd',\n",
              " 'banks',\n",
              " 'birthday',\n",
              " 'passed',\n",
              " 'homie',\n",
              " 'death',\n",
              " 'god',\n",
              " 'forbid',\n",
              " 'thinking',\n",
              " 'equals',\n",
              " 'caculator',\n",
              " 'stop',\n",
              " 'changing',\n",
              " 'year',\n",
              " 'birth',\n",
              " 'bye',\n",
              " 'come',\n",
              " 'comming',\n",
              " 'back',\n",
              " 'tosser',\n",
              " 'redirect',\n",
              " 'voydan',\n",
              " 'pop',\n",
              " 'georgiev',\n",
              " 'chernodrinski',\n",
              " 'mitsurugi',\n",
              " 'point',\n",
              " 'sense',\n",
              " 'argue',\n",
              " 'hindi',\n",
              " 'ryo',\n",
              " 'sakazaki',\n",
              " 'mean',\n",
              " 'bother',\n",
              " 'writing',\n",
              " 'something',\n",
              " 'regarding',\n",
              " 'posted',\n",
              " 'acctually',\n",
              " 'even',\n",
              " 'better',\n",
              " 'take',\n",
              " 'closer',\n",
              " 'premature',\n",
              " 'wrestling',\n",
              " 'deaths',\n",
              " 'catagory',\n",
              " 'men',\n",
              " 'surely',\n",
              " 'besides',\n",
              " 'delting',\n",
              " 'recent',\n",
              " 'read',\n",
              " 'filmplot',\n",
              " 'editing',\n",
              " 'film',\n",
              " 'simply',\n",
              " 'entirely',\n",
              " 'many',\n",
              " 'unnecessary',\n",
              " 'details',\n",
              " 'bad',\n",
              " 'damage',\n",
              " 'yeah',\n",
              " 'studying',\n",
              " 'deepu',\n",
              " 'snowflakes',\n",
              " 'always',\n",
              " 'symmetrical',\n",
              " 'geometry',\n",
              " 'stated',\n",
              " 'snowflake',\n",
              " 'six',\n",
              " 'symmetric',\n",
              " 'arms',\n",
              " 'assertion',\n",
              " 'true',\n",
              " 'according',\n",
              " 'kenneth',\n",
              " 'libbrecht',\n",
              " 'rather',\n",
              " 'unattractive',\n",
              " 'irregular',\n",
              " 'crystals',\n",
              " 'far',\n",
              " 'common',\n",
              " 'variety',\n",
              " 'http',\n",
              " 'www',\n",
              " 'caltech',\n",
              " 'edu',\n",
              " 'atomic',\n",
              " 'snowcrystals',\n",
              " 'myths',\n",
              " 'htm',\n",
              " 'perfection',\n",
              " 'site',\n",
              " 'get',\n",
              " 'facts',\n",
              " 'still',\n",
              " 'decent',\n",
              " 'number',\n",
              " 'falsities',\n",
              " 'forgive',\n",
              " 'im',\n",
              " 'dont',\n",
              " 'signpost',\n",
              " 'september',\n",
              " 'unsubscribe',\n",
              " 'considering',\n",
              " 'st',\n",
              " 'paragraph',\n",
              " 'understand',\n",
              " 'reasons',\n",
              " 'sure',\n",
              " 'data',\n",
              " 'necessarily',\n",
              " 'wrong',\n",
              " 'persuaded',\n",
              " 'strategy',\n",
              " 'introducing',\n",
              " 'academic',\n",
              " 'honors',\n",
              " 'unhelpful',\n",
              " 'approach',\n",
              " 'sitting',\n",
              " 'justices',\n",
              " 'similarly',\n",
              " 'enhanced',\n",
              " 'changes',\n",
              " 'support',\n",
              " 'view',\n",
              " 'invite',\n",
              " 'anyone',\n",
              " 'visit',\n",
              " 'written',\n",
              " 'pairs',\n",
              " 'jurists',\n",
              " 'benjamin',\n",
              " 'cardozo',\n",
              " 'learned',\n",
              " 'hand',\n",
              " 'john',\n",
              " 'marshall',\n",
              " 'harlan',\n",
              " 'ii',\n",
              " 'question',\n",
              " 'becomes',\n",
              " 'current',\n",
              " 'version',\n",
              " 'either',\n",
              " 'pair',\n",
              " 'improved',\n",
              " 'credentials',\n",
              " 'introductory',\n",
              " 'helps',\n",
              " 'repeat',\n",
              " 'wry',\n",
              " 'kathleen',\n",
              " 'sullivan',\n",
              " 'stanford',\n",
              " 'law',\n",
              " 'suggests',\n",
              " 'harvard',\n",
              " 'faculty',\n",
              " 'wonder',\n",
              " 'antonin',\n",
              " 'scalia',\n",
              " 'avoided',\n",
              " 'learning',\n",
              " 'others',\n",
              " 'managed',\n",
              " 'grasp',\n",
              " 'processes',\n",
              " 'judging',\n",
              " 'hope',\n",
              " 'anecdote',\n",
              " 'gently',\n",
              " 'illustrates',\n",
              " 'less',\n",
              " 'humorous',\n",
              " 'stronger',\n",
              " 'clarence',\n",
              " 'thomas',\n",
              " 'mentions',\n",
              " 'wanting',\n",
              " 'return',\n",
              " 'degree',\n",
              " 'yale',\n",
              " 'minimum',\n",
              " 'questioning',\n",
              " 'deserves',\n",
              " 'reconsidered',\n",
              " 'radial',\n",
              " 'symmetry',\n",
              " 'several',\n",
              " 'extinct',\n",
              " 'lineages',\n",
              " 'included',\n",
              " 'echinodermata',\n",
              " 'bilateral',\n",
              " 'homostelea',\n",
              " 'asymmetrical',\n",
              " 'cothurnocystis',\n",
              " 'stylophora',\n",
              " 'apologize',\n",
              " 'reconciling',\n",
              " 'knowledge',\n",
              " 'sources',\n",
              " 'done',\n",
              " 'history',\n",
              " 'studies',\n",
              " 'archaeology',\n",
              " 'scan',\n",
              " 'mail',\n",
              " 'translate',\n",
              " 'mother',\n",
              " 'child',\n",
              " 'case',\n",
              " 'michael',\n",
              " 'jackson',\n",
              " 'studied',\n",
              " 'motives',\n",
              " 'reasonings',\n",
              " 'judged',\n",
              " 'upon',\n",
              " 'character',\n",
              " 'harshly',\n",
              " 'wacko',\n",
              " 'jacko',\n",
              " 'tell',\n",
              " 'ignore',\n",
              " 'incriminate',\n",
              " 'continue',\n",
              " 'refuting',\n",
              " 'bullshit',\n",
              " 'jayjg',\n",
              " 'keeps',\n",
              " 'jun',\n",
              " 'ok',\n",
              " 'bit',\n",
              " 'example',\n",
              " 'base',\n",
              " 'duck',\n",
              " 'barnstar',\n",
              " 'life',\n",
              " 'us',\n",
              " 'stars',\n",
              " 'post',\n",
              " 'block',\n",
              " 'expires',\n",
              " 'funny',\n",
              " 'thing',\n",
              " 'uncivil',\n",
              " 'heading',\n",
              " 'fight',\n",
              " 'freedom',\n",
              " 'contain',\n",
              " 'praise',\n",
              " 'looked',\n",
              " 'months',\n",
              " 'ago',\n",
              " 'much',\n",
              " 'able',\n",
              " 'quickly',\n",
              " 'text',\n",
              " 'hard',\n",
              " 'drive',\n",
              " 'meaning',\n",
              " 'updating',\n",
              " 'sound',\n",
              " 'time',\n",
              " 'generating',\n",
              " 'interest',\n",
              " 'spent',\n",
              " 'four',\n",
              " 'drum',\n",
              " 'freely',\n",
              " 'licensed',\n",
              " 'length',\n",
              " 'classical',\n",
              " 'music',\n",
              " 'unfortunately',\n",
              " 'attempts',\n",
              " 'failed',\n",
              " 'effectively',\n",
              " 'wikiproject',\n",
              " 'interested',\n",
              " 'archive',\n",
              " 'help',\n",
              " 'helpwikipedia',\n",
              " 'raulbot',\n",
              " 'given',\n",
              " 'featured',\n",
              " 'digg',\n",
              " 'com',\n",
              " 'downloads',\n",
              " 'got',\n",
              " 'diggs',\n",
              " 'imo',\n",
              " 'impressive',\n",
              " 'process',\n",
              " 'things',\n",
              " 'subpages',\n",
              " 'rfa',\n",
              " 'noseptember',\n",
              " 'differences',\n",
              " 'el',\n",
              " 'surprised',\n",
              " 'left',\n",
              " 'straw',\n",
              " 'never',\n",
              " 'claimed',\n",
              " 'donohue',\n",
              " 'position',\n",
              " 'practitioners',\n",
              " 'researchers',\n",
              " 'field',\n",
              " 'ignored',\n",
              " 'dsm',\n",
              " 'exactly',\n",
              " 'quote',\n",
              " 'says',\n",
              " 'agrees',\n",
              " 'combating',\n",
              " 'notion',\n",
              " 'absurd',\n",
              " 'part',\n",
              " 'claim',\n",
              " 'pedophilia',\n",
              " 'sexual',\n",
              " 'orientation',\n",
              " 'hold',\n",
              " 'unfair',\n",
              " 'call',\n",
              " 'disorder',\n",
              " 'divided',\n",
              " 'end',\n",
              " 'day',\n",
              " 'value',\n",
              " 'judgment',\n",
              " 'cantor',\n",
              " 'pointed',\n",
              " 'earlier',\n",
              " 'thread',\n",
              " 'scientific',\n",
              " 'judgement',\n",
              " 'choose',\n",
              " 'clearly',\n",
              " 'pretend',\n",
              " 'basis',\n",
              " 'mainland',\n",
              " 'asia',\n",
              " 'includes',\n",
              " 'lower',\n",
              " 'basin',\n",
              " 'china',\n",
              " 'yangtze',\n",
              " 'river',\n",
              " 'korea',\n",
              " 'fine',\n",
              " 'found',\n",
              " 'citation',\n",
              " 'comprehensive',\n",
              " 'dna',\n",
              " 'study',\n",
              " 'hammer',\n",
              " 'generarizations',\n",
              " 'speculation',\n",
              " 'yayoi',\n",
              " 'culture',\n",
              " 'brought',\n",
              " 'japan',\n",
              " 'migrants',\n",
              " 'trace',\n",
              " 'roots',\n",
              " 'southeast',\n",
              " 'south',\n",
              " 'describes',\n",
              " 'migration',\n",
              " 'based',\n",
              " 'sry',\n",
              " 'genes',\n",
              " 'close',\n",
              " 'lineage',\n",
              " 'haplogroups',\n",
              " 'reiterates',\n",
              " 'entire',\n",
              " 'haplogroup',\n",
              " 'proposed',\n",
              " 'asian',\n",
              " 'origin',\n",
              " 'definition',\n",
              " 'southern',\n",
              " 'hypothesizes',\n",
              " 'dispersals',\n",
              " 'neolithic',\n",
              " 'farmers',\n",
              " 'eventually',\n",
              " 'concluding',\n",
              " 'states',\n",
              " 'propose',\n",
              " 'chromosomes',\n",
              " 'descend',\n",
              " 'prehistoric',\n",
              " 'origins',\n",
              " 'southeastern',\n",
              " 'agriculture',\n",
              " 'region',\n",
              " 'global',\n",
              " 'sample',\n",
              " 'consisted',\n",
              " 'males',\n",
              " 'populations',\n",
              " 'including',\n",
              " 'sampled',\n",
              " 'across',\n",
              " 'japanese',\n",
              " 'archipelago',\n",
              " 'pretty',\n",
              " 'everyone',\n",
              " 'warren',\n",
              " 'county',\n",
              " 'surrounding',\n",
              " 'regions',\n",
              " 'glens',\n",
              " 'falls',\n",
              " 'hospital',\n",
              " 'qualifies',\n",
              " 'native',\n",
              " 'rachel',\n",
              " 'ray',\n",
              " 'actually',\n",
              " 'lake',\n",
              " 'luzerne',\n",
              " 'preceding',\n",
              " 'unsigned',\n",
              " 'comment',\n",
              " 'added',\n",
              " 'august',\n",
              " 'hi',\n",
              " 'explicit',\n",
              " 'fenian',\n",
              " 'warring',\n",
              " 'giant',\n",
              " 'causeway',\n",
              " 'terrorism',\n",
              " 'notability',\n",
              " 'rurika',\n",
              " 'kasuga',\n",
              " 'placed',\n",
              " 'speedily',\n",
              " 'person',\n",
              " 'people',\n",
              " 'band',\n",
              " 'club',\n",
              " 'company',\n",
              " 'web',\n",
              " 'indicate',\n",
              " 'assert',\n",
              " 'guidelines',\n",
              " 'generally',\n",
              " 'accepted',\n",
              " 'contest',\n",
              " 'tagging',\n",
              " 'existing',\n",
              " 'db',\n",
              " 'leave',\n",
              " 'explaining',\n",
              " 'hesitate',\n",
              " 'confirm',\n",
              " 'check',\n",
              " 'biographies',\n",
              " 'sites',\n",
              " 'bands',\n",
              " 'companies',\n",
              " 'feel',\n",
              " 'lead',\n",
              " 'briefly',\n",
              " 'summarize',\n",
              " 'armenia',\n",
              " 'necessary',\n",
              " 'thinks',\n",
              " 'sentence',\n",
              " 'redundant',\n",
              " 'welcome',\n",
              " 'tfd',\n",
              " 'eced',\n",
              " 'responded',\n",
              " 'without',\n",
              " 'seeing',\n",
              " 'responses',\n",
              " 'response',\n",
              " 'saw',\n",
              " 'mine',\n",
              " 'chicago',\n",
              " 'gay',\n",
              " 'antisemmitian',\n",
              " 'archangel',\n",
              " 'white',\n",
              " 'tiger',\n",
              " 'meow',\n",
              " 'greetingshhh',\n",
              " 'uh',\n",
              " 'two',\n",
              " 'ways',\n",
              " 'erased',\n",
              " 'ww',\n",
              " 'holocaust',\n",
              " 'brutally',\n",
              " 'slaying',\n",
              " 'jews',\n",
              " 'gays',\n",
              " 'gypsys',\n",
              " 'slavs',\n",
              " 'semitian',\n",
              " 'shave',\n",
              " 'head',\n",
              " 'bald',\n",
              " 'skinhead',\n",
              " 'meetings',\n",
              " 'doubt',\n",
              " 'words',\n",
              " 'bible',\n",
              " 'homosexuality',\n",
              " 'deadly',\n",
              " 'sin',\n",
              " 'pentagram',\n",
              " 'tatoo',\n",
              " 'forehead',\n",
              " 'satanistic',\n",
              " 'masses',\n",
              " 'pals',\n",
              " 'last',\n",
              " 'warning',\n",
              " 'fucking',\n",
              " 'appreciate',\n",
              " 'nazi',\n",
              " 'shwain',\n",
              " 'wish',\n",
              " 'anymore',\n",
              " 'beware',\n",
              " 'dark',\n",
              " 'side',\n",
              " 'fuck',\n",
              " 'filthy',\n",
              " 'ass',\n",
              " 'dry',\n",
              " 'screwed',\n",
              " 'someones',\n",
              " 'templates',\n",
              " 'dominance',\n",
              " 'bow',\n",
              " 'almighty',\n",
              " 'administrators',\n",
              " 'play',\n",
              " 'outside',\n",
              " 'mom',\n",
              " 'lisak',\n",
              " 'criticism',\n",
              " 'present',\n",
              " 'conforms',\n",
              " 'npv',\n",
              " 'rule',\n",
              " 'neutral',\n",
              " 'begin',\n",
              " 'offer',\n",
              " 'polygraph',\n",
              " 'concerned',\n",
              " 'results',\n",
              " 'shocks',\n",
              " 'complainant',\n",
              " 'lies',\n",
              " 'uncovered',\n",
              " 'recantation',\n",
              " 'perfectly',\n",
              " 'valid',\n",
              " 'telling',\n",
              " 'truth',\n",
              " 'machine',\n",
              " 'investigator',\n",
              " 'kanin',\n",
              " 'research',\n",
              " 'followup',\n",
              " 'recanted',\n",
              " 'story',\n",
              " 'possible',\n",
              " 'verify',\n",
              " 'false',\n",
              " 'recantations',\n",
              " 'followups',\n",
              " 'events',\n",
              " 'matched',\n",
              " 'accused',\n",
              " 'happened',\n",
              " 'arguing',\n",
              " 'respected',\n",
              " 'phd',\n",
              " 'baseless',\n",
              " 'agree',\n",
              " 'though',\n",
              " 'ammended',\n",
              " 'appropriate',\n",
              " 'significance',\n",
              " 'lazy',\n",
              " 'change',\n",
              " 'goes',\n",
              " 'claims',\n",
              " 'stalking',\n",
              " 'absolute',\n",
              " 'rubbish',\n",
              " 'serves',\n",
              " 'aggravate',\n",
              " 'assumed',\n",
              " 'faith',\n",
              " 'intentions',\n",
              " 'suggested',\n",
              " 'seen',\n",
              " 'reason',\n",
              " 'suggest',\n",
              " 'might',\n",
              " 'ulterior',\n",
              " 'motive',\n",
              " 'mass',\n",
              " 'links',\n",
              " 'ever',\n",
              " 'suggestion',\n",
              " 'administrative',\n",
              " 'mentioned',\n",
              " 'role',\n",
              " 'party',\n",
              " 'disagreement',\n",
              " 'rate',\n",
              " 'conflict',\n",
              " 'thus',\n",
              " 'extend',\n",
              " 'toward',\n",
              " ...]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZieL6fEKtifF",
        "trusted": true,
        "outputId": "8fee0cd4-2c64-4bf4-a71a-d6577b5269bd"
      },
      "source": [
        "len(list(vectorizer.vocabulary_.keys()))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "168645"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kmGNMOiStifG",
        "trusted": true
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "vectorizer = TfidfVectorizer(stop_words = my_stopwords)\n",
        "bag_of_words = vectorizer.fit_transform(X)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xwbot_8LtifG",
        "trusted": true,
        "outputId": "aa960dec-77eb-4378-a763-287e7aa125c2"
      },
      "source": [
        "print(bag_of_words.shape, len(vectorizer.vocabulary_))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(159571, 168645) 168645\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KBHdug-JtifI",
        "trusted": true,
        "outputId": "d23b3f03-1b82-4a59-d89b-155cb79bcbcf"
      },
      "source": [
        "embeds = []\n",
        "zeros_embed = 100*[0]\n",
        "for key,val in sorted(vectorizer.vocabulary_.items(), key = lambda kv:(kv[1], kv[0])):\n",
        "# vectorizer.vocabulary_.keys():\n",
        "    if key in embeddings_dictionary:\n",
        "        embeds.append(embeddings_dictionary[key])\n",
        "    else:\n",
        "        embeds.append(zeros_embed)\n",
        "    if val%1000 == 0:\n",
        "        print(val,key)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0 aa\n",
            "1000 accorta\n",
            "2000 adminnaizsi\n",
            "3000 aggrevation\n",
            "4000 albo\n",
            "5000 altor\n",
            "6000 andjapan\n",
            "7000 antiquary\n",
            "8000 aquainted\n",
            "9000 arrogans\n",
            "10000 associazioni\n",
            "11000 austrolestes\n",
            "12000 baceline\n",
            "13000 bannock\n",
            "14000 bd\n",
            "15000 benegiciary\n",
            "16000 bidhonene\n",
            "17000 blakey\n",
            "18000 boilers\n",
            "19000 bpm\n",
            "20000 bronze\n",
            "21000 burgnich\n",
            "22000 calvary\n",
            "23000 carpelan\n",
            "24000 ceirtain\n",
            "25000 charge\n",
            "26000 chlorophyta\n",
            "27000 civillist\n",
            "28000 coauthor\n",
            "29000 commeneced\n",
            "30000 coneheads\n",
            "31000 contflict\n",
            "32000 correspondant\n",
            "33000 creb\n",
            "34000 cudnt\n",
            "35000 dainomite\n",
            "36000 deadend\n",
            "37000 delamarche\n",
            "38000 derrier\n",
            "39000 dialectal\n",
            "40000 diry\n",
            "41000 distant\n",
            "42000 donkeykong\n",
            "43000 drowning\n",
            "44000 eachothers\n",
            "45000 egad\n",
            "46000 emaneuel\n",
            "47000 enigmasoldier\n",
            "48000 ern\n",
            "49000 euwe\n",
            "50000 experimental\n",
            "51000 falcionek\n",
            "52000 feckless\n",
            "53000 finalist\n",
            "54000 flunk\n",
            "55000 fozzy\n",
            "56000 ftrl\n",
            "57000 gallium\n",
            "58000 generated\n",
            "59000 gilliam\n",
            "60000 golat\n",
            "61000 gratim\n",
            "62000 guidelinesthe\n",
            "63000 halab\n",
            "64000 harvested\n",
            "65000 helicopterllama\n",
            "66000 hiineedrequestforcomment\n",
            "67000 homann\n",
            "68000 htttp\n",
            "69000 ibar\n",
            "70000 imagemap\n",
            "71000 incomplete\n",
            "72000 ingest\n",
            "73000 interdialects\n",
            "74000 irdgas\n",
            "75000 jaa\n",
            "76000 jello\n",
            "77000 jonridinger\n",
            "78000 kahlenkahlenkahlen\n",
            "79000 kazeshini\n",
            "80000 kieffer\n",
            "81000 kocharly\n",
            "82000 kudos\n",
            "83000 landrace\n",
            "84000 leda\n",
            "85000 libraries\n",
            "86000 lituanised\n",
            "87000 loudoun\n",
            "88000 macgregors\n",
            "89000 malda\n",
            "90000 mariasamanthamelissamelissa\n",
            "91000 maturely\n",
            "92000 medico\n",
            "93000 mesopotamians\n",
            "94000 milbourneone\n",
            "95000 misspoke\n",
            "96000 monga\n",
            "97000 mountains\n",
            "98000 muruga\n",
            "99000 namking\n",
            "100000 necessitate\n",
            "101000 newsreaders\n",
            "102000 nnnnnnnnnnnnniiiiggggggggggggaaaaaaaaa\n",
            "103000 notrhbys\n",
            "104000 obfuscations\n",
            "105000 ojonelson\n",
            "106000 opportunity\n",
            "107000 otoh\n",
            "108000 padilla\n",
            "109000 parchments\n",
            "110000 pbs\n",
            "111000 permdude\n",
            "112000 phobe\n",
            "113000 pks\n",
            "114000 policymakers\n",
            "115000 potsworth\n",
            "116000 preseted\n",
            "117000 projecte\n",
            "118000 psychodynamic\n",
            "119000 qbcjr\n",
            "120000 radially\n",
            "121000 rationales\n",
            "122000 recognizing\n",
            "123000 regreted\n",
            "124000 reperfusion\n",
            "125000 retore\n",
            "126000 rill\n",
            "127000 ronanmckenna\n",
            "128000 runnersworld\n",
            "129000 salutation\n",
            "130000 savants\n",
            "131000 scrambling\n",
            "132000 sellings\n",
            "133000 shaddadeh\n",
            "134000 shitskin\n",
            "135000 sigurdsson\n",
            "136000 skjema\n",
            "137000 sniffs\n",
            "138000 soothe\n",
            "139000 spicant\n",
            "140000 stains\n",
            "141000 stipan\n",
            "142000 stylings\n",
            "143000 sunbelt\n",
            "144000 swares\n",
            "145000 tabthis\n",
            "146000 tast\n",
            "147000 tengri\n",
            "148000 theme\n",
            "149000 thuluva\n",
            "150000 tocqueville\n",
            "151000 trackable\n",
            "152000 trinity\n",
            "153000 turkist\n",
            "154000 ullmann\n",
            "155000 unencombered\n",
            "156000 unrestored\n",
            "157000 usnchistory\n",
            "158000 varifed\n",
            "159000 vibs\n",
            "160000 volcker\n",
            "161000 ware\n",
            "162000 welkom\n",
            "163000 wiener\n",
            "164000 winceb\n",
            "165000 worldnetdaily\n",
            "166000 xmrv\n",
            "167000 yogh\n",
            "168000 zent\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G0yOBqoytifJ",
        "trusted": true
      },
      "source": [
        "import scipy.sparse as sparse\n",
        "my_embeddings = sparse.csr_matrix(embeds)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IvyFcweztifK",
        "trusted": true
      },
      "source": [
        "doc_embeds = bag_of_words.dot(my_embeddings)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dbCPg2xgtifL",
        "trusted": true,
        "outputId": "d6b82154-af6a-47f1-e81f-2670b315166b"
      },
      "source": [
        "doc_embeds.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(159571, 100)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s7Y7y6uYtifM",
        "trusted": true
      },
      "source": [
        "X_train_de, X_test_de, y_train_de, y_test_de = train_test_split(doc_embeds, toxic, test_size=0.20, random_state=42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ve0ro8VAtifM",
        "trusted": true
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.metrics import plot_roc_curve"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "id": "8DxESK5UtifN",
        "trusted": false
      },
      "source": [
        "lr = LogisticRegression()\n",
        "dt = DecisionTreeClassifier(max_depth=25)\n",
        "rf = RandomForestClassifier(max_depth=15, n_estimators=40)\n",
        "clf = SVC(kernel='linear',gamma='auto',degree=2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "Elu8yfGTExYV"
      },
      "source": [
        "from sklearn.model_selection import RepeatedKFold\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.model_selection import RandomizedSearchCV"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "vbmuulK2ExYV"
      },
      "source": [
        "model = LogisticRegression(max_iter=1000)\n",
        "cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
        "space = dict()\n",
        "space['fit_intercept'] = [True, False]\n",
        "space['C']= np.linspace(1, 10, 10)\n",
        "search = RandomizedSearchCV(model, space, scoring='recall', n_jobs=1, cv=cv)\n",
        "result = search.fit(X_train_de, y_train_de)\n",
        "print('Best Score: %s' % result.best_score_)\n",
        "print('Best Hyperparameters: %s' % result.best_params_)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "821uD5L-FCyL"
      },
      "source": [
        "Best Score: 0.5476605288942155\r\n",
        "Best Hyperparameters: {'fit_intercept': False, 'C': 7.0}"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "YPO1w0k3ExYV"
      },
      "source": [
        "model = RandomForestClassifier()\n",
        "cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
        "space = dict()\n",
        "space['max_depth'] = [5,6,7,8,9,10,11,12,13,14,15]\n",
        "space['n_estimators']= [10,20,30,40,50]\n",
        "search = RandomizedSearchCV(model, space, scoring='recall', n_jobs=-1, cv=cv)\n",
        "result = search.fit(X_train_de,y_train_de)\n",
        "print('Best Score: %s' % result.best_score_)\n",
        "print('Best Hyperparameters: %s' % result.best_params_)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i8r_AJ5cFgGj"
      },
      "source": [
        "Best Score: 0.56676578668789757 Best Hyperparameters: {'max_depth': 15, 'n_estimators': 40}"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "YsbhYD0sExYW"
      },
      "source": [
        "from sklearn.naive_bayes import GaussianNB"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "ls0B79LOExYW"
      },
      "source": [
        "model = GaussianNB()\n",
        "cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
        "space = dict()\n",
        "space['var_smoothing'] = [1e-5, 1e-4, 1e-3, 1e-2, 1e-1,1e-6,1e-7,1e-8,1e-9,1e-10,1e-11,1e-12,1e-13,1e-14]\n",
        "search = GridSearchCV(model, space, scoring='recall', n_jobs=-1, cv=cv)\n",
        "result = search.fit(X_train_de, y_train_de)\n",
        "print('Best Score: %s' % result.best_score_)\n",
        "print('Best Hyperparameters: %s' % result.best_params_)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u0RTWsi-GLzR"
      },
      "source": [
        "Best Score: 0.5367545446 Best Hyperparameters: {'var_smoothing': 1e-3}"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "LP3_1_XPExYW"
      },
      "source": [
        "model = svm.SVC()\n",
        "cv = RepeatedKFold(n_splits=5, n_repeats=3, random_state=1)\n",
        "space = dict()\n",
        "space['kernel'] = ['rbf','linear','poly']\n",
        "search = GridSearchCV(model, space, scoring='recall', n_jobs=-1, cv=cv)\n",
        "result = search.fit(X_train_de, y_train_de)\n",
        "print('Best Score: %s' % result.best_score_)\n",
        "print('Best Hyperparameters: %s' % result.best_params_)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WRxktPlPGwC6"
      },
      "source": [
        "Best Score: 0.595645577546478 Best Hyperparameters: {'kernel': 'linear'}"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "EE8sDUi6ExYW"
      },
      "source": [
        "model = DecisionTreeClassifier()\n",
        "cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
        "space = dict()\n",
        "space['max_depth']= [5,10,15,20,25,30]\n",
        "search = GridSearchCV(model, space, scoring='recall', n_jobs=-1, cv=cv)\n",
        "result = search.fit(X_train_res, Y_train_res)\n",
        "print('Best Score: %s' % result.best_score_)\n",
        "print('Best Hyperparameters: %s' % result.best_params_)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6odLRLBhQUXB"
      },
      "source": [
        "Best Score: 0.5367545446 Best Hyperparameters: {'max_depth': 25}"
      ]
    }
  ]
}