ML Pipeline Preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ML Pipeline Preparation\n",
    "Follow the instructions below to help you create your ML pipeline.\n",
    "### 1. Import libraries and load data from database.\n",
    "- Import Python libraries\n",
    "- Load dataset from database with [`read_sql_table`](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.read_sql_table.html)\n",
    "- Define feature and target variables X and Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: nltk in /opt/conda/lib/python3.6/site-packages (3.2.5)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from nltk) (1.11.0)\n",
      "Requirement already satisfied: plotly in /opt/conda/lib/python3.6/site-packages (2.0.15)\n",
      "Requirement already satisfied: decorator>=4.0.6 in /opt/conda/lib/python3.6/site-packages (from plotly) (4.0.11)\n",
      "Requirement already satisfied: nbformat>=4.2 in /opt/conda/lib/python3.6/site-packages (from plotly) (4.4.0)\n",
      "Requirement already satisfied: pytz in /opt/conda/lib/python3.6/site-packages (from plotly) (2017.3)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.6/site-packages (from plotly) (2.18.4)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from plotly) (1.11.0)\n",
      "Requirement already satisfied: jupyter-core in /opt/conda/lib/python3.6/site-packages (from nbformat>=4.2->plotly) (4.4.0)\n",
      "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /opt/conda/lib/python3.6/site-packages (from nbformat>=4.2->plotly) (2.6.0)\n",
      "Requirement already satisfied: traitlets>=4.1 in /opt/conda/lib/python3.6/site-packages (from nbformat>=4.2->plotly) (4.3.2)\n",
      "Requirement already satisfied: ipython-genutils in /opt/conda/lib/python3.6/site-packages (from nbformat>=4.2->plotly) (0.2.0)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->plotly) (3.0.4)\n",
      "Requirement already satisfied: idna<2.7,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->plotly) (2.6)\n",
      "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /opt/conda/lib/python3.6/site-packages (from requests->plotly) (1.22)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.6/site-packages (from requests->plotly) (2019.11.28)\n"
     ]
    }
   ],
   "source": [
    "!pip install nltk\n",
    "!pip install plotly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /root/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import libraries\n",
    "\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('averaged_perceptron_tagger')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_columns', 500)\n",
    "\n",
    "import sys\n",
    "import os\n",
    "import re\n",
    "from sqlalchemy import create_engine\n",
    "import pickle\n",
    "\n",
    "from scipy.stats import gmean\n",
    "# import relevant functions/modules from the sklearn\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import fbeta_score, make_scorer\n",
    "from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.base import BaseEstimator,TransformerMixin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data from database\n",
    "database_filepath = \"./disaster_response_db.db\"\n",
    "engine = create_engine('sqlite:///' + database_filepath)\n",
    "\n",
    "df = pd.read_sql_table('Disaster',engine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>related</th>\n",
       "      <th>request</th>\n",
       "      <th>offer</th>\n",
       "      <th>aid_related</th>\n",
       "      <th>medical_help</th>\n",
       "      <th>medical_products</th>\n",
       "      <th>search_and_rescue</th>\n",
       "      <th>security</th>\n",
       "      <th>military</th>\n",
       "      <th>child_alone</th>\n",
       "      <th>water</th>\n",
       "      <th>food</th>\n",
       "      <th>shelter</th>\n",
       "      <th>clothing</th>\n",
       "      <th>money</th>\n",
       "      <th>missing_people</th>\n",
       "      <th>refugees</th>\n",
       "      <th>death</th>\n",
       "      <th>other_aid</th>\n",
       "      <th>infrastructure_related</th>\n",
       "      <th>transport</th>\n",
       "      <th>buildings</th>\n",
       "      <th>electricity</th>\n",
       "      <th>tools</th>\n",
       "      <th>hospitals</th>\n",
       "      <th>shops</th>\n",
       "      <th>aid_centers</th>\n",
       "      <th>other_infrastructure</th>\n",
       "      <th>weather_related</th>\n",
       "      <th>floods</th>\n",
       "      <th>storm</th>\n",
       "      <th>fire</th>\n",
       "      <th>earthquake</th>\n",
       "      <th>cold</th>\n",
       "      <th>other_weather</th>\n",
       "      <th>direct_report</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>26216.00000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.0</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "      <td>26216.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>15224.82133</td>\n",
       "      <td>0.773650</td>\n",
       "      <td>0.170659</td>\n",
       "      <td>0.004501</td>\n",
       "      <td>0.414251</td>\n",
       "      <td>0.079493</td>\n",
       "      <td>0.050084</td>\n",
       "      <td>0.027617</td>\n",
       "      <td>0.017966</td>\n",
       "      <td>0.032804</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.063778</td>\n",
       "      <td>0.111497</td>\n",
       "      <td>0.088267</td>\n",
       "      <td>0.015449</td>\n",
       "      <td>0.023039</td>\n",
       "      <td>0.011367</td>\n",
       "      <td>0.033377</td>\n",
       "      <td>0.045545</td>\n",
       "      <td>0.131446</td>\n",
       "      <td>0.065037</td>\n",
       "      <td>0.045812</td>\n",
       "      <td>0.050847</td>\n",
       "      <td>0.020293</td>\n",
       "      <td>0.006065</td>\n",
       "      <td>0.010795</td>\n",
       "      <td>0.004577</td>\n",
       "      <td>0.011787</td>\n",
       "      <td>0.043904</td>\n",
       "      <td>0.278341</td>\n",
       "      <td>0.082202</td>\n",
       "      <td>0.093187</td>\n",
       "      <td>0.010757</td>\n",
       "      <td>0.093645</td>\n",
       "      <td>0.020217</td>\n",
       "      <td>0.052487</td>\n",
       "      <td>0.193584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>8826.88914</td>\n",
       "      <td>0.435276</td>\n",
       "      <td>0.376218</td>\n",
       "      <td>0.066940</td>\n",
       "      <td>0.492602</td>\n",
       "      <td>0.270513</td>\n",
       "      <td>0.218122</td>\n",
       "      <td>0.163875</td>\n",
       "      <td>0.132831</td>\n",
       "      <td>0.178128</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.244361</td>\n",
       "      <td>0.314752</td>\n",
       "      <td>0.283688</td>\n",
       "      <td>0.123331</td>\n",
       "      <td>0.150031</td>\n",
       "      <td>0.106011</td>\n",
       "      <td>0.179621</td>\n",
       "      <td>0.208500</td>\n",
       "      <td>0.337894</td>\n",
       "      <td>0.246595</td>\n",
       "      <td>0.209081</td>\n",
       "      <td>0.219689</td>\n",
       "      <td>0.141003</td>\n",
       "      <td>0.077643</td>\n",
       "      <td>0.103338</td>\n",
       "      <td>0.067502</td>\n",
       "      <td>0.107927</td>\n",
       "      <td>0.204887</td>\n",
       "      <td>0.448191</td>\n",
       "      <td>0.274677</td>\n",
       "      <td>0.290700</td>\n",
       "      <td>0.103158</td>\n",
       "      <td>0.291340</td>\n",
       "      <td>0.140743</td>\n",
       "      <td>0.223011</td>\n",
       "      <td>0.395114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>2.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>7446.75000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>15662.50000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>22924.25000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>30265.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                id       related       request         offer   aid_related  \\\n",
       "count  26216.00000  26216.000000  26216.000000  26216.000000  26216.000000   \n",
       "mean   15224.82133      0.773650      0.170659      0.004501      0.414251   \n",
       "std     8826.88914      0.435276      0.376218      0.066940      0.492602   \n",
       "min        2.00000      0.000000      0.000000      0.000000      0.000000   \n",
       "25%     7446.75000      1.000000      0.000000      0.000000      0.000000   \n",
       "50%    15662.50000      1.000000      0.000000      0.000000      0.000000   \n",
       "75%    22924.25000      1.000000      0.000000      0.000000      1.000000   \n",
       "max    30265.00000      2.000000      1.000000      1.000000      1.000000   \n",
       "\n",
       "       medical_help  medical_products  search_and_rescue      security  \\\n",
       "count  26216.000000      26216.000000       26216.000000  26216.000000   \n",
       "mean       0.079493          0.050084           0.027617      0.017966   \n",
       "std        0.270513          0.218122           0.163875      0.132831   \n",
       "min        0.000000          0.000000           0.000000      0.000000   \n",
       "25%        0.000000          0.000000           0.000000      0.000000   \n",
       "50%        0.000000          0.000000           0.000000      0.000000   \n",
       "75%        0.000000          0.000000           0.000000      0.000000   \n",
       "max        1.000000          1.000000           1.000000      1.000000   \n",
       "\n",
       "           military  child_alone         water          food       shelter  \\\n",
       "count  26216.000000      26216.0  26216.000000  26216.000000  26216.000000   \n",
       "mean       0.032804          0.0      0.063778      0.111497      0.088267   \n",
       "std        0.178128          0.0      0.244361      0.314752      0.283688   \n",
       "min        0.000000          0.0      0.000000      0.000000      0.000000   \n",
       "25%        0.000000          0.0      0.000000      0.000000      0.000000   \n",
       "50%        0.000000          0.0      0.000000      0.000000      0.000000   \n",
       "75%        0.000000          0.0      0.000000      0.000000      0.000000   \n",
       "max        1.000000          0.0      1.000000      1.000000      1.000000   \n",
       "\n",
       "           clothing         money  missing_people      refugees         death  \\\n",
       "count  26216.000000  26216.000000    26216.000000  26216.000000  26216.000000   \n",
       "mean       0.015449      0.023039        0.011367      0.033377      0.045545   \n",
       "std        0.123331      0.150031        0.106011      0.179621      0.208500   \n",
       "min        0.000000      0.000000        0.000000      0.000000      0.000000   \n",
       "25%        0.000000      0.000000        0.000000      0.000000      0.000000   \n",
       "50%        0.000000      0.000000        0.000000      0.000000      0.000000   \n",
       "75%        0.000000      0.000000        0.000000      0.000000      0.000000   \n",
       "max        1.000000      1.000000        1.000000      1.000000      1.000000   \n",
       "\n",
       "          other_aid  infrastructure_related     transport     buildings  \\\n",
       "count  26216.000000            26216.000000  26216.000000  26216.000000   \n",
       "mean       0.131446                0.065037      0.045812      0.050847   \n",
       "std        0.337894                0.246595      0.209081      0.219689   \n",
       "min        0.000000                0.000000      0.000000      0.000000   \n",
       "25%        0.000000                0.000000      0.000000      0.000000   \n",
       "50%        0.000000                0.000000      0.000000      0.000000   \n",
       "75%        0.000000                0.000000      0.000000      0.000000   \n",
       "max        1.000000                1.000000      1.000000      1.000000   \n",
       "\n",
       "        electricity         tools     hospitals         shops   aid_centers  \\\n",
       "count  26216.000000  26216.000000  26216.000000  26216.000000  26216.000000   \n",
       "mean       0.020293      0.006065      0.010795      0.004577      0.011787   \n",
       "std        0.141003      0.077643      0.103338      0.067502      0.107927   \n",
       "min        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "max        1.000000      1.000000      1.000000      1.000000      1.000000   \n",
       "\n",
       "       other_infrastructure  weather_related        floods         storm  \\\n",
       "count          26216.000000     26216.000000  26216.000000  26216.000000   \n",
       "mean               0.043904         0.278341      0.082202      0.093187   \n",
       "std                0.204887         0.448191      0.274677      0.290700   \n",
       "min                0.000000         0.000000      0.000000      0.000000   \n",
       "25%                0.000000         0.000000      0.000000      0.000000   \n",
       "50%                0.000000         0.000000      0.000000      0.000000   \n",
       "75%                0.000000         1.000000      0.000000      0.000000   \n",
       "max                1.000000         1.000000      1.000000      1.000000   \n",
       "\n",
       "               fire    earthquake          cold  other_weather  direct_report  \n",
       "count  26216.000000  26216.000000  26216.000000   26216.000000   26216.000000  \n",
       "mean       0.010757      0.093645      0.020217       0.052487       0.193584  \n",
       "std        0.103158      0.291340      0.140743       0.223011       0.395114  \n",
       "min        0.000000      0.000000      0.000000       0.000000       0.000000  \n",
       "25%        0.000000      0.000000      0.000000       0.000000       0.000000  \n",
       "50%        0.000000      0.000000      0.000000       0.000000       0.000000  \n",
       "75%        0.000000      0.000000      0.000000       0.000000       0.000000  \n",
       "max        1.000000      1.000000      1.000000       1.000000       1.000000  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Remove child alone as it has all zeros only\n",
    "df = df.drop(['child_alone'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>message</th>\n",
       "      <th>original</th>\n",
       "      <th>genre</th>\n",
       "      <th>request</th>\n",
       "      <th>offer</th>\n",
       "      <th>aid_related</th>\n",
       "      <th>medical_help</th>\n",
       "      <th>medical_products</th>\n",
       "      <th>search_and_rescue</th>\n",
       "      <th>security</th>\n",
       "      <th>military</th>\n",
       "      <th>water</th>\n",
       "      <th>food</th>\n",
       "      <th>shelter</th>\n",
       "      <th>clothing</th>\n",
       "      <th>money</th>\n",
       "      <th>missing_people</th>\n",
       "      <th>refugees</th>\n",
       "      <th>death</th>\n",
       "      <th>other_aid</th>\n",
       "      <th>infrastructure_related</th>\n",
       "      <th>transport</th>\n",
       "      <th>buildings</th>\n",
       "      <th>electricity</th>\n",
       "      <th>tools</th>\n",
       "      <th>hospitals</th>\n",
       "      <th>shops</th>\n",
       "      <th>aid_centers</th>\n",
       "      <th>other_infrastructure</th>\n",
       "      <th>weather_related</th>\n",
       "      <th>floods</th>\n",
       "      <th>storm</th>\n",
       "      <th>fire</th>\n",
       "      <th>earthquake</th>\n",
       "      <th>cold</th>\n",
       "      <th>other_weather</th>\n",
       "      <th>direct_report</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>related</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>3395</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "      <td>6122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>6643</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "      <td>19906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>132</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "      <td>188</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id  message  original  genre  request  offer  aid_related  \\\n",
       "related                                                                 \n",
       "0         6122     6122      3395   6122     6122   6122         6122   \n",
       "1        19906    19906      6643  19906    19906  19906        19906   \n",
       "2          188      188       132    188      188    188          188   \n",
       "\n",
       "         medical_help  medical_products  search_and_rescue  security  \\\n",
       "related                                                                \n",
       "0                6122              6122               6122      6122   \n",
       "1               19906             19906              19906     19906   \n",
       "2                 188               188                188       188   \n",
       "\n",
       "         military  water   food  shelter  clothing  money  missing_people  \\\n",
       "related                                                                     \n",
       "0            6122   6122   6122     6122      6122   6122            6122   \n",
       "1           19906  19906  19906    19906     19906  19906           19906   \n",
       "2             188    188    188      188       188    188             188   \n",
       "\n",
       "         refugees  death  other_aid  infrastructure_related  transport  \\\n",
       "related                                                                  \n",
       "0            6122   6122       6122                    6122       6122   \n",
       "1           19906  19906      19906                   19906      19906   \n",
       "2             188    188        188                     188        188   \n",
       "\n",
       "         buildings  electricity  tools  hospitals  shops  aid_centers  \\\n",
       "related                                                                 \n",
       "0             6122         6122   6122       6122   6122         6122   \n",
       "1            19906        19906  19906      19906  19906        19906   \n",
       "2              188          188    188        188    188          188   \n",
       "\n",
       "         other_infrastructure  weather_related  floods  storm   fire  \\\n",
       "related                                                                \n",
       "0                        6122             6122    6122   6122   6122   \n",
       "1                       19906            19906   19906  19906  19906   \n",
       "2                         188              188     188    188    188   \n",
       "\n",
       "         earthquake   cold  other_weather  direct_report  \n",
       "related                                                   \n",
       "0              6122   6122           6122           6122  \n",
       "1             19906  19906          19906          19906  \n",
       "2               188    188            188            188  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(\"related\").count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Given value 2 in the related field are neglible so it could be error. Replacing 2 with 1 to consider it a valid response.\n",
    "# Alternatively, we could have assumed it to be 0 also. In the absence of information I have gone with majority class.\n",
    "df['related']=df['related'].map(lambda x: 1 if x == 2 else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract X and y variables from the data for the modelling\n",
    "X = df['message']\n",
    "y = df.iloc[:,4:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Write a tokenization function to process your text data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(text,url_place_holder_string=\"urlplaceholder\"):\n",
    "    \"\"\"\n",
    "    Tokenize the text function\n",
    "    \n",
    "    Arguments:\n",
    "        text -> Text message which needs to be tokenized\n",
    "    Output:\n",
    "        clean_tokens -> List of tokens extracted from the provided text\n",
    "    \"\"\"\n",
    "    \n",
    "    # Replace all urls with a urlplaceholder string\n",
    "    url_regex = 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'\n",
    "    \n",
    "    # Extract all the urls from the provided text \n",
    "    detected_urls = re.findall(url_regex, text)\n",
    "    \n",
    "    # Replace url with a url placeholder string\n",
    "    for detected_url in detected_urls:\n",
    "        text = text.replace(detected_url, url_place_holder_string)\n",
    "\n",
    "    # Extract the word tokens from the provided text\n",
    "    tokens = nltk.word_tokenize(text)\n",
    "    \n",
    "    #Lemmanitizer to remove inflectional and derivationally related forms of a word\n",
    "    lemmatizer = nltk.WordNetLemmatizer()\n",
    "\n",
    "    # List of clean tokens\n",
    "    clean_tokens = [lemmatizer.lemmatize(w).lower().strip() for w in tokens]\n",
    "    return clean_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build a custom transformer which will extract the starting verb of a sentence\n",
    "class StartingVerbExtractor(BaseEstimator, TransformerMixin):\n",
    "    \"\"\"\n",
    "    Starting Verb Extractor class\n",
    "    \n",
    "    This class extract the starting verb of a sentence,\n",
    "    creating a new feature for the ML classifier\n",
    "    \"\"\"\n",
    "\n",
    "    def starting_verb(self, text):\n",
    "        sentence_list = nltk.sent_tokenize(text)\n",
    "        for sentence in sentence_list:\n",
    "            pos_tags = nltk.pos_tag(tokenize(sentence))\n",
    "            first_word, first_tag = pos_tags[0]\n",
    "            if first_tag in ['VB', 'VBP'] or first_word == 'RT':\n",
    "                return True\n",
    "        return False\n",
    "\n",
    "    # Given it is a tranformer we can return the self \n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        X_tagged = pd.Series(X).apply(self.starting_verb)\n",
    "        return pd.DataFrame(X_tagged)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Build a machine learning pipeline\n",
    "This machine pipeline should take in the `message` column as input and output classification results on the other 36 categories in the dataset. You may find the [MultiOutputClassifier](http://scikit-learn.org/stable/modules/generated/sklearn.multioutput.MultiOutputClassifier.html) helpful for predicting multiple target variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline1 = Pipeline([\n",
    "        ('features', FeatureUnion([\n",
    "\n",
    "            ('text_pipeline', Pipeline([\n",
    "                ('count_vectorizer', CountVectorizer(tokenizer=tokenize)),\n",
    "                ('tfidf_transformer', TfidfTransformer())\n",
    "            ]))\n",
    "            \n",
    "        ])),\n",
    "\n",
    "        ('classifier', MultiOutputClassifier(AdaBoostClassifier()))\n",
    "    ])\n",
    "\n",
    "pipeline2 = Pipeline([\n",
    "        ('features', FeatureUnion([\n",
    "\n",
    "            ('text_pipeline', Pipeline([\n",
    "                ('count_vectorizer', CountVectorizer(tokenizer=tokenize)),\n",
    "                ('tfidf_transformer', TfidfTransformer())\n",
    "            ])),\n",
    "\n",
    "            ('starting_verb_transformer', StartingVerbExtractor())\n",
    "        ])),\n",
    "\n",
    "        ('classifier', MultiOutputClassifier(AdaBoostClassifier()))\n",
    "    ])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Train pipeline\n",
    "- Split data into train and test sets\n",
    "- Train pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "pipeline_fitted = pipeline1.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Test your model\n",
    "Report the f1 score, precision and recall for each output category of the dataset. You can do this by iterating through the columns and calling sklearn's `classification_report` on each."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                        precision    recall  f1-score   support\n",
      "\n",
      "               related       0.82      0.94      0.88      4974\n",
      "               request       0.79      0.54      0.64      1110\n",
      "                 offer       0.15      0.07      0.10        27\n",
      "           aid_related       0.76      0.58      0.66      2687\n",
      "          medical_help       0.59      0.27      0.37       519\n",
      "      medical_products       0.58      0.35      0.44       289\n",
      "     search_and_rescue       0.63      0.15      0.25       189\n",
      "              security       0.28      0.06      0.10       118\n",
      "              military       0.60      0.36      0.45       222\n",
      "                 water       0.75      0.62      0.68       420\n",
      "                  food       0.77      0.64      0.70       706\n",
      "               shelter       0.76      0.52      0.62       579\n",
      "              clothing       0.65      0.28      0.39       107\n",
      "                 money       0.60      0.27      0.38       157\n",
      "        missing_people       0.54      0.20      0.29        71\n",
      "              refugees       0.58      0.27      0.37       206\n",
      "                 death       0.78      0.42      0.55       314\n",
      "             other_aid       0.50      0.15      0.23       814\n",
      "infrastructure_related       0.38      0.09      0.14       453\n",
      "             transport       0.70      0.22      0.33       288\n",
      "             buildings       0.61      0.38      0.47       341\n",
      "           electricity       0.54      0.29      0.38       119\n",
      "                 tools       0.00      0.00      0.00        36\n",
      "             hospitals       0.14      0.03      0.05        74\n",
      "                 shops       0.00      0.00      0.00        33\n",
      "           aid_centers       0.36      0.09      0.14        91\n",
      "  other_infrastructure       0.37      0.08      0.14       290\n",
      "       weather_related       0.86      0.66      0.75      1768\n",
      "                floods       0.88      0.54      0.67       544\n",
      "                 storm       0.75      0.50      0.60       597\n",
      "                  fire       0.49      0.27      0.35        67\n",
      "            earthquake       0.88      0.76      0.82       584\n",
      "                  cold       0.56      0.25      0.35       134\n",
      "         other_weather       0.52      0.17      0.26       349\n",
      "         direct_report       0.72      0.50      0.59      1208\n",
      "\n",
      "           avg / total       0.73      0.58      0.63     20485\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "y_prediction_train = pipeline_fitted.predict(X_train)\n",
    "y_prediction_test = pipeline_fitted.predict(X_test)\n",
    "\n",
    "# Print classification report on test data\n",
    "print(classification_report(y_test.values, y_prediction_test, target_names=y.columns.values))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. Improve your model\n",
    "Use grid search to find better parameters. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score='raise',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('features', FeatureUnion(n_jobs=1,\n",
       "       transformer_list=[('text_pipeline', Pipeline(memory=None,\n",
       "     steps=[('count_vectorizer', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase...mator=None,\n",
       "          learning_rate=1.0, n_estimators=50, random_state=None),\n",
       "           n_jobs=1))]),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'classifier__estimator__learning_rate': [0.01, 0.02, 0.05], 'classifier__estimator__n_estimators': [10, 20, 40]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='f1_micro', verbose=0)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parameters_grid = {'classifier__estimator__learning_rate': [0.01, 0.02, 0.05],\n",
    "              'classifier__estimator__n_estimators': [10, 20, 40]}\n",
    "\n",
    "cv = GridSearchCV(pipeline1, param_grid=parameters_grid, scoring='f1_micro', n_jobs=-1)\n",
    "cv.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the prediction values from the grid search cross validator\n",
    "y_prediction_test = cv.predict(X_test)\n",
    "y_prediction_train = cv.predict(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. Test your model\n",
    "Show the accuracy, precision, and recall of the tuned model.  \n",
    "\n",
    "Since this project focuses on code quality, process, and  pipelines, there is no minimum performance metric needed to pass. However, make sure to fine tune your models for accuracy, precision and recall to make your project stand out - especially for your portfolio!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                        precision    recall  f1-score   support\n",
      "\n",
      "               related       0.76      1.00      0.86      4974\n",
      "               request       0.59      0.37      0.46      1110\n",
      "                 offer       0.00      0.00      0.00        27\n",
      "           aid_related       0.78      0.18      0.29      2687\n",
      "          medical_help       0.59      0.10      0.17       519\n",
      "      medical_products       0.64      0.14      0.23       289\n",
      "     search_and_rescue       0.67      0.15      0.24       189\n",
      "              security       0.00      0.00      0.00       118\n",
      "              military       0.53      0.16      0.25       222\n",
      "                 water       0.57      0.82      0.68       420\n",
      "                  food       0.75      0.64      0.69       706\n",
      "               shelter       0.83      0.31      0.45       579\n",
      "              clothing       0.72      0.26      0.38       107\n",
      "                 money       0.57      0.15      0.24       157\n",
      "        missing_people       0.78      0.35      0.49        71\n",
      "              refugees       0.40      0.01      0.02       206\n",
      "                 death       0.75      0.01      0.02       314\n",
      "             other_aid       0.00      0.00      0.00       814\n",
      "infrastructure_related       0.00      0.00      0.00       453\n",
      "             transport       0.59      0.26      0.36       288\n",
      "             buildings       0.66      0.09      0.16       341\n",
      "           electricity       0.00      0.00      0.00       119\n",
      "                 tools       0.00      0.00      0.00        36\n",
      "             hospitals       0.00      0.00      0.00        74\n",
      "                 shops       0.00      0.00      0.00        33\n",
      "           aid_centers       0.00      0.00      0.00        91\n",
      "  other_infrastructure       0.00      0.00      0.00       290\n",
      "       weather_related       0.92      0.22      0.35      1768\n",
      "                floods       0.90      0.34      0.49       544\n",
      "                 storm       0.76      0.28      0.41       597\n",
      "                  fire       0.47      0.34      0.40        67\n",
      "            earthquake       0.90      0.65      0.76       584\n",
      "                  cold       0.72      0.19      0.31       134\n",
      "         other_weather       0.52      0.10      0.17       349\n",
      "         direct_report       0.62      0.39      0.48      1208\n",
      "\n",
      "           avg / total       0.66      0.43      0.46     20485\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "# Print classification report on test data\n",
    "print(classification_report(y_test.values, y_prediction_test, target_names=y.columns.values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "                         precision    recall  f1-score   support\n",
      "\n",
      "               related       0.77      1.00      0.87     15120\n",
      "               request       0.61      0.39      0.47      3364\n",
      "                 offer       0.00      0.00      0.00        91\n",
      "           aid_related       0.81      0.19      0.31      8173\n",
      "          medical_help       0.65      0.12      0.20      1565\n",
      "      medical_products       0.76      0.12      0.21      1024\n",
      "     search_and_rescue       0.62      0.21      0.31       535\n",
      "              security       0.00      0.00      0.00       353\n",
      "              military       0.51      0.13      0.21       638\n",
      "                 water       0.58      0.85      0.69      1252\n",
      "                  food       0.78      0.69      0.73      2217\n",
      "               shelter       0.85      0.29      0.43      1735\n",
      "              clothing       0.75      0.34      0.47       298\n",
      "                 money       0.55      0.16      0.24       447\n",
      "        missing_people       0.63      0.23      0.34       227\n",
      "              refugees       0.85      0.02      0.03       669\n",
      "                 death       0.75      0.01      0.01       880\n",
      "             other_aid       0.00      0.00      0.00      2632\n",
      "infrastructure_related       0.00      0.00      0.00      1252\n",
      "             transport       0.58      0.25      0.35       913\n",
      "             buildings       0.84      0.09      0.17       992\n",
      "           electricity       0.00      0.00      0.00       413\n",
      "                 tools       0.00      0.00      0.00       123\n",
      "             hospitals       0.00      0.00      0.00       209\n",
      "                 shops       1.00      0.01      0.02        87\n",
      "           aid_centers       0.00      0.00      0.00       218\n",
      "  other_infrastructure       0.00      0.00      0.00       861\n",
      "       weather_related       0.91      0.22      0.36      5529\n",
      "                floods       0.92      0.34      0.49      1611\n",
      "                 storm       0.73      0.25      0.37      1846\n",
      "                  fire       0.53      0.42      0.47       215\n",
      "            earthquake       0.89      0.64      0.75      1871\n",
      "                  cold       0.68      0.23      0.34       396\n",
      "         other_weather       0.59      0.13      0.21      1027\n",
      "         direct_report       0.64      0.39      0.48      3867\n",
      "\n",
      "           avg / total       0.69      0.44      0.47     62650\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "# Print classification report on training data\n",
    "print('\\n',classification_report(y_train.values, y_prediction_train, target_names=y.columns.values))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. Try improving your model further. Here are a few ideas:\n",
    "* try other machine learning algorithms\n",
    "* add other features besides the TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                        precision    recall  f1-score   support\n",
      "\n",
      "               related       0.83      0.94      0.88      4998\n",
      "               request       0.77      0.56      0.65      1102\n",
      "                 offer       0.00      0.00      0.00        30\n",
      "           aid_related       0.76      0.59      0.67      2686\n",
      "          medical_help       0.65      0.27      0.39       522\n",
      "      medical_products       0.71      0.28      0.40       360\n",
      "     search_and_rescue       0.62      0.18      0.28       176\n",
      "              security       0.22      0.04      0.06       114\n",
      "              military       0.61      0.28      0.38       213\n",
      "                 water       0.77      0.63      0.69       434\n",
      "                  food       0.80      0.69      0.74       734\n",
      "               shelter       0.81      0.54      0.65       574\n",
      "              clothing       0.66      0.38      0.48        98\n",
      "                 money       0.62      0.32      0.42       157\n",
      "        missing_people       0.65      0.16      0.25        70\n",
      "              refugees       0.54      0.22      0.31       212\n",
      "                 death       0.71      0.44      0.54       280\n",
      "             other_aid       0.48      0.15      0.23       838\n",
      "infrastructure_related       0.38      0.06      0.11       432\n",
      "             transport       0.71      0.22      0.34       288\n",
      "             buildings       0.65      0.38      0.47       312\n",
      "           electricity       0.52      0.28      0.36       127\n",
      "                 tools       0.00      0.00      0.00        41\n",
      "             hospitals       0.25      0.09      0.13        70\n",
      "                 shops       0.00      0.00      0.00        33\n",
      "           aid_centers       0.43      0.14      0.22        70\n",
      "  other_infrastructure       0.35      0.09      0.14       293\n",
      "       weather_related       0.85      0.65      0.74      1826\n",
      "                floods       0.86      0.57      0.69       526\n",
      "                 storm       0.77      0.52      0.62       606\n",
      "                  fire       0.47      0.24      0.31        68\n",
      "            earthquake       0.88      0.76      0.82       612\n",
      "                  cold       0.66      0.32      0.43       123\n",
      "         other_weather       0.45      0.15      0.22       339\n",
      "         direct_report       0.72      0.50      0.59      1241\n",
      "\n",
      "           avg / total       0.74      0.58      0.63     20605\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#cv.best_params_\n",
    "\n",
    "#Use pipeline2 which includes StartingVerbEstimator\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "pipeline_fitted = pipeline2.fit(X_train, y_train)\n",
    "\n",
    "y_prediction_train = pipeline_fitted.predict(X_train)\n",
    "y_prediction_test = pipeline_fitted.predict(X_test)\n",
    "\n",
    "# Print classification report on test data\n",
    "print(classification_report(y_test.values, y_prediction_test, target_names=y.columns.values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "                         precision    recall  f1-score   support\n",
      "\n",
      "               related       0.83      0.95      0.88     15096\n",
      "               request       0.80      0.57      0.66      3372\n",
      "                 offer       0.39      0.10      0.16        88\n",
      "           aid_related       0.77      0.60      0.68      8174\n",
      "          medical_help       0.65      0.28      0.39      1562\n",
      "      medical_products       0.70      0.36      0.48       953\n",
      "     search_and_rescue       0.67      0.18      0.29       548\n",
      "              security       0.42      0.07      0.12       357\n",
      "              military       0.68      0.36      0.47       647\n",
      "                 water       0.78      0.67      0.72      1238\n",
      "                  food       0.82      0.72      0.76      2189\n",
      "               shelter       0.79      0.55      0.65      1740\n",
      "              clothing       0.77      0.48      0.59       307\n",
      "                 money       0.63      0.35      0.45       447\n",
      "        missing_people       0.72      0.25      0.38       228\n",
      "              refugees       0.65      0.29      0.40       663\n",
      "                 death       0.79      0.48      0.60       914\n",
      "             other_aid       0.58      0.17      0.26      2608\n",
      "infrastructure_related       0.53      0.11      0.18      1273\n",
      "             transport       0.78      0.26      0.39       913\n",
      "             buildings       0.72      0.40      0.51      1021\n",
      "           electricity       0.68      0.34      0.45       405\n",
      "                 tools       0.42      0.08      0.14       118\n",
      "             hospitals       0.40      0.12      0.18       213\n",
      "                 shops       0.56      0.06      0.10        87\n",
      "           aid_centers       0.49      0.14      0.22       239\n",
      "  other_infrastructure       0.47      0.12      0.19       858\n",
      "       weather_related       0.86      0.66      0.75      5471\n",
      "                floods       0.88      0.57      0.69      1629\n",
      "                 storm       0.78      0.54      0.64      1837\n",
      "                  fire       0.76      0.32      0.45       214\n",
      "            earthquake       0.89      0.78      0.83      1843\n",
      "                  cold       0.77      0.39      0.52       407\n",
      "         other_weather       0.57      0.18      0.28      1037\n",
      "         direct_report       0.75      0.48      0.59      3834\n",
      "\n",
      "           avg / total       0.77      0.60      0.65     62530\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print classification report on training data\n",
    "print('\\n',classification_report(y_train.values, y_prediction_train, target_names=y.columns.values))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9. Export your model as a pickle file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(cv, open('model.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 10. Use this notebook to complete `train.py`\n",
    "Use the template file attached in the Resources folder to write a script that runs the steps above to create a database and export a model based on a new dataset specified by the user."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}