mnist.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "mnist.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/roshank1605A04/MNIST-Dataset/blob/master/mnist.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "7A3cH5retzUY",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Installing the Libraries**"
      ]
    },
    {
      "metadata": {
        "id": "_iRN0EcmtqoQ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "#import seaborn as sns"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "4IAKoMvcuEFK",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Importing the Dataset**"
      ]
    },
    {
      "metadata": {
        "id": "G1awFgYLuDYh",
        "colab_type": "code",
        "outputId": "39031914-5cf8-4a7e-fc91-244bd17fa80d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 292
        }
      },
      "cell_type": "code",
      "source": [
        "train = pd.read_csv('drive/My Drive/Projects/mnist text/mnist_train.csv')\n",
        "test = pd.read_csv('drive/My Drive/Projects/mnist text/mnist_test.csv')\n",
        "\n",
        "print(train.shape)\n",
        "print(test.shape)\n",
        "\n",
        "train.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(60000, 785)\n",
            "(10000, 785)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>1x1</th>\n",
              "      <th>1x2</th>\n",
              "      <th>1x3</th>\n",
              "      <th>1x4</th>\n",
              "      <th>1x5</th>\n",
              "      <th>1x6</th>\n",
              "      <th>1x7</th>\n",
              "      <th>1x8</th>\n",
              "      <th>1x9</th>\n",
              "      <th>...</th>\n",
              "      <th>28x19</th>\n",
              "      <th>28x20</th>\n",
              "      <th>28x21</th>\n",
              "      <th>28x22</th>\n",
              "      <th>28x23</th>\n",
              "      <th>28x24</th>\n",
              "      <th>28x25</th>\n",
              "      <th>28x26</th>\n",
              "      <th>28x27</th>\n",
              "      <th>28x28</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 785 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "   label  1x1  1x2  1x3  1x4  1x5  1x6  1x7  1x8  1x9  ...    28x19  28x20  \\\n",
              "0      5    0    0    0    0    0    0    0    0    0  ...        0      0   \n",
              "1      0    0    0    0    0    0    0    0    0    0  ...        0      0   \n",
              "2      4    0    0    0    0    0    0    0    0    0  ...        0      0   \n",
              "3      1    0    0    0    0    0    0    0    0    0  ...        0      0   \n",
              "4      9    0    0    0    0    0    0    0    0    0  ...        0      0   \n",
              "\n",
              "   28x21  28x22  28x23  28x24  28x25  28x26  28x27  28x28  \n",
              "0      0      0      0      0      0      0      0      0  \n",
              "1      0      0      0      0      0      0      0      0  \n",
              "2      0      0      0      0      0      0      0      0  \n",
              "3      0      0      0      0      0      0      0      0  \n",
              "4      0      0      0      0      0      0      0      0  \n",
              "\n",
              "[5 rows x 785 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "metadata": {
        "id": "-Ta2GOItugd0",
        "colab_type": "code",
        "outputId": "b6385891-c302-4605-87f5-92b23caec027",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        }
      },
      "cell_type": "code",
      "source": [
        "test.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>1x1</th>\n",
              "      <th>1x2</th>\n",
              "      <th>1x3</th>\n",
              "      <th>1x4</th>\n",
              "      <th>1x5</th>\n",
              "      <th>1x6</th>\n",
              "      <th>1x7</th>\n",
              "      <th>1x8</th>\n",
              "      <th>1x9</th>\n",
              "      <th>...</th>\n",
              "      <th>28x19</th>\n",
              "      <th>28x20</th>\n",
              "      <th>28x21</th>\n",
              "      <th>28x22</th>\n",
              "      <th>28x23</th>\n",
              "      <th>28x24</th>\n",
              "      <th>28x25</th>\n",
              "      <th>28x26</th>\n",
              "      <th>28x27</th>\n",
              "      <th>28x28</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>7</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 785 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "   label  1x1  1x2  1x3  1x4  1x5  1x6  1x7  1x8  1x9  ...    28x19  28x20  \\\n",
              "0      7    0    0    0    0    0    0    0    0    0  ...        0      0   \n",
              "1      2    0    0    0    0    0    0    0    0    0  ...        0      0   \n",
              "2      1    0    0    0    0    0    0    0    0    0  ...        0      0   \n",
              "3      0    0    0    0    0    0    0    0    0    0  ...        0      0   \n",
              "4      4    0    0    0    0    0    0    0    0    0  ...        0      0   \n",
              "\n",
              "   28x21  28x22  28x23  28x24  28x25  28x26  28x27  28x28  \n",
              "0      0      0      0      0      0      0      0      0  \n",
              "1      0      0      0      0      0      0      0      0  \n",
              "2      0      0      0      0      0      0      0      0  \n",
              "3      0      0      0      0      0      0      0      0  \n",
              "4      0      0      0      0      0      0      0      0  \n",
              "\n",
              "[5 rows x 785 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "metadata": {
        "id": "c9HeBJqqw68G",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Data Preprocessing**"
      ]
    },
    {
      "metadata": {
        "id": "PM3tDmvbxy2R",
        "colab_type": "code",
        "outputId": "75ddbf00-1c40-4236-f532-271361b5d740",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 217
        }
      },
      "cell_type": "code",
      "source": [
        "# It has 10 digits ranging from 0-9\n",
        "\n",
        "train.iloc[:,0].value_counts()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1    6742\n",
              "7    6265\n",
              "3    6131\n",
              "2    5958\n",
              "9    5949\n",
              "0    5923\n",
              "6    5918\n",
              "8    5851\n",
              "4    5842\n",
              "5    5421\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "metadata": {
        "id": "6mrI4MbSverx",
        "colab_type": "code",
        "outputId": "c81b2698-38ea-4f24-ef54-744c4de3086f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        }
      },
      "cell_type": "code",
      "source": [
        "# splitting the data into dependent and independent variables\n",
        "\n",
        "train_x = train.iloc[:,1:785]\n",
        "train_y = train.iloc[:,0]\n",
        "\n",
        "test_x = test.iloc[:,1:785]\n",
        "test_y = test.iloc[:,0]\n",
        "\n",
        "print(train_x.shape)\n",
        "print(train_y.shape)\n",
        "\n",
        "print(test_x.shape)\n",
        "print(test_y.shape)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(60000, 784)\n",
            "(60000,)\n",
            "(10000, 784)\n",
            "(10000,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "c0lehAFNzNIx",
        "colab_type": "code",
        "outputId": "971db706-1b1f-4211-87ed-e9896d922149",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        }
      },
      "cell_type": "code",
      "source": [
        "# splitting the dataset into training and testing set\n",
        "\n",
        "from sklearn.cross_validation import train_test_split\n",
        "\n",
        "x_train, x_cv, y_train, y_cv = train_test_split(train_x, train_y, test_size = 0.25, random_state = 35)\n",
        "\n",
        "print(x_train.shape)\n",
        "print(x_cv.shape)\n",
        "print(y_train.shape)\n",
        "print(y_cv.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(45000, 784)\n",
            "(15000, 784)\n",
            "(45000,)\n",
            "(15000,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "6WX5OKC0vE_D",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# reshaping them as matrix\n",
        "\n",
        "x_train = np.asmatrix(x_train).reshape(45000, 784)\n",
        "x_cv = np.asmatrix(x_cv).reshape(15000, 784)\n",
        "\n",
        "test_x = np.asmatrix(test_x).reshape(10000, 784)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "4ScaPgmM69j5",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# feature normalization\n",
        "\n",
        "x_train = x_train.astype('float32')\n",
        "x_cv = x_cv.astype('float32')\n",
        "\n",
        "test_x = test_x.astype('float32')\n",
        "\n",
        "x_train = x_train/255\n",
        "x_cv = x_cv/255\n",
        "test_x = test_x/255"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "40oNAvWe7uf7",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# converting the labels into one hot encoding\n",
        "\n",
        "# immporting keras \n",
        "import keras\n",
        "\n",
        "digits = 10\n",
        "y_train = keras.utils.to_categorical(y_train, digits)\n",
        "y_cv = keras.utils.to_categorical(y_cv, digits)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "dRIy4JNX8Xu8",
        "colab_type": "code",
        "outputId": "698e5017-1281-47ea-956e-7903d6648e22",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "cell_type": "code",
      "source": [
        "# viewing the labels after one hot encoding\n",
        "\n",
        "print(y_train[0])  #7\n",
        "print(y_train[3])  #2\n",
        "print(y_train[4])  #4"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]\n",
            "[0. 1. 0. 0. 0. 0. 0. 0. 0. 0.]\n",
            "[0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "QuabCzxyzYVm",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Modelling**"
      ]
    },
    {
      "metadata": {
        "id": "rNxCox5R7V3m",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Artificial Neural Networks**"
      ]
    },
    {
      "metadata": {
        "id": "Bj-xnYHj8-Py",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import keras \n",
        "from keras.layers import Dense \n",
        "from keras.models import Sequential\n",
        "from keras import optimizers"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7d3rc1Hl0Gju",
        "colab_type": "code",
        "outputId": "bfe256e2-dcea-4e01-fc72-121e83db948a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "cell_type": "code",
      "source": [
        "# creating the model\n",
        "model = Sequential()\n",
        "\n",
        "# first hidden layer\n",
        "model.add(Dense(output_dim = 400, init = 'uniform', activation = 'relu', input_dim = 784))\n",
        "\n",
        "# second hidden layer\n",
        "model.add(Dense(output_dim = 300, init = 'uniform', activation = 'relu'))\n",
        "\n",
        "# third hidden layer\n",
        "model.add(Dense(output_dim = 300, init = 'uniform', activation = 'relu'))\n",
        "\n",
        "# fourth hidden layer\n",
        "model.add(Dense(output_dim = 300, init = 'uniform', activation = 'relu'))\n",
        "\n",
        "# fifth hidden layer\n",
        "model.add(Dense(output_dim = 100, init = 'uniform', activation = 'relu'))\n",
        "\n",
        "# output layer\n",
        "# output_dim = no. of digits\n",
        "# softmax activation function is used for multiple outputs\n",
        "model.add(Dense(output_dim = 10, init = 'uniform', activation = 'softmax'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(activation=\"relu\", input_dim=784, units=400, kernel_initializer=\"uniform\")`\n",
            "  after removing the cwd from sys.path.\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(activation=\"relu\", units=300, kernel_initializer=\"uniform\")`\n",
            "  import sys\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:10: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(activation=\"relu\", units=300, kernel_initializer=\"uniform\")`\n",
            "  # Remove the CWD from sys.path while we load stuff.\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:13: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(activation=\"relu\", units=300, kernel_initializer=\"uniform\")`\n",
            "  del sys.path[0]\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:16: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(activation=\"relu\", units=100, kernel_initializer=\"uniform\")`\n",
            "  app.launch_new_instance()\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:21: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(activation=\"softmax\", units=10, kernel_initializer=\"uniform\")`\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "FEID2ZaM15Vu",
        "colab_type": "code",
        "outputId": "b9e5af83-e397-46b2-c9da-1659b648dd47",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        }
      },
      "cell_type": "code",
      "source": [
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "dense_1 (Dense)              (None, 400)               314000    \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 300)               120300    \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 300)               90300     \n",
            "_________________________________________________________________\n",
            "dense_4 (Dense)              (None, 300)               90300     \n",
            "_________________________________________________________________\n",
            "dense_5 (Dense)              (None, 100)               30100     \n",
            "_________________________________________________________________\n",
            "dense_6 (Dense)              (None, 10)                1010      \n",
            "=================================================================\n",
            "Total params: 646,010\n",
            "Trainable params: 646,010\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "GJfP0frC8bEH",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Using Stochastic gradient descent as our Optimizer**"
      ]
    },
    {
      "metadata": {
        "id": "z1hPTfvR2ap3",
        "colab_type": "code",
        "outputId": "42aa4f72-9bd8-47bf-a3c0-a4035b9d5e48",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 3693
        }
      },
      "cell_type": "code",
      "source": [
        "# setting the learning rate\n",
        "learning_rate = 0.01\n",
        "sgd = optimizers.SGD(lr = learning_rate)\n",
        "\n",
        "\n",
        "# compiling the model\n",
        "# using the plain vanilla stochastic gradient descent as our optimizing technique\n",
        "# using categorical cross entropy for multiple outputs\n",
        "model.compile(optimizer = 'sgd', loss = 'categorical_crossentropy', metrics = ['accuracy'])\n",
        "\n",
        "# feeding the training data to the model\n",
        "model.fit(x_train, y_train, batch_size = 100, epochs = 60, verbose = 2, validation_data = (x_cv, y_cv))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 45000 samples, validate on 15000 samples\n",
            "Epoch 1/100\n",
            " - 7s - loss: 2.2999 - acc: 0.1122 - val_loss: 2.2994 - val_acc: 0.1129\n",
            "Epoch 2/100\n",
            " - 7s - loss: 2.2992 - acc: 0.1122 - val_loss: 2.2986 - val_acc: 0.1129\n",
            "Epoch 3/100\n",
            " - 7s - loss: 2.2982 - acc: 0.1122 - val_loss: 2.2974 - val_acc: 0.1129\n",
            "Epoch 4/100\n",
            " - 7s - loss: 2.2966 - acc: 0.1122 - val_loss: 2.2952 - val_acc: 0.1129\n",
            "Epoch 5/100\n",
            " - 7s - loss: 2.2934 - acc: 0.1125 - val_loss: 2.2906 - val_acc: 0.1170\n",
            "Epoch 6/100\n",
            " - 7s - loss: 2.2852 - acc: 0.1694 - val_loss: 2.2761 - val_acc: 0.2090\n",
            "Epoch 7/100\n",
            " - 7s - loss: 2.2451 - acc: 0.2142 - val_loss: 2.1825 - val_acc: 0.2109\n",
            "Epoch 8/100\n",
            " - 7s - loss: 2.0459 - acc: 0.2308 - val_loss: 1.8748 - val_acc: 0.2833\n",
            "Epoch 9/100\n",
            " - 7s - loss: 1.6617 - acc: 0.3657 - val_loss: 1.4902 - val_acc: 0.4318\n",
            "Epoch 10/100\n",
            " - 7s - loss: 1.4199 - acc: 0.4646 - val_loss: 1.3250 - val_acc: 0.5090\n",
            "Epoch 11/100\n",
            " - 7s - loss: 1.1910 - acc: 0.5924 - val_loss: 0.9512 - val_acc: 0.7245\n",
            "Epoch 12/100\n",
            " - 7s - loss: 0.8321 - acc: 0.7368 - val_loss: 0.7171 - val_acc: 0.7835\n",
            "Epoch 13/100\n",
            " - 7s - loss: 0.6671 - acc: 0.7922 - val_loss: 0.6027 - val_acc: 0.8121\n",
            "Epoch 14/100\n",
            " - 7s - loss: 0.5075 - acc: 0.8478 - val_loss: 0.4418 - val_acc: 0.8728\n",
            "Epoch 15/100\n",
            " - 7s - loss: 0.4133 - acc: 0.8809 - val_loss: 0.3852 - val_acc: 0.8929\n",
            "Epoch 16/100\n",
            " - 7s - loss: 0.3631 - acc: 0.8971 - val_loss: 0.3467 - val_acc: 0.9037\n",
            "Epoch 17/100\n",
            " - 7s - loss: 0.3200 - acc: 0.9106 - val_loss: 0.3289 - val_acc: 0.9079\n",
            "Epoch 18/100\n",
            " - 7s - loss: 0.2848 - acc: 0.9214 - val_loss: 0.2988 - val_acc: 0.9160\n",
            "Epoch 19/100\n",
            " - 7s - loss: 0.2546 - acc: 0.9288 - val_loss: 0.2640 - val_acc: 0.9278\n",
            "Epoch 20/100\n",
            " - 7s - loss: 0.2307 - acc: 0.9358 - val_loss: 0.2721 - val_acc: 0.9215\n",
            "Epoch 21/100\n",
            " - 7s - loss: 0.2054 - acc: 0.9432 - val_loss: 0.2286 - val_acc: 0.9370\n",
            "Epoch 22/100\n",
            " - 7s - loss: 0.1859 - acc: 0.9480 - val_loss: 0.2250 - val_acc: 0.9381\n",
            "Epoch 23/100\n",
            " - 7s - loss: 0.1708 - acc: 0.9525 - val_loss: 0.2049 - val_acc: 0.9421\n",
            "Epoch 24/100\n",
            " - 7s - loss: 0.1557 - acc: 0.9570 - val_loss: 0.2037 - val_acc: 0.9449\n",
            "Epoch 25/100\n",
            " - 7s - loss: 0.1434 - acc: 0.9598 - val_loss: 0.1916 - val_acc: 0.9477\n",
            "Epoch 26/100\n",
            " - 7s - loss: 0.1320 - acc: 0.9638 - val_loss: 0.1910 - val_acc: 0.9463\n",
            "Epoch 27/100\n",
            " - 7s - loss: 0.1212 - acc: 0.9658 - val_loss: 0.1781 - val_acc: 0.9516\n",
            "Epoch 28/100\n",
            " - 7s - loss: 0.1124 - acc: 0.9689 - val_loss: 0.1686 - val_acc: 0.9535\n",
            "Epoch 29/100\n",
            " - 7s - loss: 0.1037 - acc: 0.9711 - val_loss: 0.1646 - val_acc: 0.9550\n",
            "Epoch 30/100\n",
            " - 7s - loss: 0.0960 - acc: 0.9740 - val_loss: 0.1623 - val_acc: 0.9557\n",
            "Epoch 31/100\n",
            " - 7s - loss: 0.0888 - acc: 0.9752 - val_loss: 0.1661 - val_acc: 0.9557\n",
            "Epoch 32/100\n",
            " - 7s - loss: 0.0810 - acc: 0.9774 - val_loss: 0.1713 - val_acc: 0.9537\n",
            "Epoch 33/100\n",
            " - 7s - loss: 0.0755 - acc: 0.9792 - val_loss: 0.1697 - val_acc: 0.9544\n",
            "Epoch 34/100\n",
            " - 7s - loss: 0.0700 - acc: 0.9806 - val_loss: 0.1574 - val_acc: 0.9589\n",
            "Epoch 35/100\n",
            " - 7s - loss: 0.0641 - acc: 0.9830 - val_loss: 0.1751 - val_acc: 0.9545\n",
            "Epoch 36/100\n",
            " - 7s - loss: 0.0589 - acc: 0.9844 - val_loss: 0.1643 - val_acc: 0.9561\n",
            "Epoch 37/100\n",
            " - 7s - loss: 0.0550 - acc: 0.9851 - val_loss: 0.1699 - val_acc: 0.9567\n",
            "Epoch 38/100\n",
            " - 7s - loss: 0.0505 - acc: 0.9867 - val_loss: 0.1593 - val_acc: 0.9590\n",
            "Epoch 39/100\n",
            " - 7s - loss: 0.0446 - acc: 0.9885 - val_loss: 0.1669 - val_acc: 0.9577\n",
            "Epoch 40/100\n",
            " - 7s - loss: 0.0431 - acc: 0.9886 - val_loss: 0.1652 - val_acc: 0.9596\n",
            "Epoch 41/100\n",
            " - 7s - loss: 0.0379 - acc: 0.9908 - val_loss: 0.1630 - val_acc: 0.9601\n",
            "Epoch 42/100\n",
            " - 7s - loss: 0.0346 - acc: 0.9918 - val_loss: 0.1609 - val_acc: 0.9598\n",
            "Epoch 43/100\n",
            " - 7s - loss: 0.0303 - acc: 0.9928 - val_loss: 0.1834 - val_acc: 0.9568\n",
            "Epoch 44/100\n",
            " - 7s - loss: 0.0283 - acc: 0.9936 - val_loss: 0.1850 - val_acc: 0.9564\n",
            "Epoch 45/100\n",
            " - 7s - loss: 0.0261 - acc: 0.9942 - val_loss: 0.1663 - val_acc: 0.9604\n",
            "Epoch 46/100\n",
            " - 7s - loss: 0.2927 - acc: 0.9672 - val_loss: 0.1604 - val_acc: 0.9599\n",
            "Epoch 47/100\n",
            " - 7s - loss: 0.0251 - acc: 0.9949 - val_loss: 0.1690 - val_acc: 0.9599\n",
            "Epoch 48/100\n",
            " - 7s - loss: 0.0203 - acc: 0.9962 - val_loss: 0.1700 - val_acc: 0.9600\n",
            "Epoch 49/100\n",
            " - 7s - loss: 0.0179 - acc: 0.9970 - val_loss: 0.1679 - val_acc: 0.9621\n",
            "Epoch 50/100\n",
            " - 7s - loss: 0.0153 - acc: 0.9977 - val_loss: 0.1869 - val_acc: 0.9580\n",
            "Epoch 51/100\n",
            " - 7s - loss: 0.0139 - acc: 0.9980 - val_loss: 0.1726 - val_acc: 0.9613\n",
            "Epoch 52/100\n",
            " - 7s - loss: 0.0126 - acc: 0.9980 - val_loss: 0.1768 - val_acc: 0.9619\n",
            "Epoch 53/100\n",
            " - 7s - loss: 0.0110 - acc: 0.9984 - val_loss: 0.1780 - val_acc: 0.9615\n",
            "Epoch 54/100\n",
            " - 7s - loss: 0.0097 - acc: 0.9989 - val_loss: 0.1817 - val_acc: 0.9617\n",
            "Epoch 55/100\n",
            " - 7s - loss: 0.0088 - acc: 0.9991 - val_loss: 0.1824 - val_acc: 0.9617\n",
            "Epoch 56/100\n",
            " - 7s - loss: 0.0082 - acc: 0.9992 - val_loss: 0.1823 - val_acc: 0.9617\n",
            "Epoch 57/100\n",
            " - 7s - loss: 0.0073 - acc: 0.9995 - val_loss: 0.1904 - val_acc: 0.9611\n",
            "Epoch 58/100\n",
            " - 7s - loss: 0.0066 - acc: 0.9995 - val_loss: 0.1868 - val_acc: 0.9621\n",
            "Epoch 59/100\n",
            " - 7s - loss: 0.0060 - acc: 0.9997 - val_loss: 0.1876 - val_acc: 0.9611\n",
            "Epoch 60/100\n",
            " - 7s - loss: 0.0054 - acc: 0.9997 - val_loss: 0.1904 - val_acc: 0.9623\n",
            "Epoch 61/100\n",
            " - 7s - loss: 0.0051 - acc: 0.9998 - val_loss: 0.1894 - val_acc: 0.9627\n",
            "Epoch 62/100\n",
            " - 7s - loss: 0.0047 - acc: 0.9998 - val_loss: 0.1918 - val_acc: 0.9624\n",
            "Epoch 63/100\n",
            " - 7s - loss: 0.0045 - acc: 0.9998 - val_loss: 0.1941 - val_acc: 0.9619\n",
            "Epoch 64/100\n",
            " - 7s - loss: 0.0042 - acc: 0.9998 - val_loss: 0.1963 - val_acc: 0.9630\n",
            "Epoch 65/100\n",
            " - 7s - loss: 0.0039 - acc: 0.9998 - val_loss: 0.1965 - val_acc: 0.9621\n",
            "Epoch 66/100\n",
            " - 7s - loss: 0.0037 - acc: 0.9999 - val_loss: 0.1971 - val_acc: 0.9627\n",
            "Epoch 67/100\n",
            " - 7s - loss: 0.0035 - acc: 0.9999 - val_loss: 0.1986 - val_acc: 0.9626\n",
            "Epoch 68/100\n",
            " - 7s - loss: 0.0034 - acc: 0.9999 - val_loss: 0.2048 - val_acc: 0.9610\n",
            "Epoch 69/100\n",
            " - 7s - loss: 0.0032 - acc: 0.9999 - val_loss: 0.2013 - val_acc: 0.9615\n",
            "Epoch 70/100\n",
            " - 7s - loss: 0.0031 - acc: 0.9999 - val_loss: 0.2031 - val_acc: 0.9623\n",
            "Epoch 71/100\n",
            " - 7s - loss: 0.0030 - acc: 0.9999 - val_loss: 0.2039 - val_acc: 0.9623\n",
            "Epoch 72/100\n",
            " - 7s - loss: 0.0029 - acc: 0.9999 - val_loss: 0.2054 - val_acc: 0.9621\n",
            "Epoch 73/100\n",
            " - 7s - loss: 0.0028 - acc: 0.9999 - val_loss: 0.2054 - val_acc: 0.9623\n",
            "Epoch 74/100\n",
            " - 7s - loss: 0.0027 - acc: 0.9999 - val_loss: 0.2073 - val_acc: 0.9623\n",
            "Epoch 75/100\n",
            " - 7s - loss: 0.0026 - acc: 0.9999 - val_loss: 0.2076 - val_acc: 0.9621\n",
            "Epoch 76/100\n",
            " - 7s - loss: 0.0026 - acc: 0.9999 - val_loss: 0.2086 - val_acc: 0.9619\n",
            "Epoch 77/100\n",
            " - 7s - loss: 0.0025 - acc: 0.9999 - val_loss: 0.2112 - val_acc: 0.9621\n",
            "Epoch 78/100\n",
            " - 7s - loss: 0.0024 - acc: 0.9999 - val_loss: 0.2108 - val_acc: 0.9621\n",
            "Epoch 79/100\n",
            " - 7s - loss: 0.0024 - acc: 0.9999 - val_loss: 0.2108 - val_acc: 0.9621\n",
            "Epoch 80/100\n",
            " - 7s - loss: 0.0023 - acc: 0.9999 - val_loss: 0.2123 - val_acc: 0.9621\n",
            "Epoch 81/100\n",
            " - 7s - loss: 0.0023 - acc: 0.9999 - val_loss: 0.2127 - val_acc: 0.9623\n",
            "Epoch 82/100\n",
            " - 7s - loss: 0.0022 - acc: 0.9999 - val_loss: 0.2125 - val_acc: 0.9624\n",
            "Epoch 83/100\n",
            " - 7s - loss: 0.0022 - acc: 0.9999 - val_loss: 0.2138 - val_acc: 0.9618\n",
            "Epoch 84/100\n",
            " - 7s - loss: 0.0021 - acc: 0.9999 - val_loss: 0.2153 - val_acc: 0.9620\n",
            "Epoch 85/100\n",
            " - 7s - loss: 0.0021 - acc: 0.9999 - val_loss: 0.2145 - val_acc: 0.9623\n",
            "Epoch 86/100\n",
            " - 7s - loss: 0.0021 - acc: 0.9999 - val_loss: 0.2165 - val_acc: 0.9617\n",
            "Epoch 87/100\n",
            " - 7s - loss: 0.0020 - acc: 0.9999 - val_loss: 0.2168 - val_acc: 0.9621\n",
            "Epoch 88/100\n",
            " - 7s - loss: 0.0020 - acc: 0.9999 - val_loss: 0.2193 - val_acc: 0.9621\n",
            "Epoch 89/100\n",
            " - 7s - loss: 0.0020 - acc: 0.9999 - val_loss: 0.2191 - val_acc: 0.9617\n",
            "Epoch 90/100\n",
            " - 7s - loss: 0.0019 - acc: 0.9999 - val_loss: 0.2190 - val_acc: 0.9620\n",
            "Epoch 91/100\n",
            " - 7s - loss: 0.0019 - acc: 0.9999 - val_loss: 0.2196 - val_acc: 0.9626\n",
            "Epoch 92/100\n",
            " - 7s - loss: 0.0019 - acc: 0.9999 - val_loss: 0.2192 - val_acc: 0.9621\n",
            "Epoch 93/100\n",
            " - 7s - loss: 0.0019 - acc: 0.9999 - val_loss: 0.2205 - val_acc: 0.9623\n",
            "Epoch 94/100\n",
            " - 7s - loss: 0.0018 - acc: 0.9999 - val_loss: 0.2213 - val_acc: 0.9624\n",
            "Epoch 95/100\n",
            " - 7s - loss: 0.0018 - acc: 0.9999 - val_loss: 0.2214 - val_acc: 0.9625\n",
            "Epoch 96/100\n",
            " - 7s - loss: 0.0018 - acc: 0.9999 - val_loss: 0.2226 - val_acc: 0.9625\n",
            "Epoch 97/100\n",
            " - 7s - loss: 0.0018 - acc: 0.9999 - val_loss: 0.2231 - val_acc: 0.9621\n",
            "Epoch 98/100\n",
            " - 7s - loss: 0.0018 - acc: 0.9999 - val_loss: 0.2230 - val_acc: 0.9623\n",
            "Epoch 99/100\n",
            " - 7s - loss: 0.0017 - acc: 0.9999 - val_loss: 0.2240 - val_acc: 0.9623\n",
            "Epoch 100/100\n",
            " - 7s - loss: 0.0017 - acc: 0.9999 - val_loss: 0.2238 - val_acc: 0.9622\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f5435e39fd0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 58
        }
      ]
    },
    {
      "metadata": {
        "id": "PitBVei28Ql0",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Using Adam Optimizer**"
      ]
    },
    {
      "metadata": {
        "id": "yt7jiwPI7Oys",
        "colab_type": "code",
        "outputId": "b89ddf7e-037a-4e2e-da39-5f7604dc16a9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 237
        }
      },
      "cell_type": "code",
      "source": [
        "# creating the model\n",
        "model = Sequential()\n",
        "\n",
        "# first hidden layer\n",
        "model.add(Dense(output_dim = 400, init = 'uniform', activation = 'relu', input_dim = 784))\n",
        "\n",
        "# second hidden layer\n",
        "model.add(Dense(output_dim = 300, init = 'uniform', activation = 'relu'))\n",
        "\n",
        "# third hidden layer\n",
        "model.add(Dense(output_dim = 300, init = 'uniform', activation = 'relu'))\n",
        "\n",
        "# fourth hidden layer\n",
        "model.add(Dense(output_dim = 300, init = 'uniform', activation = 'relu'))\n",
        "\n",
        "# fifth hidden layer\n",
        "model.add(Dense(output_dim = 100, init = 'uniform', activation = 'relu'))\n",
        "\n",
        "# output layer\n",
        "# output_dim = no. of digits\n",
        "# softmax activation function is used for multiple outputs\n",
        "model.add(Dense(output_dim = 10, init = 'uniform', activation = 'softmax'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(activation=\"relu\", input_dim=784, units=400, kernel_initializer=\"uniform\")`\n",
            "  after removing the cwd from sys.path.\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(activation=\"relu\", units=300, kernel_initializer=\"uniform\")`\n",
            "  import sys\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:10: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(activation=\"relu\", units=300, kernel_initializer=\"uniform\")`\n",
            "  # Remove the CWD from sys.path while we load stuff.\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:13: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(activation=\"relu\", units=300, kernel_initializer=\"uniform\")`\n",
            "  del sys.path[0]\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:16: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(activation=\"relu\", units=100, kernel_initializer=\"uniform\")`\n",
            "  app.launch_new_instance()\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:21: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(activation=\"softmax\", units=10, kernel_initializer=\"uniform\")`\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "X0guDc5p8y7z",
        "colab_type": "code",
        "outputId": "36382a54-b198-4538-a915-ff64d46e7e64",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        }
      },
      "cell_type": "code",
      "source": [
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "dense_7 (Dense)              (None, 400)               314000    \n",
            "_________________________________________________________________\n",
            "dense_8 (Dense)              (None, 300)               120300    \n",
            "_________________________________________________________________\n",
            "dense_9 (Dense)              (None, 300)               90300     \n",
            "_________________________________________________________________\n",
            "dense_10 (Dense)             (None, 300)               90300     \n",
            "_________________________________________________________________\n",
            "dense_11 (Dense)             (None, 100)               30100     \n",
            "_________________________________________________________________\n",
            "dense_12 (Dense)             (None, 10)                1010      \n",
            "=================================================================\n",
            "Total params: 646,010\n",
            "Trainable params: 646,010\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "7vkqJ9Z-80yV",
        "colab_type": "code",
        "outputId": "5750b1d3-da61-4dbf-e8a1-fea5f7e4b611",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 781
        }
      },
      "cell_type": "code",
      "source": [
        "# setting the learning rate\n",
        "learning_rate = 0.01\n",
        "adam = optimizers.Adam(lr = learning_rate)\n",
        "\n",
        "\n",
        "# compiling the model\n",
        "# using the plain vanilla stochastic gradient descent as our optimizing technique\n",
        "# using categorical cross entropy for multiple outputs\n",
        "model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])\n",
        "\n",
        "# feeding the training data to the model\n",
        "model.fit(x_train, y_train, batch_size = 100, epochs = 20, verbose = 2, validation_data = (x_cv, y_cv))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 45000 samples, validate on 15000 samples\n",
            "Epoch 1/20\n",
            " - 9s - loss: 0.4441 - acc: 0.8571 - val_loss: 0.1924 - val_acc: 0.9418\n",
            "Epoch 2/20\n",
            " - 9s - loss: 0.1457 - acc: 0.9574 - val_loss: 0.1520 - val_acc: 0.9557\n",
            "Epoch 3/20\n",
            " - 9s - loss: 0.0949 - acc: 0.9720 - val_loss: 0.1151 - val_acc: 0.9679\n",
            "Epoch 4/20\n",
            " - 9s - loss: 0.0696 - acc: 0.9799 - val_loss: 0.1102 - val_acc: 0.9685\n",
            "Epoch 5/20\n",
            " - 9s - loss: 0.0549 - acc: 0.9831 - val_loss: 0.1200 - val_acc: 0.9685\n",
            "Epoch 6/20\n",
            " - 9s - loss: 0.0452 - acc: 0.9868 - val_loss: 0.1269 - val_acc: 0.9679\n",
            "Epoch 7/20\n",
            " - 9s - loss: 0.0413 - acc: 0.9877 - val_loss: 0.1094 - val_acc: 0.9722\n",
            "Epoch 8/20\n",
            " - 9s - loss: 0.0338 - acc: 0.9901 - val_loss: 0.1130 - val_acc: 0.9719\n",
            "Epoch 9/20\n",
            " - 8s - loss: 0.0291 - acc: 0.9917 - val_loss: 0.1220 - val_acc: 0.9715\n",
            "Epoch 10/20\n",
            " - 9s - loss: 0.0290 - acc: 0.9917 - val_loss: 0.1236 - val_acc: 0.9720\n",
            "Epoch 11/20\n",
            " - 9s - loss: 0.0228 - acc: 0.9931 - val_loss: 0.1115 - val_acc: 0.9737\n",
            "Epoch 12/20\n",
            " - 9s - loss: 0.0200 - acc: 0.9938 - val_loss: 0.1286 - val_acc: 0.9743\n",
            "Epoch 13/20\n",
            " - 9s - loss: 0.0217 - acc: 0.9935 - val_loss: 0.1313 - val_acc: 0.9713\n",
            "Epoch 14/20\n",
            " - 9s - loss: 0.0195 - acc: 0.9943 - val_loss: 0.1215 - val_acc: 0.9754\n",
            "Epoch 15/20\n",
            " - 9s - loss: 0.0206 - acc: 0.9944 - val_loss: 0.1101 - val_acc: 0.9769\n",
            "Epoch 16/20\n",
            " - 9s - loss: 0.0142 - acc: 0.9958 - val_loss: 0.1134 - val_acc: 0.9773\n",
            "Epoch 17/20\n",
            " - 9s - loss: 0.0127 - acc: 0.9963 - val_loss: 0.1137 - val_acc: 0.9789\n",
            "Epoch 18/20\n",
            " - 9s - loss: 0.0143 - acc: 0.9961 - val_loss: 0.1370 - val_acc: 0.9756\n",
            "Epoch 19/20\n",
            " - 9s - loss: 0.0179 - acc: 0.9951 - val_loss: 0.1147 - val_acc: 0.9741\n",
            "Epoch 20/20\n",
            " - 9s - loss: 0.0104 - acc: 0.9974 - val_loss: 0.1477 - val_acc: 0.9727\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f5435a4ee10>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 63
        }
      ]
    },
    {
      "metadata": {
        "id": "Btj577xZBJ7v",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Multi Layer Perceptron**"
      ]
    },
    {
      "metadata": {
        "id": "KfuJuDf-_Lv_",
        "colab_type": "code",
        "outputId": "245c6b0c-813c-44a7-d9bb-954e1b502dea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "cell_type": "code",
      "source": [
        "from sklearn.neural_network import MLPClassifier\n",
        "\n",
        "# creating the model\n",
        "model = MLPClassifier()\n",
        "\n",
        "# feeding the training set to the modeel\n",
        "model.fit(x_train, y_train)\n",
        "\n",
        "# predicting the results for test set\n",
        "y_pred = model.predict(test_x)\n",
        "\n",
        "# calculating the accuracies\n",
        "print(\"Training accuracy :\", model.score(x_train, y_train))\n",
        "print(\"Testing accuracy :\", model.score(x_cv, y_cv))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training accuracy : 1.0\n",
            "Testing accuracy : 0.9518\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "G0wVrjt6EnEl",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "**Random Forest Classifier**"
      ]
    },
    {
      "metadata": {
        "id": "rRdx5n_fIN6x",
        "colab_type": "code",
        "outputId": "12d5a615-4dc4-47aa-936e-4af15c5e1110",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "cell_type": "code",
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "\n",
        "# creating model\n",
        "model = RandomForestClassifier()\n",
        "\n",
        "# feeding the training data into the model\n",
        "model.fit(x_train, y_train)\n",
        "\n",
        "# calculating the accuracy\n",
        "print(\"Training accuracy :\", model.score(x_train, y_train))\n",
        "print(\"Testing accuracy :\", model.score(x_cv, y_cv))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training accuracy : 0.9911777777777778\n",
            "Testing accuracy : 0.8478666666666667\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}