Tensroflow- Conv 28x28-dropout.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from src.data import Data\n",
    "from src.plothelp import *\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and pre-process the dataset:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = Data(\"img\")\n",
    "data.load_pickle(\"28x28.pickle \")\n",
    "X_train, X_test, y_train, y_test = data.train_test_split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4698, 28, 28) (522, 28, 28) (4698,) (522,)\n"
     ]
    }
   ],
   "source": [
    "#Convert python lists to np arrays\n",
    "X_trains = np.asarray(X_train)\n",
    "X_tests = np.asarray(X_test)\n",
    "y_trains = np.asarray(y_train)\n",
    "y_tests = np.asarray(y_test)\n",
    "print(X_trains.shape, X_tests.shape, y_trains.shape, y_tests.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_oh, y_test_oh= data.one_hot(y_trains, y_tests)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(54, 87, 4)\n"
     ]
    }
   ],
   "source": [
    "# Split y_train into 54 batches containing 87 img\n",
    "y_batch = np.array_split(y_oh, 54 )\n",
    "y_batch = np.asarray(y_batch)\n",
    "print(y_batch.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(54, 87, 28, 28, 1)\n"
     ]
    }
   ],
   "source": [
    "# Split X_test into 54 batches of 87 img\n",
    "X_train_batch = np.array_split(X_trains, 54 )\n",
    "X_train_batch = np.asarray(X_train_batch)\n",
    "X_train_batch = np.expand_dims(X_train_batch, axis=-1)\n",
    "print(X_train_batch.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(522, 28, 28, 1)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_tests = np.expand_dims(X_tests, axis=3)\n",
    "X_tests.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.003\n",
    "batch_size = 87\n",
    "n_epochs = 40\n",
    "keep_prob = 0.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# input X: 28x28 grayscale images\n",
    "X = tf.placeholder(tf.float32, [None, 28, 28, 1])\n",
    "Y = tf.placeholder(tf.float32, [None, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "l1 = 4  # first convolutional layer output depth\n",
    "l2 = 8  # second convolutional layer output depth\n",
    "l3 = 12  # third convolutional layer\n",
    "l4 = 200  # fully connected layer\n",
    "num_classes = 4 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "w1 = tf.Variable(tf.truncated_normal([5, 5, 1, l1], stddev=0.1))  # 5x5 patch, 1 input channel, K output channels\n",
    "b1 = tf.Variable(tf.random_normal([l1]))\n",
    "\n",
    "w2 = tf.Variable(tf.truncated_normal([5, 5, l1, l2], stddev=0.1))\n",
    "b2 = tf.Variable(tf.random_normal([l2]))\n",
    "\n",
    "w3 = tf.Variable(tf.truncated_normal([4, 4, l2, l3], stddev=0.1))\n",
    "b3 = tf.Variable(tf.random_normal([l3]))\n",
    "\n",
    "w4 = tf.Variable(tf.truncated_normal([7 * 7 * l3, l4], stddev=0.1))\n",
    "b4 = tf.Variable(tf.random_normal([l4]))\n",
    "\n",
    "w = tf.Variable(tf.truncated_normal([l4, num_classes], stddev=0.1))\n",
    "b = tf.Variable(tf.random_normal([num_classes]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# The model\n",
    "Y1 = tf.nn.relu(tf.nn.conv2d(X, w1, strides=[1, 1, 1, 1], padding='SAME') + b1)\n",
    "Y1 = tf.nn.dropout(Y1, keep_prob)\n",
    "\n",
    "Y2 = tf.nn.conv2d(Y1, w2, strides=[1, 1, 1, 1], padding='SAME')\n",
    "Y2 = tf.nn.max_pool(Y2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1],padding='SAME')  # reduce to 14x14\n",
    "Y2 = tf.nn.relu( Y2+ b2)\n",
    "Y2 = tf.nn.dropout(Y2, keep_prob)\n",
    "\n",
    "Y3 = tf.nn.conv2d(Y2, w3, strides=[1, 1, 1, 1], padding='SAME')\n",
    "Y3 = tf.nn.max_pool(Y3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1],padding='SAME')  # reduce to 7x7\n",
    "Y3 = tf.nn.relu(Y3+ b3)\n",
    "Y3 = tf.nn.dropout(Y3, keep_prob)\n",
    "\n",
    "# fully connected layer\n",
    "fc = tf.reshape(Y3, shape=[-1, 7 * 7 * l3])\n",
    "\n",
    "Y4 = tf.nn.relu(tf.matmul(fc, w4) + b4)\n",
    "Y4 = tf.nn.dropout(Y4, keep_prob)\n",
    "Ylogits = tf.matmul(Y4, w) + b\n",
    "Ylogits = tf.nn.dropout(Ylogits, keep_prob)\n",
    "logits = tf.nn.softmax(Ylogits)\n",
    "\n",
    "cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=Ylogits, labels=Y)\n",
    "cross_entropy = tf.reduce_mean(cross_entropy)\n",
    "\n",
    "# Calculate the accuracy of the model\n",
    "correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(Y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "train_step = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0001 accuracy= 0.248829289\n",
      "Epoch: 0002 accuracy= 0.259046403\n",
      "Epoch: 0003 accuracy= 0.256492124\n",
      "Epoch: 0004 accuracy= 0.265219241\n",
      "Epoch: 0005 accuracy= 0.268199233\n",
      "Epoch: 0006 accuracy= 0.296722009\n",
      "Epoch: 0007 accuracy= 0.295444869\n",
      "Epoch: 0008 accuracy= 0.285653470\n",
      "Epoch: 0009 accuracy= 0.313963390\n",
      "Epoch: 0010 accuracy= 0.312047680\n",
      "Epoch: 0011 accuracy= 0.347169009\n",
      "Epoch: 0012 accuracy= 0.342486164\n",
      "Epoch: 0013 accuracy= 0.379310345\n",
      "Epoch: 0014 accuracy= 0.400170286\n",
      "Epoch: 0015 accuracy= 0.413154533\n",
      "Epoch: 0016 accuracy= 0.416560239\n",
      "Epoch: 0017 accuracy= 0.437420179\n",
      "Epoch: 0018 accuracy= 0.445508725\n",
      "Epoch: 0019 accuracy= 0.441890166\n",
      "Epoch: 0020 accuracy= 0.425500214\n",
      "Epoch: 0021 accuracy= 0.468284377\n",
      "Epoch: 0022 accuracy= 0.481481476\n",
      "Epoch: 0023 accuracy= 0.444018732\n",
      "Epoch: 0024 accuracy= 0.467432950\n",
      "Epoch: 0025 accuracy= 0.485951467\n",
      "Epoch: 0026 accuracy= 0.453597271\n",
      "Epoch: 0027 accuracy= 0.459557257\n",
      "Epoch: 0028 accuracy= 0.487654316\n",
      "Epoch: 0029 accuracy= 0.468284373\n",
      "Epoch: 0030 accuracy= 0.482120048\n",
      "Epoch: 0031 accuracy= 0.489570029\n",
      "Epoch: 0032 accuracy= 0.485738609\n",
      "Epoch: 0033 accuracy= 0.486164321\n",
      "Epoch: 0034 accuracy= 0.498935715\n",
      "Epoch: 0035 accuracy= 0.507449975\n",
      "Epoch: 0036 accuracy= 0.502554273\n",
      "Epoch: 0037 accuracy= 0.513835671\n",
      "Epoch: 0038 accuracy= 0.490847166\n",
      "Epoch: 0039 accuracy= 0.525117067\n",
      "Epoch: 0040 accuracy= 0.524478500\n",
      "Epoch: 0041 accuracy= 0.530012770\n",
      "Epoch: 0042 accuracy= 0.513409959\n",
      "Epoch: 0043 accuracy= 0.527245635\n",
      "Epoch: 0044 accuracy= 0.527458488\n",
      "Epoch: 0045 accuracy= 0.530651336\n",
      "Epoch: 0046 accuracy= 0.524478500\n",
      "Epoch: 0047 accuracy= 0.540229880\n",
      "Epoch: 0048 accuracy= 0.537462745\n",
      "Epoch: 0049 accuracy= 0.527032777\n",
      "Epoch: 0050 accuracy= 0.541719875\n",
      "Epoch: 0051 accuracy= 0.548318434\n",
      "Epoch: 0052 accuracy= 0.551085565\n",
      "Epoch: 0053 accuracy= 0.563644102\n",
      "Epoch: 0054 accuracy= 0.561728393\n",
      "Epoch: 0055 accuracy= 0.572371221\n",
      "Epoch: 0056 accuracy= 0.565346958\n",
      "Epoch: 0057 accuracy= 0.571519797\n",
      "Epoch: 0058 accuracy= 0.562366964\n",
      "Epoch: 0059 accuracy= 0.555129842\n",
      "Epoch: 0060 accuracy= 0.568114094\n",
      "Epoch: 0061 accuracy= 0.586845468\n",
      "Epoch: 0062 accuracy= 0.591741162\n",
      "Epoch: 0063 accuracy= 0.587271177\n",
      "Epoch: 0064 accuracy= 0.591954022\n",
      "Epoch: 0065 accuracy= 0.604938268\n",
      "Epoch: 0066 accuracy= 0.586845463\n",
      "Epoch: 0067 accuracy= 0.574925499\n",
      "Epoch: 0068 accuracy= 0.611536822\n",
      "Model saved in file: ./models/conv28x28-drop.ckpt\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    n_batches = 54\n",
    "    for epochs in range(68):\n",
    "        acc = 0\n",
    "        for i in range(n_batches):\n",
    "            batch_X = X_train_batch[i, :]\n",
    "            batch_Y = y_batch[i, :]\n",
    "            _, a = sess.run([train_step, accuracy], {X: batch_X, Y: batch_Y})\n",
    "            acc=acc+a\n",
    "        print(\"Epoch:\", '%04d' % (epochs + 1), \"accuracy=\", \"{:.9f}\".format(acc/54))\n",
    "\n",
    "    saver = tf.train.Saver()\n",
    "    save_path = saver.save(sess, \"./models/conv28x28-drop.ckpt\")\n",
    "    print(\"Model saved in file: %s\" % save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model restored.\n",
      "0.408046\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    saver = tf.train.Saver().restore(sess, \"./models/conv28x28-drop.ckpt\")\n",
    "    print(\"Model restored.\")\n",
    "    correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(Y, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict={X: X_tests, Y: y_test_oh}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With added 0.2 dropout  rate and increasing the start learning rate the model improved to 46.55% accuracy with training dataset\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}