Tensroflow- Conv 56x56-drop-final-only.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from src.data import Data\n",
    "from src.plothelp import *\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load and pre-process the dataset:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = Data(\"img\")\n",
    "data.load_pickle(\"100x100.pickle \")\n",
    "data.resize_img((56,56)) \n",
    "X_train, X_test, y_train, y_test = data.train_test_split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4698, 56, 56, 3) (522, 56, 56, 3) (4698,) (522,)\n"
     ]
    }
   ],
   "source": [
    "#Convert python lists to np arrays\n",
    "X_trains = np.asarray(X_train)\n",
    "X_tests = np.asarray(X_test)\n",
    "y_trains = np.asarray(y_train)\n",
    "y_tests = np.asarray(y_test)\n",
    "print(X_trains.shape, X_tests.shape, y_trains.shape, y_tests.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_oh, y_test_oh= data.one_hot(y_trains, y_tests)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(54, 87, 4)\n"
     ]
    }
   ],
   "source": [
    "# Split y_train into 54 batches containing 87 img\n",
    "y_batch = np.array_split(y_oh, 54 )\n",
    "y_batch = np.asarray(y_batch)\n",
    "print(y_batch.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(54, 87, 56, 56, 3)\n"
     ]
    }
   ],
   "source": [
    "# Split X_test into 54 batches of 87 img\n",
    "X_train_batch = np.array_split(X_trains, 54 )\n",
    "X_train_batch = np.asarray(X_train_batch)\n",
    "print(X_train_batch.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.002\n",
    "batch_size = 87\n",
    "n_epochs = 100\n",
    "keep_prob  = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# input X: 28x28 grayscale images\n",
    "X = tf.placeholder(tf.float32, [None, 56, 56, 3])\n",
    "Y = tf.placeholder(tf.float32, [None, 4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "l1 = 4  # first convolutional layer output depth\n",
    "l2 = 8  # second convolutional layer output depth\n",
    "l3 = 12  # third convolutional layer\n",
    "l4 = 12  # forth conv\n",
    "l5 = 200  # fully connected layer\n",
    "num_classes = 4 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "w1 = tf.Variable(tf.truncated_normal([5, 5, 3, l1], stddev=0.1))  # 5x5 patch, 3 input channel, K output channels\n",
    "b1 = tf.Variable(tf.random_normal([l1]))\n",
    "\n",
    "w2 = tf.Variable(tf.truncated_normal([5, 5, l1, l2], stddev=0.1))\n",
    "b2 = tf.Variable(tf.random_normal([l2]))\n",
    "\n",
    "w3 = tf.Variable(tf.truncated_normal([4, 4, l2, l3], stddev=0.1))\n",
    "b3 = tf.Variable(tf.random_normal([l3]))\n",
    "\n",
    "w4 = tf.Variable(tf.truncated_normal([4, 4, l3,l4], stddev=0.1))\n",
    "b4 = tf.Variable(tf.random_normal([l4]))\n",
    "\n",
    "w5 = tf.Variable(tf.truncated_normal([7 * 7 * l4, l5], stddev=0.1))\n",
    "b5 = tf.Variable(tf.random_normal([l5]))\n",
    "\n",
    "w = tf.Variable(tf.truncated_normal([l5, num_classes], stddev=0.1))\n",
    "b = tf.Variable(tf.random_normal([num_classes]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# The model\n",
    "Y1 = tf.nn.relu(tf.nn.conv2d(X, w1, strides=[1, 1, 1, 1], padding='SAME') + b1)\n",
    "#Y1 = tf.nn.dropout(Y1,keep_prob=keep_prob) #for adding dropout, comment to disable\n",
    "\n",
    "Y2 = tf.nn.conv2d(Y1, w2, strides=[1, 1, 1, 1], padding='SAME')\n",
    "Y2 = tf.nn.max_pool(Y2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1],padding='SAME')  # reduce to 28x28\n",
    "Y2 = tf.nn.relu( Y2+ b2)\n",
    "#Y2 = tf.nn.dropout(Y2,keep_prob=keep_prob) #for adding dropout, comment to disable\n",
    "\n",
    "Y3 = tf.nn.conv2d(Y2, w3, strides=[1, 1, 1, 1], padding='SAME')\n",
    "Y3 = tf.nn.max_pool(Y3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1],padding='SAME')  # reduce to 14x14\n",
    "Y3 = tf.nn.relu(Y3+ b3)\n",
    "#Y3 = tf.nn.dropout(Y3,keep_prob=keep_prob) #for adding dropout, comment to disable\n",
    "\n",
    "Y4 = tf.nn.conv2d(Y3, w4, strides=[1, 1, 1, 1], padding='SAME')\n",
    "Y4 = tf.nn.max_pool(Y3, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1],padding='SAME')  # reduce to 7x7\n",
    "Y4 = tf.nn.relu(Y4+ b4)\n",
    "#Y4 = tf.nn.dropout(Y4,keep_prob=keep_prob)\n",
    "\n",
    "# fully connected layer\n",
    "fc = tf.reshape(Y4, shape=[-1, 7 * 7 * l4])\n",
    "\n",
    "Y5 = tf.nn.relu(tf.matmul(fc, w5) + b5)\n",
    "Y5 = tf.nn.dropout(Y5,keep_prob=keep_prob)\n",
    "Ylogits = tf.matmul(Y5, w) + b\n",
    "logits = tf.nn.softmax(Ylogits)\n",
    "\n",
    "cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=Ylogits, labels=Y)\n",
    "cross_entropy = tf.reduce_mean(cross_entropy)\n",
    "\n",
    "# Calculate the accuracy of the model\n",
    "correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(Y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "train_step = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0001 accuracy= 0.245210729\n",
      "Epoch: 0002 accuracy= 0.252873564\n",
      "Epoch: 0003 accuracy= 0.252873563\n",
      "Epoch: 0004 accuracy= 0.249680716\n",
      "Epoch: 0005 accuracy= 0.256066411\n",
      "Epoch: 0006 accuracy= 0.261387826\n",
      "Epoch: 0007 accuracy= 0.252873563\n",
      "Epoch: 0008 accuracy= 0.245210730\n",
      "Epoch: 0009 accuracy= 0.258620691\n",
      "Epoch: 0010 accuracy= 0.250744998\n",
      "Epoch: 0011 accuracy= 0.251170711\n",
      "Epoch: 0012 accuracy= 0.261174969\n",
      "Epoch: 0013 accuracy= 0.244997871\n",
      "Epoch: 0014 accuracy= 0.249255002\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-1d524114885e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m             \u001b[0mbatch_X\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX_train_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m             \u001b[0mbatch_Y\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0my_batch\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m             \u001b[0m_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ma\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtrain_step\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maccuracy\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m{\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mbatch_X\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mY\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mbatch_Y\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m             \u001b[0macc\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0macc\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Epoch:\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'%04d'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mepochs\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"accuracy=\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"{:.9f}\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0macc\u001b[0m\u001b[1;33m/\u001b[0m\u001b[1;36m54\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\laurynas\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    765\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    766\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 767\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    768\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    769\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\laurynas\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    963\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    964\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 965\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    966\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m    967\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\laurynas\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1013\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   1014\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1015\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1016\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   1017\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32mC:\\Users\\laurynas\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1020\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   1021\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1022\u001b[0;31m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1023\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   1024\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mC:\\Users\\laurynas\\Anaconda3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1002\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m   1003\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1004\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1005\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m   1006\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    n_batches = 54\n",
    "    for epochs in range(100):\n",
    "        acc = 0\n",
    "        for i in range(n_batches):\n",
    "            batch_X = X_train_batch[i, :]\n",
    "            batch_Y = y_batch[i, :]\n",
    "            _, a = sess.run([train_step, accuracy], {X: batch_X, Y: batch_Y})\n",
    "            acc=acc+a\n",
    "        print(\"Epoch:\", '%04d' % (epochs + 1), \"accuracy=\", \"{:.9f}\".format(acc/54))\n",
    "\n",
    "    saver = tf.train.Saver()\n",
    "    save_path = saver.save(sess, \"./models/temp_conv56x56x3-drop-100lr3.ckpt\")\n",
    "    print(\"Model saved in file: %s\" % save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model restored.\n",
      "0.511494\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    saver = tf.train.Saver().restore(sess, \"./models/temp_conv56x56x3-drop-100lr15.ckpt\")\n",
    "    print(\"Model restored.\")\n",
    "    correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(Y, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict={X: X_tests, Y: y_test_oh}))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}