fashion-mnist.ipynb

{
  "cells": [
    {
      "metadata": {
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "trusted": true
      },
      "cell_type": "code",
      "source": "import tensorflow as tf\nimport numpy as np\nimport matplotlib.pyplot as plt",
      "execution_count": 1,
      "outputs": []
    },
    {
      "metadata": {
        "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
        "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
        "trusted": true
      },
      "cell_type": "code",
      "source": "import pandas as pd\nimport numpy as np\nimport math",
      "execution_count": 2,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "5be551c7b8627f978dbe114ef6716cb0124ee250"
      },
      "cell_type": "code",
      "source": "def load_mnist_data():\n    dataset1=pd.read_csv(\"../input/fashion-mnist_train.csv\")\n    y1=dataset1['label']\n    y_train=np.array(y1).reshape(1,-1)\n    xt1=[]\n    for i in range(1,785):\n        x1=dataset1['pixel'+str(i)]\n        xt1.append(np.array(x1))\n    x_t1=np.array(xt1) \n    dataset2=pd.read_csv(\"../input/fashion-mnist_test.csv\")\n    y2=dataset2['label']\n    y_test=np.array(y2).reshape(1,-1)\n    xt2=[]\n    for i in range(1,785):\n       x1=dataset2['pixel'+str(i)]\n       xt2.append(np.array(x1))\n    x_t2=np.array(xt2)\n    return x_t1.T.reshape(-1,28,28),y_train,x_t2.T.reshape(-1,28,28),y_test\n",
      "execution_count": 3,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "ffdd68dc6547fe36f753e12b643ca17e44dea716"
      },
      "cell_type": "code",
      "source": "import os\nos.listdir('../input')",
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "execution_count": 4,
          "data": {
            "text/plain": "['fashion-mnist_train.csv', 'fashion-mnist_test.csv']"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "e85e049b912cf177edf92f91281ad528a2daba6f"
      },
      "cell_type": "code",
      "source": "def convert_to_one_hot(y,C):\n    Y=np.eye(C)[y.reshape(-1)]\n    return Y",
      "execution_count": 5,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "b326c4a159e7830a48bc6a61b8b08713b8630be9"
      },
      "cell_type": "code",
      "source": "def random_mini_batches(X,Y,minibatch_size):\n    \n    \n    m=X.shape[0]\n    mini_batches=[]\n    \n    permutation=list(np.random.permutation(m))\n    \n    shuffled_X=X[permutation,:,:]\n    shuffled_Y=Y[permutation,:]\n    \n    \n    number_of_minibatches=math.floor(m/minibatch_size)\n    \n    \n    \n    for k in range(number_of_minibatches):\n        mini_batch_X=shuffled_X[k*minibatch_size:k*minibatch_size+minibatch_size,:,:]\n        mini_batch_Y=shuffled_Y[k*minibatch_size:k*minibatch_size+minibatch_size,:]\n        minibatch=(mini_batch_X,mini_batch_Y)\n        mini_batches.append(minibatch)\n        \n    \n    \n    if m%minibatch_size != 0:\n        mini_batch_X = shuffled_X[number_of_minibatches*minibatch_size:m,:,:]\n        mini_batch_Y = shuffled_Y[number_of_minibatches*minibatch_size:m,:]\n        minibatch=(mini_batch_X,mini_batch_Y)\n    \n    \n    \n    mini_batches.append(minibatch)\n    return mini_batches",
      "execution_count": 6,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "5669a0f249411289ff8e09b01a5ebce4f6c23684"
      },
      "cell_type": "code",
      "source": "from tensorflow.python.framework import ops",
      "execution_count": 7,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "f7e3a20898f26224d0d8805f4262489de344c6a0"
      },
      "cell_type": "code",
      "source": "train_X_orig,train_y_orig,test_X_orig,test_y_orig = load_mnist_data()\ntrain_X=train_X_orig.reshape(-1,28,28,1)/255\ntest_X=test_X_orig.reshape(-1,28,28,1)/255\ntrain_y=convert_to_one_hot(train_y_orig,10)\ntest_y=convert_to_one_hot(test_y_orig,10)",
      "execution_count": 8,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "269c54ed0588bc76499764ac905f50d6c68062ec"
      },
      "cell_type": "code",
      "source": "def network(X,krnls):\n    inputs=tf.cast(tf.reshape(X,[-1,28,28,1]),tf.float32)    \n    conv1=tf.nn.conv2d(inputs,krnls[\"1\"],padding=\"SAME\",strides=[1,1,1,1],name='conv1')\n    pool1=tf.layers.max_pooling2d(inputs=conv1,pool_size=[2,2],strides=[2,2],name='pool1')\n    b_norm1=tf.nn.relu(tf.layers.batch_normalization(inputs=pool1,momentum=0.99,center=True,scale=True,epsilon=0.000001,name=\"b_norm1\"))\n#    conv1=tf.layers.conv2d(inputs,filters=32,kernel_size=[3,3],strides=1,padding='same',activation='relu')\n    \n#    conv2=tf.layers.conv2d(inputs=pool1,filters=64,kernel_size=[3,3],strides=1,padding='same',activation='relu')\n    conv2=tf.nn.conv2d(b_norm1,krnls[\"2\"],padding=\"SAME\",strides=[1,1,1,1],name='conv2')\n    pool2=tf.layers.max_pooling2d(inputs=conv2,pool_size=[2,2],strides=[2,2],name='pool2')\n    b_norm2=tf.nn.relu(tf.layers.batch_normalization(inputs=pool2,momentum=0.99,center=True,scale=True,epsilon=0.000001,name=\"b_norm2\"))\n    \n#    conv3=tf.layers.conv2d(inputs=pool2,filters=128,kernel_size=[3,3],strides=1,padding='same',activation='relu')\n    conv3=tf.nn.conv2d(b_norm2,krnls[\"3\"],padding=\"SAME\",strides=[1,1,1,1],name='conv3')\n    pool3=tf.layers.max_pooling2d(inputs=conv3,pool_size=[2,2],strides=[2,2],name='pool3')\n    b_norm3=tf.nn.relu(tf.layers.batch_normalization(inputs=pool3,momentum=0.99,center=True,scale=True,epsilon=0.000001,name=\"b_norm3\"))\n    \n    \n    flat=tf.reshape(pool3,[-1,pool3.shape[1]*pool3.shape[2]*pool3.shape[3]])\n    dense1=tf.layers.dense(inputs=flat,units=512,activation=None,name=\"dense1\")\n    b_norm4=tf.nn.relu(tf.layers.batch_normalization(inputs=dense1,momentum=0.99,scale=True,center=True,epsilon=0.000001,name=\"b_norm4\"))\n    dropout1=tf.layers.dropout(inputs=b_norm4,rate=0.5,name=\"dropout1\")\n    dense2=tf.layers.dense(inputs=dropout1,units=256,activation=None,name=\"dense2\")\n    b_norm5=tf.nn.relu(tf.layers.batch_normalization(inputs=dense2,momentum=0.99,scale=True,center=True,epsilon=0.000001,name=\"b_norm5\"))\n    dropout2=tf.layers.dropout(inputs=b_norm5,rate=0.5,name=\"dropout2\")\n    logits=tf.layers.dense(inputs=dropout2,units=10,name=\"dense3\")\n    \n    return logits",
      "execution_count": 15,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "9a2ff494f04d6fb58b18de42f56c1328d392039d"
      },
      "cell_type": "code",
      "source": "def create_placeholders(w,h,ny):\n    x=tf.placeholder(tf.float32,[None,w,h,None],name='x')\n    y=tf.placeholder(tf.int32,[None,10],name='y')\n    return x,y\n",
      "execution_count": 16,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "dcb4bd2f7916fadffda44aaab99f69d3018cfa8e"
      },
      "cell_type": "code",
      "source": "def initialize_kernels(channels):\n    L=len(channels)\n\n    parameter=dict()\n    for i in range(1,L):\n        parameter[str(i)]=tf.get_variable(name=str(i),shape=[3,3,channels[i-1],channels[i]],dtype=tf.float32,initializer=tf.contrib.layers.xavier_initializer())\n        #parameter[\"b\"+str(i)]=tf.get_variable(name=\"b\"+str(i),shape=[channels[i]],dtype=tf.float32,initializer=tf.zeros_initializer())\n        print(parameter[str(i)].shape)\n        \n    return parameter",
      "execution_count": 17,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "b710cab7dafc1ec6943067ee37b0f70ab3edadb5"
      },
      "cell_type": "code",
      "source": "def predict(X1,Y1):\n    tf.reset_default_graph()\n    (nx1,nx2,c)=X1.shape\n    n_y=Y1.shape[0]\n    X,Y=create_placeholders(nx1,nx2,n_y)\n    costs=[]\n    channels=np.array([1,16,32,64])\n    krnls=initialize_kernels(channels)\n    logits=network(X,krnls)\n    saver=tf.train.Saver()\n    sess=tf.Session()\n    saver.restore(sess,\"..input/mnist.ckpt\")\n    p=tf.argmax(logits,axis=1)\n    prediction=sess.run(p,feed_dict={X:X1.reshape(-1,28,28,1),Y:Y1.reshape(-1,10)})\n    plt.imshow(X1.reshape(28,28))\n    print(\"Predicted class:\",prediction,\" Original class:\",np.argmax(Y1,axis=-1))\n    return None",
      "execution_count": 18,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "36cff4c503657869fe966d59e8f1bdd164fd5444"
      },
      "cell_type": "code",
      "source": "def model(train_X,train_y,test_X,test_y,learning_rate,epoch,batch_size):\n    ops.reset_default_graph()\n    (m,nx1,nx2,_)=train_X.shape\n    n_y=train_y.shape[0]\n    X,Y=create_placeholders(nx1,nx2,n_y)\n    costs=[]\n    channels=np.array([1,16,32,64])\n    krnls=initialize_kernels(channels)\n    logits=network(X,krnls)\n    loss=tf.losses.softmax_cross_entropy(onehot_labels=Y,logits=logits)\n    optimizer=tf.train.AdamOptimizer(learning_rate).minimize(loss)\n    \n    init=tf.global_variables_initializer()\n    saver=tf.train.Saver()\n    with tf.Session() as sess:\n        sess.run(init)\n       \n        for epoch in range(epoch):\n            epoch_cost=0\n            minibatch_cost=0\n            num_mini=int(m/batch_size)\n            minibatch=random_mini_batches(train_X,train_y,100)\n            for mini in minibatch:\n                (mini_X,mini_y)=mini\n                _,minibatch_cost=sess.run([optimizer,loss],feed_dict={X:mini_X,Y:mini_y})\n                epoch_cost=epoch_cost+minibatch_cost/num_mini\n#            if epoch%100==0:\n            print(\"Cost after epoch %i: %f\" %(epoch,epoch_cost))  \n            if epoch%5==0:\n                costs.append(epoch_cost)\n        plt.plot(np.squeeze(costs))\n        plt.ylabel('cost')\n        plt.xlabel('iterations (per tens)')\n        plt.title(\"Learning rate =\" + str(learning_rate))\n        plt.show() \n        correct_prediction = tf.equal(tf.argmax(logits,axis=1), tf.argmax(Y,axis=1))\n        accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n        print (\"Train Accuracy:\", accuracy.eval({X: train_X, Y: train_y}))\n        print (\"Test Accuracy:\", accuracy.eval({X: test_X, Y: test_y}))   \n        saver.save(sess,\"..input/mnist.ckpt\")            \n    return None",
      "execution_count": 19,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "b71a20f71bc3b83f4d3ed02e4435e12dc1d984e4"
      },
      "cell_type": "code",
      "source": "model(train_X,train_y,test_X,test_y,0.0001,500,400)",
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": "(3, 3, 1, 16)\n(3, 3, 16, 32)\n(3, 3, 32, 64)\nCost after epoch 0: 12.526098\nCost after epoch 1: 12.184338\nCost after epoch 2: 11.663555\nCost after epoch 3: 10.832148\nCost after epoch 4: 9.646704\nCost after epoch 5: 8.199902\nCost after epoch 6: 6.844727\nCost after epoch 7: 5.874282\nCost after epoch 8: 5.393362\nCost after epoch 9: 5.015622\nCost after epoch 10: 4.804063\nCost after epoch 11: 4.687207\nCost after epoch 12: 4.514578\nCost after epoch 13: 4.576196\nCost after epoch 14: 4.403019\nCost after epoch 15: 4.279096\nCost after epoch 16: 4.081602\nCost after epoch 17: 4.132589\nCost after epoch 18: 3.939197\nCost after epoch 19: 3.872988\nCost after epoch 20: 3.878765\nCost after epoch 21: 3.821872\nCost after epoch 22: 3.750824\nCost after epoch 23: 3.634014\nCost after epoch 24: 3.604513\nCost after epoch 25: 3.499451\nCost after epoch 26: 3.511227\nCost after epoch 27: 3.390928\nCost after epoch 28: 3.377362\nCost after epoch 29: 3.289319\nCost after epoch 30: 3.332503\nCost after epoch 31: 3.294708\nCost after epoch 32: 3.235949\nCost after epoch 33: 3.305313\nCost after epoch 34: 3.187823\nCost after epoch 35: 3.179315\nCost after epoch 36: 3.169003\nCost after epoch 37: 3.119284\nCost after epoch 38: 3.066318\nCost after epoch 39: 3.033179\nCost after epoch 40: 2.880508\nCost after epoch 41: 2.919845\nCost after epoch 42: 2.943937\nCost after epoch 43: 2.999069\nCost after epoch 44: 2.854460\nCost after epoch 45: 2.787389\nCost after epoch 46: 2.759092\nCost after epoch 47: 2.667463\nCost after epoch 48: 2.680928\nCost after epoch 49: 2.816701\nCost after epoch 50: 2.816999\nCost after epoch 51: 2.784781\nCost after epoch 52: 2.786599\nCost after epoch 53: 2.666836\nCost after epoch 54: 2.736490\nCost after epoch 55: 2.637987\nCost after epoch 56: 2.524348\nCost after epoch 57: 2.482881\nCost after epoch 58: 2.587947\nCost after epoch 59: 2.488665\nCost after epoch 60: 2.606792\nCost after epoch 61: 2.605700\nCost after epoch 62: 2.413991\nCost after epoch 63: 2.486398\nCost after epoch 64: 2.501125\nCost after epoch 65: 2.402578\nCost after epoch 66: 2.337139\nCost after epoch 67: 2.308156\nCost after epoch 68: 2.417359\nCost after epoch 69: 2.305653\nCost after epoch 70: 2.288635\nCost after epoch 71: 2.234855\nCost after epoch 72: 2.203464\nCost after epoch 73: 2.205130\nCost after epoch 74: 2.185047\nCost after epoch 75: 2.194386\nCost after epoch 76: 2.209145\nCost after epoch 77: 2.289666\nCost after epoch 78: 2.178095\nCost after epoch 79: 2.134282\nCost after epoch 80: 2.077987\nCost after epoch 81: 2.028073\nCost after epoch 82: 2.050076\nCost after epoch 83: 2.142632\nCost after epoch 84: 2.049476\nCost after epoch 85: 2.055392\nCost after epoch 86: 2.064340\nCost after epoch 87: 2.017732\nCost after epoch 88: 2.009384\nCost after epoch 89: 1.976317\nCost after epoch 90: 1.907912\nCost after epoch 91: 2.000265\nCost after epoch 92: 1.940232\nCost after epoch 93: 1.883618\nCost after epoch 94: 1.850032\nCost after epoch 95: 1.837185\nCost after epoch 96: 1.769359\nCost after epoch 97: 1.740658\nCost after epoch 98: 1.737215\nCost after epoch 99: 1.783679\nCost after epoch 100: 1.745010\nCost after epoch 101: 1.678194\nCost after epoch 102: 1.785771\nCost after epoch 103: 1.737993\nCost after epoch 104: 1.722941\nCost after epoch 105: 1.844544\nCost after epoch 106: 1.674944\nCost after epoch 107: 1.623354\nCost after epoch 108: 1.557166\nCost after epoch 109: 1.658585\nCost after epoch 110: 1.613944\nCost after epoch 111: 1.589497\nCost after epoch 112: 1.590726\nCost after epoch 113: 1.572587\nCost after epoch 114: 1.519375\nCost after epoch 115: 1.530427\nCost after epoch 116: 1.628829\nCost after epoch 117: 1.573970\nCost after epoch 118: 1.603712\nCost after epoch 119: 1.502146\nCost after epoch 120: 1.409105\nCost after epoch 121: 1.407829\nCost after epoch 122: 1.410875\nCost after epoch 123: 1.372874\nCost after epoch 124: 1.335335\nCost after epoch 125: 1.374718\nCost after epoch 126: 1.307427\nCost after epoch 127: 1.310600\nCost after epoch 128: 1.324491\nCost after epoch 129: 1.279381\nCost after epoch 130: 1.286913\nCost after epoch 131: 1.365294\nCost after epoch 132: 1.304801\nCost after epoch 133: 1.313213\nCost after epoch 134: 1.239349\nCost after epoch 135: 1.212739\nCost after epoch 136: 1.226468\nCost after epoch 137: 1.245510\nCost after epoch 138: 1.157047\nCost after epoch 139: 1.271002\nCost after epoch 140: 1.230343\nCost after epoch 141: 1.173049\nCost after epoch 142: 1.133555\nCost after epoch 143: 1.084057\nCost after epoch 144: 1.122642\nCost after epoch 145: 1.124965\nCost after epoch 146: 1.057037\nCost after epoch 147: 1.108618\nCost after epoch 148: 1.124747\nCost after epoch 149: 1.043622\nCost after epoch 150: 1.013742\nCost after epoch 151: 1.074172\nCost after epoch 152: 1.121708\nCost after epoch 153: 1.111635\nCost after epoch 154: 1.018747\nCost after epoch 155: 1.009628\nCost after epoch 156: 0.990517\nCost after epoch 157: 0.974049\nCost after epoch 158: 0.920522\nCost after epoch 159: 0.928595\nCost after epoch 160: 0.957670\nCost after epoch 161: 0.999770\nCost after epoch 162: 0.948068\nCost after epoch 163: 0.959852\nCost after epoch 164: 0.959428\nCost after epoch 165: 0.855288\nCost after epoch 166: 0.899255\nCost after epoch 167: 0.836996\nCost after epoch 168: 0.798242\nCost after epoch 169: 0.838354\nCost after epoch 170: 0.858880\nCost after epoch 171: 0.821542\nCost after epoch 172: 0.857133\nCost after epoch 173: 0.943773\nCost after epoch 174: 0.871270\nCost after epoch 175: 0.748308\nCost after epoch 176: 0.808723\nCost after epoch 177: 0.759138\nCost after epoch 178: 0.814304\nCost after epoch 179: 0.805472\nCost after epoch 180: 0.862259\nCost after epoch 181: 0.848895\nCost after epoch 182: 0.787011\nCost after epoch 183: 0.794823\nCost after epoch 184: 0.970783\nCost after epoch 185: 1.034874\nCost after epoch 186: 0.823720\nCost after epoch 187: 0.745154\nCost after epoch 188: 0.709292\nCost after epoch 189: 0.668850\nCost after epoch 190: 0.666827\nCost after epoch 191: 0.634540\nCost after epoch 192: 0.619471\nCost after epoch 193: 0.607197\nCost after epoch 194: 0.594491\nCost after epoch 195: 0.590179\nCost after epoch 196: 0.612574\nCost after epoch 197: 0.619169\nCost after epoch 198: 0.566350\nCost after epoch 199: 0.602499\nCost after epoch 200: 0.667580\nCost after epoch 201: 0.582422\nCost after epoch 202: 0.587176\nCost after epoch 203: 0.645588\nCost after epoch 204: 0.646649\nCost after epoch 205: 0.560821\nCost after epoch 206: 0.564473\nCost after epoch 207: 0.629143\nCost after epoch 208: 0.536472\nCost after epoch 209: 0.562202\nCost after epoch 210: 0.538504\nCost after epoch 211: 0.482283\nCost after epoch 212: 0.455170\nCost after epoch 213: 0.526363\nCost after epoch 214: 0.544943\nCost after epoch 215: 0.572730\nCost after epoch 216: 0.622817\nCost after epoch 217: 0.493074\nCost after epoch 218: 0.470981\nCost after epoch 219: 0.468784\nCost after epoch 220: 0.423704\nCost after epoch 221: 0.466828\nCost after epoch 222: 0.412157\nCost after epoch 223: 0.434853\nCost after epoch 224: 0.441611\nCost after epoch 225: 0.440652\nCost after epoch 226: 0.502876\nCost after epoch 227: 0.465289\nCost after epoch 228: 0.449989\nCost after epoch 229: 0.476812\nCost after epoch 230: 0.407941\nCost after epoch 231: 0.385696\nCost after epoch 232: 0.396982\nCost after epoch 233: 0.436827\nCost after epoch 234: 0.415084\nCost after epoch 235: 0.407407\nCost after epoch 236: 0.410422\nCost after epoch 237: 0.356682\nCost after epoch 238: 0.405179\nCost after epoch 239: 0.347479\nCost after epoch 240: 0.360576\nCost after epoch 241: 0.413347\nCost after epoch 242: 0.380977\nCost after epoch 243: 0.327258\nCost after epoch 244: 0.343656\nCost after epoch 245: 0.333472\nCost after epoch 246: 0.317172\nCost after epoch 247: 0.338912\nCost after epoch 248: 0.341451\nCost after epoch 249: 0.293320\nCost after epoch 250: 0.303308\nCost after epoch 251: 0.347269\nCost after epoch 252: 0.304119\nCost after epoch 253: 0.320665\nCost after epoch 254: 0.307672\nCost after epoch 255: 0.383589\nCost after epoch 256: 0.483376\nCost after epoch 257: 0.325225\nCost after epoch 258: 0.305872\nCost after epoch 259: 0.310131\nCost after epoch 260: 0.308781\nCost after epoch 261: 0.274856\nCost after epoch 262: 0.258926\nCost after epoch 263: 0.257994\nCost after epoch 264: 0.259224\nCost after epoch 265: 0.268738\nCost after epoch 266: 0.283612\nCost after epoch 267: 0.274984\n",
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": "Cost after epoch 268: 0.312575\nCost after epoch 269: 0.262707\nCost after epoch 270: 0.354208\nCost after epoch 271: 0.332905\nCost after epoch 272: 0.290584\nCost after epoch 273: 0.301933\nCost after epoch 274: 0.210376\nCost after epoch 275: 0.229864\nCost after epoch 276: 0.231498\nCost after epoch 277: 0.235608\nCost after epoch 278: 0.213401\nCost after epoch 279: 0.222416\nCost after epoch 280: 0.222954\nCost after epoch 281: 0.199061\nCost after epoch 282: 0.209045\nCost after epoch 283: 0.222427\nCost after epoch 284: 0.248927\nCost after epoch 285: 0.218236\nCost after epoch 286: 0.229412\nCost after epoch 287: 0.208142\nCost after epoch 288: 0.212929\nCost after epoch 289: 0.244122\nCost after epoch 290: 0.229019\nCost after epoch 291: 0.222876\nCost after epoch 292: 0.160650\nCost after epoch 293: 0.180379\nCost after epoch 294: 0.173048\nCost after epoch 295: 0.214843\nCost after epoch 296: 0.224863\nCost after epoch 297: 0.214386\nCost after epoch 298: 0.251600\nCost after epoch 299: 0.198387\nCost after epoch 300: 0.152543\nCost after epoch 301: 0.151467\nCost after epoch 302: 0.152220\nCost after epoch 303: 0.162143\nCost after epoch 304: 0.141496\nCost after epoch 305: 0.137469\nCost after epoch 306: 0.134357\nCost after epoch 307: 0.127597\nCost after epoch 308: 0.122815\nCost after epoch 309: 0.149716\nCost after epoch 310: 0.128471\nCost after epoch 311: 0.143128\nCost after epoch 312: 0.174980\nCost after epoch 313: 0.130490\nCost after epoch 314: 0.116422\nCost after epoch 315: 0.132232\nCost after epoch 316: 0.115090\nCost after epoch 317: 0.116515\nCost after epoch 318: 0.123079\nCost after epoch 319: 0.119549\nCost after epoch 320: 0.121258\nCost after epoch 321: 0.114795\nCost after epoch 322: 0.123911\nCost after epoch 323: 0.146185\nCost after epoch 324: 0.184401\nCost after epoch 325: 0.151541\nCost after epoch 326: 0.105195\nCost after epoch 327: 0.110358\nCost after epoch 328: 0.116705\nCost after epoch 329: 0.125870\nCost after epoch 330: 0.104368\nCost after epoch 331: 0.091337\nCost after epoch 332: 0.095344\nCost after epoch 333: 0.094981\nCost after epoch 334: 0.108183\nCost after epoch 335: 0.116653\nCost after epoch 336: 0.119255\nCost after epoch 337: 0.145200\nCost after epoch 338: 0.173338\nCost after epoch 339: 0.110134\nCost after epoch 340: 0.088953\nCost after epoch 341: 0.090633\nCost after epoch 342: 0.088017\nCost after epoch 343: 0.092942\nCost after epoch 344: 0.100501\nCost after epoch 345: 0.097922\nCost after epoch 346: 0.095165\nCost after epoch 347: 0.088966\nCost after epoch 348: 0.094863\nCost after epoch 349: 0.155136\nCost after epoch 350: 0.108942\nCost after epoch 351: 0.124308\nCost after epoch 352: 0.111360\nCost after epoch 353: 0.125784\nCost after epoch 354: 0.082853\nCost after epoch 355: 0.070318\nCost after epoch 356: 0.076487\nCost after epoch 357: 0.070494\nCost after epoch 358: 0.065074\nCost after epoch 359: 0.069077\nCost after epoch 360: 0.074131\nCost after epoch 361: 0.073232\nCost after epoch 362: 0.063454\nCost after epoch 363: 0.060276\nCost after epoch 364: 0.066898\nCost after epoch 365: 0.060990\nCost after epoch 366: 0.058257\nCost after epoch 367: 0.053342\nCost after epoch 368: 0.057177\nCost after epoch 369: 0.070512\nCost after epoch 370: 0.071976\nCost after epoch 371: 0.058137\nCost after epoch 372: 0.054038\nCost after epoch 373: 0.058647\nCost after epoch 374: 0.065334\nCost after epoch 375: 0.067965\nCost after epoch 376: 0.074462\nCost after epoch 377: 0.069302\nCost after epoch 378: 0.062298\nCost after epoch 379: 0.055150\nCost after epoch 380: 0.051900\nCost after epoch 381: 0.057364\nCost after epoch 382: 0.059771\nCost after epoch 383: 0.059295\nCost after epoch 384: 0.042767\nCost after epoch 385: 0.044625\nCost after epoch 386: 0.045446\nCost after epoch 387: 0.044732\nCost after epoch 388: 0.044693\nCost after epoch 389: 0.037518\nCost after epoch 390: 0.041733\nCost after epoch 391: 0.046562\nCost after epoch 392: 0.061672\nCost after epoch 393: 0.065298\nCost after epoch 394: 0.053361\nCost after epoch 395: 0.047705\nCost after epoch 396: 0.044947\nCost after epoch 397: 0.040347\nCost after epoch 398: 0.047205\nCost after epoch 399: 0.049732\nCost after epoch 400: 0.045591\nCost after epoch 401: 0.035164\nCost after epoch 402: 0.035189\nCost after epoch 403: 0.035558\nCost after epoch 404: 0.035041\nCost after epoch 405: 0.031933\nCost after epoch 406: 0.033900\nCost after epoch 407: 0.037010\nCost after epoch 408: 0.034661\nCost after epoch 409: 0.033666\nCost after epoch 410: 0.037339\nCost after epoch 411: 0.041504\nCost after epoch 412: 0.068567\nCost after epoch 413: 0.065176\nCost after epoch 414: 0.036284\nCost after epoch 415: 0.032501\nCost after epoch 416: 0.029948\nCost after epoch 417: 0.029755\nCost after epoch 418: 0.026881\nCost after epoch 419: 0.036066\nCost after epoch 420: 0.038175\nCost after epoch 421: 0.038123\nCost after epoch 422: 0.032145\nCost after epoch 423: 0.031538\nCost after epoch 424: 0.034092\nCost after epoch 425: 0.038888\nCost after epoch 426: 0.027938\nCost after epoch 427: 0.037589\nCost after epoch 428: 0.035146\nCost after epoch 429: 0.028064\nCost after epoch 430: 0.027707\nCost after epoch 431: 0.026848\nCost after epoch 432: 0.028079\nCost after epoch 433: 0.027058\nCost after epoch 434: 0.022802\nCost after epoch 435: 0.027555\nCost after epoch 436: 0.026498\nCost after epoch 437: 0.024327\nCost after epoch 438: 0.024816\nCost after epoch 439: 0.028606\nCost after epoch 440: 0.024666\nCost after epoch 441: 0.024839\nCost after epoch 442: 0.020720\nCost after epoch 443: 0.019152\nCost after epoch 444: 0.022413\nCost after epoch 445: 0.020266\nCost after epoch 446: 0.021546\nCost after epoch 447: 0.023779\nCost after epoch 448: 0.019571\nCost after epoch 449: 0.019595\nCost after epoch 450: 0.019457\nCost after epoch 451: 0.018235\nCost after epoch 452: 0.018483\nCost after epoch 453: 0.018856\nCost after epoch 454: 0.018226\nCost after epoch 455: 0.018162\nCost after epoch 456: 0.017801\nCost after epoch 457: 0.018122\nCost after epoch 458: 0.017875\nCost after epoch 459: 0.020738\nCost after epoch 460: 0.024119\nCost after epoch 461: 0.019697\nCost after epoch 462: 0.016859\nCost after epoch 463: 0.015398\nCost after epoch 464: 0.016030\nCost after epoch 465: 0.015029\nCost after epoch 466: 0.016006\nCost after epoch 467: 0.015533\nCost after epoch 468: 0.015729\nCost after epoch 469: 0.016702\nCost after epoch 470: 0.014494\nCost after epoch 471: 0.013993\nCost after epoch 472: 0.017118\nCost after epoch 473: 0.015957\nCost after epoch 474: 0.015738\nCost after epoch 475: 0.014598\nCost after epoch 476: 0.013883\nCost after epoch 477: 0.013470\nCost after epoch 478: 0.013256\nCost after epoch 479: 0.012819\nCost after epoch 480: 0.014223\nCost after epoch 481: 0.013754\nCost after epoch 482: 0.014263\nCost after epoch 483: 0.014698\nCost after epoch 484: 0.014767\nCost after epoch 485: 0.012806\nCost after epoch 486: 0.012357\nCost after epoch 487: 0.012943\nCost after epoch 488: 0.012652\nCost after epoch 489: 0.013166\nCost after epoch 490: 0.011916\nCost after epoch 491: 0.011979\nCost after epoch 492: 0.012008\nCost after epoch 493: 0.013171\nCost after epoch 494: 0.011956\nCost after epoch 495: 0.012328\nCost after epoch 496: 0.013545\nCost after epoch 497: 0.013680\nCost after epoch 498: 0.011645\nCost after epoch 499: 0.011958\nCost after epoch 500: 0.011415\nCost after epoch 501: 0.017261\nCost after epoch 502: 0.022139\nCost after epoch 503: 0.022044\nCost after epoch 504: 0.015441\nCost after epoch 505: 0.020531\nCost after epoch 506: 0.023907\nCost after epoch 507: 0.013313\nCost after epoch 508: 0.013001\nCost after epoch 509: 0.012551\nCost after epoch 510: 0.014356\nCost after epoch 511: 0.012702\nCost after epoch 512: 0.014396\nCost after epoch 513: 0.012167\nCost after epoch 514: 0.009569\nCost after epoch 515: 0.010353\nCost after epoch 516: 0.010278\nCost after epoch 517: 0.010719\nCost after epoch 518: 0.010933\nCost after epoch 519: 0.009067\nCost after epoch 520: 0.008224\nCost after epoch 521: 0.008373\nCost after epoch 522: 0.008755\nCost after epoch 523: 0.009994\nCost after epoch 524: 0.009168\nCost after epoch 525: 0.009188\nCost after epoch 526: 0.007893\nCost after epoch 527: 0.008320\nCost after epoch 528: 0.007874\nCost after epoch 529: 0.008100\nCost after epoch 530: 0.007548\nCost after epoch 531: 0.007410\nCost after epoch 532: 0.008438\nCost after epoch 533: 0.010502\n",
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": "Cost after epoch 534: 0.008682\nCost after epoch 535: 0.008505\nCost after epoch 536: 0.007979\nCost after epoch 537: 0.007522\nCost after epoch 538: 0.010886\nCost after epoch 539: 0.010628\nCost after epoch 540: 0.008484\nCost after epoch 541: 0.007564\nCost after epoch 542: 0.006689\nCost after epoch 543: 0.007315\nCost after epoch 544: 0.007007\nCost after epoch 545: 0.006896\nCost after epoch 546: 0.007435\nCost after epoch 547: 0.008911\nCost after epoch 548: 0.007619\nCost after epoch 549: 0.006906\nCost after epoch 550: 0.007660\nCost after epoch 551: 0.006887\nCost after epoch 552: 0.007182\nCost after epoch 553: 0.006956\nCost after epoch 554: 0.006124\nCost after epoch 555: 0.005951\nCost after epoch 556: 0.006335\nCost after epoch 557: 0.006367\nCost after epoch 558: 0.005736\nCost after epoch 559: 0.005712\nCost after epoch 560: 0.006315\nCost after epoch 561: 0.005529\nCost after epoch 562: 0.005413\nCost after epoch 563: 0.005542\nCost after epoch 564: 0.005844\nCost after epoch 565: 0.006441\nCost after epoch 566: 0.005316\nCost after epoch 567: 0.005598\nCost after epoch 568: 0.005691\nCost after epoch 569: 0.005564\nCost after epoch 570: 0.006871\nCost after epoch 571: 0.005704\nCost after epoch 572: 0.005358\nCost after epoch 573: 0.007246\nCost after epoch 574: 0.005497\nCost after epoch 575: 0.004726\nCost after epoch 576: 0.005140\nCost after epoch 577: 0.004730\nCost after epoch 578: 0.005285\nCost after epoch 579: 0.005702\nCost after epoch 580: 0.004701\nCost after epoch 581: 0.004843\nCost after epoch 582: 0.004831\nCost after epoch 583: 0.004609\nCost after epoch 584: 0.005558\nCost after epoch 585: 0.005907\nCost after epoch 586: 0.006528\nCost after epoch 587: 0.007398\nCost after epoch 588: 0.004905\nCost after epoch 589: 0.004895\nCost after epoch 590: 0.005275\nCost after epoch 591: 0.005124\nCost after epoch 592: 0.005116\nCost after epoch 593: 0.004636\nCost after epoch 594: 0.004813\nCost after epoch 595: 0.006265\nCost after epoch 596: 0.005975\nCost after epoch 597: 0.004304\nCost after epoch 598: 0.004644\nCost after epoch 599: 0.004205\nCost after epoch 600: 0.003911\nCost after epoch 601: 0.003835\nCost after epoch 602: 0.003769\nCost after epoch 603: 0.003898\nCost after epoch 604: 0.003995\nCost after epoch 605: 0.003931\nCost after epoch 606: 0.004018\nCost after epoch 607: 0.003807\nCost after epoch 608: 0.003863\nCost after epoch 609: 0.003718\nCost after epoch 610: 0.004277\nCost after epoch 611: 0.003958\nCost after epoch 612: 0.003667\nCost after epoch 613: 0.003876\nCost after epoch 614: 0.004482\nCost after epoch 615: 0.004726\nCost after epoch 616: 0.004409\nCost after epoch 617: 0.003938\nCost after epoch 618: 0.004100\nCost after epoch 619: 0.003725\nCost after epoch 620: 0.003629\nCost after epoch 621: 0.003462\nCost after epoch 622: 0.003690\nCost after epoch 623: 0.003346\nCost after epoch 624: 0.003378\nCost after epoch 625: 0.003293\nCost after epoch 626: 0.003352\nCost after epoch 627: 0.003408\nCost after epoch 628: 0.003890\nCost after epoch 629: 0.003921\nCost after epoch 630: 0.003141\nCost after epoch 631: 0.003165\nCost after epoch 632: 0.003390\nCost after epoch 633: 0.003675\nCost after epoch 634: 0.003513\nCost after epoch 635: 0.003418\nCost after epoch 636: 0.003269\nCost after epoch 637: 0.002916\nCost after epoch 638: 0.003333\nCost after epoch 639: 0.003152\nCost after epoch 640: 0.003545\nCost after epoch 641: 0.002888\nCost after epoch 642: 0.002799\nCost after epoch 643: 0.002789\nCost after epoch 644: 0.002798\nCost after epoch 645: 0.002702\nCost after epoch 646: 0.002871\nCost after epoch 647: 0.002815\nCost after epoch 648: 0.002529\nCost after epoch 649: 0.002519\nCost after epoch 650: 0.002566\nCost after epoch 651: 0.002657\nCost after epoch 652: 0.002726\nCost after epoch 653: 0.003014\nCost after epoch 654: 0.002640\nCost after epoch 655: 0.002603\nCost after epoch 656: 0.002692\nCost after epoch 657: 0.002923\nCost after epoch 658: 0.002677\nCost after epoch 659: 0.002403\nCost after epoch 660: 0.002856\nCost after epoch 661: 0.002407\nCost after epoch 662: 0.002685\nCost after epoch 663: 0.002525\nCost after epoch 664: 0.002512\nCost after epoch 665: 0.002518\nCost after epoch 666: 0.002470\nCost after epoch 667: 0.002638\nCost after epoch 668: 0.002404\nCost after epoch 669: 0.002248\nCost after epoch 670: 0.002280\nCost after epoch 671: 0.002237\nCost after epoch 672: 0.002352\nCost after epoch 673: 0.002527\nCost after epoch 674: 0.002368\nCost after epoch 675: 0.002642\nCost after epoch 676: 0.003355\nCost after epoch 677: 0.002561\nCost after epoch 678: 0.002322\nCost after epoch 679: 0.002190\nCost after epoch 680: 0.002248\nCost after epoch 681: 0.002171\nCost after epoch 682: 0.002401\nCost after epoch 683: 0.002618\nCost after epoch 684: 0.002050\nCost after epoch 685: 0.001905\nCost after epoch 686: 0.002098\nCost after epoch 687: 0.002129\nCost after epoch 688: 0.001962\nCost after epoch 689: 0.001880\nCost after epoch 690: 0.002424\nCost after epoch 691: 0.001818\nCost after epoch 692: 0.001808\nCost after epoch 693: 0.001899\nCost after epoch 694: 0.001865\nCost after epoch 695: 0.001963\nCost after epoch 696: 0.002063\nCost after epoch 697: 0.001860\nCost after epoch 698: 0.001952\nCost after epoch 699: 0.002116\nCost after epoch 700: 0.001940\nCost after epoch 701: 0.001871\nCost after epoch 702: 0.001782\nCost after epoch 703: 0.001998\nCost after epoch 704: 0.001705\nCost after epoch 705: 0.001720\nCost after epoch 706: 0.001618\nCost after epoch 707: 0.001659\nCost after epoch 708: 0.001776\nCost after epoch 709: 0.001907\nCost after epoch 710: 0.001725\nCost after epoch 711: 0.001841\nCost after epoch 712: 0.002009\nCost after epoch 713: 0.001823\nCost after epoch 714: 0.001836\nCost after epoch 715: 0.001655\nCost after epoch 716: 0.001575\nCost after epoch 717: 0.001554\nCost after epoch 718: 0.001590\nCost after epoch 719: 0.001697\nCost after epoch 720: 0.001650\nCost after epoch 721: 0.001441\nCost after epoch 722: 0.001506\nCost after epoch 723: 0.001646\nCost after epoch 724: 0.001589\nCost after epoch 725: 0.001570\nCost after epoch 726: 0.001485\nCost after epoch 727: 0.001402\nCost after epoch 728: 0.001391\nCost after epoch 729: 0.001357\nCost after epoch 730: 0.001346\nCost after epoch 731: 0.001319\nCost after epoch 732: 0.001271\nCost after epoch 733: 0.001342\nCost after epoch 734: 0.001349\nCost after epoch 735: 0.001283\nCost after epoch 736: 0.001479\nCost after epoch 737: 0.001370\nCost after epoch 738: 0.001334\nCost after epoch 739: 0.001261\nCost after epoch 740: 0.001283\nCost after epoch 741: 0.001240\nCost after epoch 742: 0.001832\nCost after epoch 743: 0.001525\nCost after epoch 744: 0.001277\nCost after epoch 745: 0.001241\nCost after epoch 746: 0.001193\nCost after epoch 747: 0.001254\nCost after epoch 748: 0.001229\nCost after epoch 749: 0.001110\nCost after epoch 750: 0.001137\nCost after epoch 751: 0.001132\nCost after epoch 752: 0.001138\nCost after epoch 753: 0.001155\nCost after epoch 754: 0.001272\nCost after epoch 755: 0.001181\nCost after epoch 756: 0.001118\nCost after epoch 757: 0.001158\nCost after epoch 758: 0.001148\nCost after epoch 759: 0.001287\nCost after epoch 760: 0.001144\nCost after epoch 761: 0.001106\nCost after epoch 762: 0.001129\nCost after epoch 763: 0.001064\nCost after epoch 764: 0.001064\nCost after epoch 765: 0.001118\nCost after epoch 766: 0.001079\nCost after epoch 767: 0.001117\nCost after epoch 768: 0.001051\nCost after epoch 769: 0.001078\nCost after epoch 770: 0.001080\nCost after epoch 771: 0.000921\nCost after epoch 772: 0.000918\nCost after epoch 773: 0.000956\nCost after epoch 774: 0.000942\nCost after epoch 775: 0.000918\nCost after epoch 776: 0.000922\nCost after epoch 777: 0.000982\nCost after epoch 778: 0.000942\nCost after epoch 779: 0.000960\nCost after epoch 780: 0.000943\nCost after epoch 781: 0.000927\nCost after epoch 782: 0.000923\nCost after epoch 783: 0.001046\nCost after epoch 784: 0.000940\nCost after epoch 785: 0.000875\nCost after epoch 786: 0.000917\nCost after epoch 787: 0.000904\nCost after epoch 788: 0.001027\nCost after epoch 789: 0.000922\nCost after epoch 790: 0.000891\nCost after epoch 791: 0.000959\nCost after epoch 792: 0.000897\nCost after epoch 793: 0.000922\nCost after epoch 794: 0.000868\nCost after epoch 795: 0.000839\nCost after epoch 796: 0.000818\nCost after epoch 797: 0.000822\nCost after epoch 798: 0.000819\nCost after epoch 799: 0.000946\n",
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": "Cost after epoch 800: 0.000824\nCost after epoch 801: 0.000748\nCost after epoch 802: 0.000750\nCost after epoch 803: 0.000771\nCost after epoch 804: 0.000856\nCost after epoch 805: 0.000726\nCost after epoch 806: 0.000774\nCost after epoch 807: 0.000719\nCost after epoch 808: 0.000705\nCost after epoch 809: 0.000704\nCost after epoch 810: 0.000767\nCost after epoch 811: 0.000756\nCost after epoch 812: 0.000716\nCost after epoch 813: 0.000692\nCost after epoch 814: 0.000752\nCost after epoch 815: 0.000745\nCost after epoch 816: 0.000744\nCost after epoch 817: 0.000694\nCost after epoch 818: 0.000690\nCost after epoch 819: 0.000667\nCost after epoch 820: 0.000741\nCost after epoch 821: 0.000655\nCost after epoch 822: 0.000684\nCost after epoch 823: 0.000686\nCost after epoch 824: 0.000682\nCost after epoch 825: 0.000813\nCost after epoch 826: 0.000678\nCost after epoch 827: 0.000745\nCost after epoch 828: 0.000754\nCost after epoch 829: 0.000715\nCost after epoch 830: 0.000704\nCost after epoch 831: 0.000747\nCost after epoch 832: 0.000645\nCost after epoch 833: 0.000617\nCost after epoch 834: 0.000650\nCost after epoch 835: 0.000568\nCost after epoch 836: 0.000564\nCost after epoch 837: 0.000626\nCost after epoch 838: 0.000567\nCost after epoch 839: 0.000548\nCost after epoch 840: 0.000581\nCost after epoch 841: 0.000685\nCost after epoch 842: 0.000619\nCost after epoch 843: 0.000588\nCost after epoch 844: 0.000550\nCost after epoch 845: 0.000564\nCost after epoch 846: 0.000537\nCost after epoch 847: 0.000562\nCost after epoch 848: 0.000607\nCost after epoch 849: 0.000612\nCost after epoch 850: 0.000550\nCost after epoch 851: 0.000556\nCost after epoch 852: 0.000534\nCost after epoch 853: 0.000507\nCost after epoch 854: 0.000527\nCost after epoch 855: 0.000528\nCost after epoch 856: 0.000518\nCost after epoch 857: 0.000514\nCost after epoch 858: 0.000511\nCost after epoch 859: 0.000539\nCost after epoch 860: 0.000520\nCost after epoch 861: 0.000475\nCost after epoch 862: 0.000493\nCost after epoch 863: 0.000466\nCost after epoch 864: 0.000482\nCost after epoch 865: 0.000499\nCost after epoch 866: 0.000475\nCost after epoch 867: 0.000473\nCost after epoch 868: 0.000508\nCost after epoch 869: 0.000487\nCost after epoch 870: 0.000508\nCost after epoch 871: 0.000557\nCost after epoch 872: 0.000486\nCost after epoch 873: 0.000461\nCost after epoch 874: 0.000473\nCost after epoch 875: 0.000475\nCost after epoch 876: 0.000442\nCost after epoch 877: 0.000445\nCost after epoch 878: 0.000459\nCost after epoch 879: 0.000443\nCost after epoch 880: 0.000420\nCost after epoch 881: 0.000433\nCost after epoch 882: 0.000411\nCost after epoch 883: 0.000408\nCost after epoch 884: 0.000429\nCost after epoch 885: 0.000422\nCost after epoch 886: 0.000407\nCost after epoch 887: 0.000433\nCost after epoch 888: 0.000386\nCost after epoch 889: 0.000426\nCost after epoch 890: 0.000401\nCost after epoch 891: 0.000408\nCost after epoch 892: 0.000395\nCost after epoch 893: 0.000402\nCost after epoch 894: 0.000386\nCost after epoch 895: 0.000428\nCost after epoch 896: 0.000390\nCost after epoch 897: 0.000393\nCost after epoch 898: 0.000366\nCost after epoch 899: 0.000365\nCost after epoch 900: 0.000384\nCost after epoch 901: 0.000380\nCost after epoch 902: 0.000414\nCost after epoch 903: 0.000456\nCost after epoch 904: 0.000397\nCost after epoch 905: 0.000367\nCost after epoch 906: 0.000362\nCost after epoch 907: 0.000371\nCost after epoch 908: 0.000358\nCost after epoch 909: 0.000358\nCost after epoch 910: 0.000331\nCost after epoch 911: 0.000356\nCost after epoch 912: 0.000351\nCost after epoch 913: 0.000359\nCost after epoch 914: 0.000332\nCost after epoch 915: 0.000336\nCost after epoch 916: 0.000305\nCost after epoch 917: 0.000330\nCost after epoch 918: 0.000329\nCost after epoch 919: 0.000378\nCost after epoch 920: 0.000315\nCost after epoch 921: 0.000315\nCost after epoch 922: 0.000312\nCost after epoch 923: 0.000313\nCost after epoch 924: 0.000342\nCost after epoch 925: 0.000352\nCost after epoch 926: 0.000367\nCost after epoch 927: 0.000320\nCost after epoch 928: 0.000312\nCost after epoch 929: 0.000304\nCost after epoch 930: 0.000325\nCost after epoch 931: 0.000307\nCost after epoch 932: 0.000283\nCost after epoch 933: 0.000348\nCost after epoch 934: 0.000358\nCost after epoch 935: 0.000550\nCost after epoch 936: 0.000382\nCost after epoch 937: 0.000386\nCost after epoch 938: 0.000343\nCost after epoch 939: 0.000350\nCost after epoch 940: 0.000286\nCost after epoch 941: 0.000292\nCost after epoch 942: 0.000281\nCost after epoch 943: 0.000272\nCost after epoch 944: 0.000267\nCost after epoch 945: 0.000285\nCost after epoch 946: 0.000269\nCost after epoch 947: 0.000296\nCost after epoch 948: 0.000331\nCost after epoch 949: 0.000269\nCost after epoch 950: 0.000249\nCost after epoch 951: 0.000263\nCost after epoch 952: 0.000263\nCost after epoch 953: 0.000232\nCost after epoch 954: 0.000245\nCost after epoch 955: 0.000242\nCost after epoch 956: 0.000235\nCost after epoch 957: 0.000242\nCost after epoch 958: 0.000245\nCost after epoch 959: 0.000236\nCost after epoch 960: 0.000251\nCost after epoch 961: 0.000232\nCost after epoch 962: 0.000242\nCost after epoch 963: 0.000234\nCost after epoch 964: 0.000231\nCost after epoch 965: 0.000229\nCost after epoch 966: 0.000236\nCost after epoch 967: 0.000219\nCost after epoch 968: 0.000212\nCost after epoch 969: 0.000235\nCost after epoch 970: 0.000228\nCost after epoch 971: 0.000250\nCost after epoch 972: 0.000273\nCost after epoch 973: 0.000230\nCost after epoch 974: 0.000213\nCost after epoch 975: 0.000220\nCost after epoch 976: 0.000232\nCost after epoch 977: 0.000210\nCost after epoch 978: 0.000205\nCost after epoch 979: 0.000208\nCost after epoch 980: 0.000214\nCost after epoch 981: 0.000225\nCost after epoch 982: 0.000216\nCost after epoch 983: 0.000210\nCost after epoch 984: 0.000218\nCost after epoch 985: 0.000215\nCost after epoch 986: 0.000202\nCost after epoch 987: 0.000190\nCost after epoch 988: 0.000196\nCost after epoch 989: 0.000197\nCost after epoch 990: 0.000194\nCost after epoch 991: 0.000195\nCost after epoch 992: 0.000195\nCost after epoch 993: 0.000182\nCost after epoch 994: 0.000223\nCost after epoch 995: 0.000182\nCost after epoch 996: 0.000188\nCost after epoch 997: 0.000174\nCost after epoch 998: 0.000177\nCost after epoch 999: 0.000178\n",
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<Figure size 432x288 with 1 Axes>",
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmYHHWdx/H3t4+ZyTG5JyEnSbhCwEAwIYCgyCXXCigqKAribhZc1hXcZUF9FNdlHxBZL1SMnMoh94qoICCHck8ggYQESEIgCTmGJOSeo2e++0fVQNN090wm012drs/refqZ6rp+36ruqW//fvWrKnN3REQkvhJRByAiItFSIhARiTklAhGRmFMiEBGJOSUCEZGYUyIQEYk5JQLZaZnZn83szKjjENnZKRHIdjOzpWZ2VNRxuPtx7n5j1HEAmNmjZvaPZSin1syuM7ONZrbKzC7oYv7zw/k2hsvVZk0bb2aPmNlWM1uY+5l2sez3zewlM8uY2SW9vqFSVkoEUpHMLBV1DJ0qKRbgEmAPYFfg48CFZnZsvhnN7BPARcCR4fwTge9lzXIr8AIwFPgWcKeZNXRz2UXAhcAfe2m7JErurpde2/UClgJHFZh2IjAHeAd4EpiSNe0iYDGwCXgZOCVr2lnAE8CPgLXAf4fj/g78EFgPvA4cl7XMo8A/Zi1fbN4JwONh2Q8BPwduKrANhwPLgf8EVgG/BQYD9wFN4frvA8aE818KtAPNwGbgqnD8JOBBYB3wCvDZXtj3bwHHZL3/PvC7AvPeAvxP1vsjgVXh8J5AC1CfNf1vwDldLZtTxk3AJVF/J/XasZdqBNJrzGwqcB3wzwS/Mn8F3JvVpLAYOAwYSPDr8iYzG5m1ihnAEmAEwcG1c9wrwDDgB8C1ZmYFQig27y3As2FclwBf7GJzdgGGEPwanklQe74+fD8O2AZcBeDu3yI4iJ7n7v3d/Twz60eQBG4BhgOnAb8ws8n5CjOzX5jZOwVeL4bzDAZGAnOzFp0L7FNgG/bJM+8IMxsaTlvi7psKrKvYslJllAikN80EfuXuz7h7uwft9y3AQQDufoe7v+XuHe5+G/AacGDW8m+5+8/cPePu28Jxb7j7r929HbiR4EA4okD5eec1s3HAdOA77t7q7n8H7u1iWzqA77p7i7tvc/e17n6Xu28ND56XAh8rsvyJwFJ3vz7cnheAu4DP5JvZ3b/q7oMKvKaEs/UP/27IWnQDUF8ghv555iWcP3da7rqKLStVRolAetOuwDeyf80CY4FRAGb2JTObkzVtX4Jf752W5Vnnqs4Bd98aDvbPM1+xeUcB67LGFSorW5O7N3e+MbO+ZvYrM3vDzDYSNDMNMrNkgeV3BWbk7IsvENQ0empz+HdA1rgBBM1dhebPnZdw/txpuesqtqxUGSUC6U3LgEtzfs32dfdbzWxX4NfAecBQdx8EzAOym3lKdSvclcAQM+ubNW5sF8vkxvINYC9ghrsPAD4ajrcC8y8DHsvZF/3d/dx8hZnZ1Wa2ucBrPoC7rw+3Zb+sRfcD5hfYhvl55l3t7mvDaRPNrD5n+vxuLCtVRolAeiptZnVZrxTBgf4cM5thgX5mdkJ4sOlHcLBsAjCzLxPUCErO3d8AGoFLzKzGzA4G/mE7V1NPcF7gHTMbAnw3Z/pqgp41ne4D9jSzL5pZOnxNN7O9C8R4Tpgo8r2yzwH8Bvi2mQ02s0nAPwE3FIj5N8BXzGyymQ0Cvt05r7u/SnBS/7vh53cKMIWg+arosgDh9tQRHENS4ToK1Y6kwikRSE/9ieDA2Pm6xN0bCQ5MVxH0rFlE0JsHd38ZuBJ4iuCg+SGCXkLl8gXgYN7rkXQbwfmL7vox0Ad4G3gauD9n+k+AU81svZn9NDyPcAzBSeK3CJqtLgdq2THfJTjp/gbwGHCFu98PYGbjwhrEOIBw/A+AR4A3w2WyE9hpwDSCz+oy4FR3b+rmsr8m+NxPJ+h6uo2uT8BLhTJ3PZhG4sfMbgMWunvuL3uR2FGNQGIhbJbZzcwS4QVYJwH/F3VcIpWgkq6YFCmlXYC7Ca4jWA6cG3bpFIk9NQ2JiMScmoZERGJup2gaGjZsmI8fPz7qMEREdiqzZ89+290buppvp0gE48ePp7GxMeowRER2Kmb2RnfmU9OQiEjMKRGIiMScEoGISMwpEYiIxJwSgYhIzCkRiIjEnBKBiEjMVXUieHjBan7x6KKowxARqWhVnQgee7WJWY8viToMEZGKVtWJIJ1MkGnXTfVERIqp+kTQ2t4RdRgiIhWtyhOB0aZEICJSVJUnggTu0N6h5iERkUJKlgjM7DozW2Nm87LGXWFmC83sRTO7x8wGlap8CBIBoFqBiEgRpawR3AAcmzPuQWBfd58CvApcXMLySScNQOcJRESKKFkicPfHgXU54/7i7pnw7dPAmFKVD1k1gowSgYhIIVGeIzgb+HMpC+hMBBmdIxARKSiSRGBm3wIywM1F5plpZo1m1tjU1NSjct5tGlKNQESkoLInAjM7CzgR+IK7F/yp7u6z3H2au09raOjykZt56WSxiEjXyvrMYjM7FrgQ+Ji7by11ee8lAjUNiYgUUsruo7cCTwF7mdlyM/sKcBVQDzxoZnPM7OpSlQ/vNQ2pRiAiUljJagTufnqe0deWqrx80ik1DYmIdKW6ryxOqGlIRKQr1Z0I1DQkItKl6k4EahoSEelSVSeCGvUaEhHpUlUngpSahkREulTViUAXlImIdK2qE4GahkREulbViUA1AhGRrlV1ItA5AhGRrlV1ItC9hkREulbViaBGTUMiIl2q6kTwbtOQnkcgIlJQdSeChM4RiIh0paoTgZlRk0zQpkdViogUVNWJAIIbz6lpSESksKpPBKlkQk1DIiJFVH0iSCcTtKr7qIhIQVWfCGqSRkY1AhGRgqo+EaRTahoSESmm6hNBKmG6slhEpIiqTwTBOQLVCERECqn6RFCTSugcgYhIEVWfCNLJhJqGRESKKFkiMLPrzGyNmc3LGjfEzB40s9fCv4NLVX6nVMLUNCQiUkQpawQ3AMfmjLsIeNjd9wAeDt+XVI16DYmIFFWyRODujwPrckafBNwYDt8InFyq8julkwkyahoSESmo3OcIRrj7ynB4FTCi0IxmNtPMGs2ssampqccFppOmGoGISBGRnSx2dwcK/lR391nuPs3dpzU0NPS4nJS6j4qIFFXuRLDazEYChH/XlLrAGjUNiYgUVe5EcC9wZjh8JvD7UheopiERkeJK2X30VuApYC8zW25mXwEuA442s9eAo8L3JaXbUIuIFJcq1Yrd/fQCk44sVZn51CQTtOrBNCIiBcXgymIjo0dViogUFINEoKYhEZFiqj4RpMJ7DQW9VUVEJFfVJ4KapAHoxnMiIgVUfSJIJ4NNzHSoeUhEJJ/YJIK2jGoEIiL5xCARBE1Dus2EiEh+MUgEYY1AiUBEJK/YJALdb0hEJL/qTwSpYBPVNCQikl/1J4JEZ/dRJQIRkXyqPxHoHIGISFHVnwhSnYlA5whERPKp/kSQVNOQiEgxMUgEahoSESkmNolA3UdFRPKLQSLQlcUiIsXEIBGoaUhEpBglAhGRmItBItDzCEREiqn6RFCjGoGISFFVnwhS7z6PQIlARCSfqk8E6jUkIlJcJInAzM43s/lmNs/MbjWzulKVVaNbTIiIFFX2RGBmo4GvAdPcfV8gCZxWqvI6zxG0qGlIRCSvqJqGUkAfM0sBfYG3SlWQmVGTStCSaS9VESIiO7WyJwJ3XwH8EHgTWAlscPe/5M5nZjPNrNHMGpuamnaozNpkglbVCERE8oqiaWgwcBIwARgF9DOzM3Lnc/dZ7j7N3ac1NDTsUJk1KSUCEZFComgaOgp43d2b3L0NuBs4pJQF1qYSOkcgIlJAFIngTeAgM+trZgYcCSwoZYGqEYiIFBbFOYJngDuB54GXwhhmlbJMJQIRkcJSURTq7t8Fvluu8mpTSfUaEhEpoOqvLIawRqAri0VE8opHIlD3URGRguKRCNRrSESkoFgkglqdLBYRKSgWiUC9hkRECotNIlDTkIhIfrFIBEH3USUCEZF8YpIIErTqOgIRkbxikQh0HYGISGGxSASdN51z11PKRERyxSIR1CQTuEOmQ4lARCRXPBJB+NxidSEVEfmgWCUC9RwSEfmgbiUCM/tMd8ZVqtpUElCNQEQkn+7WCC7u5riKpKYhEZHCij6PwMyOA44HRpvZT7MmDQAypQysN72bCNp1LYGISK6uHkzzFtAIfBKYnTV+E3B+qYLqbbVhImhuU41ARCRX0UTg7nOBuWZ2S/igecxsMDDW3deXI8De8F6NQIlARCRXd88RPGhmA8xsCMGzhn9tZj8qYVy9qjapcwQiIoV0NxEMdPeNwKeA37j7DODI0oXVu9R9VESksO4mgpSZjQQ+C9xXwnhKQt1HRUQK624i+C/gAWCxuz9nZhOB10oXVu9S91ERkcK66jUEgLvfAdyR9X4J8OmeFmpmg4BrgH0BB85296d6ur6uvNc0pO6jIiK5untl8Rgzu8fM1oSvu8xszA6U+xPgfnefBOwHLNiBdXWpVjUCEZGCuts0dD1wLzAqfP0hHLfdzGwg8FHgWgB3b3X3d3qyru5S91ERkcK6mwga3P16d8+ErxuAhh6WOQFoAq43sxfM7Boz65c7k5nNNLNGM2tsamrqYVGBd5uGdEGZiMgHdDcRrDWzM8wsGb7OANb2sMwUcADwS3efCmwBLsqdyd1nufs0d5/W0NDTnBOoVY1ARKSg7iaCswm6jq4CVgKnAmf1sMzlwHJ3fyZ8fydBYiiZmqSuIxARKWR7uo+e6e4N7j6cIDF8rycFuvsqYJmZ7RWOOhJ4uSfr6i4zoyaZ0MliEZE8utV9FJiSfW8hd19nZlN3oNx/BW42sxpgCfDlHVhXt9SkEuo+KiKSR3cTQcLMBncmg/CeQ91d9gPcfQ4wrafL90RtSjUCEZF8unswvxJ4ysw6Lyr7DHBpaUIqjRolAhGRvLp7ZfFvzKwROCIc9Sl3L2m7fm8LmoaUCEREcnW7eSc88O9UB/9sahoSEcmvu72Gdno1qYSuIxARySM+iSCpXkMiIvnEJxGoaUhEJK/YJILaVFKJQEQkj9gkAvUaEhHJL1aJQDUCEZEPik0iqFWNQEQkr1glAnUfFRH5oNgkgppkgpY2dR8VEckVm0RQm06qRiAikkdsEsHgvjU0t3WwYWtb1KGIiFSU2CSCvUfWA7Bg1caIIxERqSyxSQSTRw4A4OW3lAhERLLFJhE01NcytF8NC1YqEYiIZItNIjAzJo8aoKYhEZEcsUkEAHuPHMCrqzeTUe8hEZF3xSwR1NOa6WDJ21uiDkVEpGLELBEEJ4x1nkBE5D2xSgS7NfSnNpVg7rINUYciIlIxYpUI0skEB4wbzLNL10YdiohIxYgsEZhZ0sxeMLP7ylnujIlDmP/WRjZs0xXGIiIQbY3g34AF5S70oIlDcYfnXl9X7qJFRCpSJInAzMYAJwDXlLvs/ccOoiaV4Oklah4SEYHoagQ/Bi4ECnboN7OZZtZoZo1NTU29VnBdOsnUsYN4RjUCEREggkRgZicCa9x9drH53H2Wu09z92kNDQ29GsNBE4cy/60NOk8gIkI0NYKPAJ80s6XA74AjzOymcgYwY+IQOhwal6pWICJS9kTg7he7+xh3Hw+cBvzV3c8oZwwHjBtMTTKh5iEREWJ2HUGnunSS/ccN0gljEREiTgTu/qi7nxhF2QdNGMK8FRvY2KzzBCISb7GsEUBwwrjDYfbS9VGHIiISqdgmgqnheYIH5q+KOhQRkUjFNhH0qUly+oFjub1xGfNW6CZ0IhJfsU0EABccsxdD+tXy7f+bR0eHRx2OiEgkYp0IBvZJ883jJzFn2Tvc3rgs6nBERCIR60QAcMrU0Rw4fgiX37+Q9Vtaow5HRKTsYp8IzIz/OnkfNjZnuOqRRVGHIyJSdrFPBACTdhnAMZNH8Ie5b+lcgYjEjhJB6Nh9d2HNphaef1PXFYhIvCgRhI6YNJyaZII/z9N1BSISL0oEofq6NIftMYz7563CXc1DIhIfSgRZTtxvJCve2cYPHniFTc1tzF32TtQhiYiUXCrqACrJSfuN5rml6/nlo4uZ9fgS2jucn3/+AE6YMjLq0ERESkaJIEsiYVx68r7s3tCf1Zua+euCNVzxwEKO2WcE6aQqTyJSnXR0y2FmnH3oBC4+bm8uPn4SS9du5acPv8bmlkzUoYmIlIQSQREf32s4H9+rgZ/9dREzLn1IN6cTkaqkRFCEmXHNmdO5beZBpFMJfvzQa1GHJCLS65QIupBMGDMmDuXLh0zgoQWreWXVpqhDEhHpVUoE3XTmIbvSrybJjx58VdcZiEhVUSLopkF9azj38N24f/4qZj2+JOpwRER6jbqPboevHr47C1Zt4rL7F9JQX8unDhgTdUgiIjtMiWA7JBLGlZ/Zj/VbWvnGHXPJtDufnT426rBERHZI2ZuGzGysmT1iZi+b2Xwz+7dyx7Aj6tJJrjtrOofuPoyL7n6Rx15tijokEZEdEsU5ggzwDXefDBwE/IuZTY4gjh6rSye5+owPs+eIes675Xmu+dsSXn97S9RhiYj0SNkTgbuvdPfnw+FNwAJgdLnj2FH9alNcc+Y0Rg/qw3//cQEn/PRvLFu3NeqwRES2W6S9hsxsPDAVeCbPtJlm1mhmjU1Nldn8MmZwX+7/+kd56IKPYcBFd7+orqUistOJLBGYWX/gLuDr7r4xd7q7z3L3ae4+raGhofwBbofdh/fn4uP35olFa/n+fQtozXREHZKISLdFkgjMLE2QBG5297ujiKG3ff7AcZxx0Diue+J1PjfrKdZvaeXav7/Oqb98km2t7VGHJyJSUNm7j5qZAdcCC9z9f8tdfqkkEsZ/n/whDpo4lAtun8uxP3mc1RtbALh37go+N31cxBGKiOQXRY3gI8AXgSPMbE74Oj6COErixCmjuOGs6WxuznD8h3Zh0i713PjkGzp3ICIVq+w1Anf/O2DlLrecDtl9GM99+yj6pJPc+uwyvnnPSzS+sZ7p44dEHZqIyAfoXkMl0rcmhZlx8tRRDOqb5vzb5rBoje5cKiKVR4mgxPrWpLjhywfS3NbOp3/5FIubNkcdkojI+ygRlMH+Ywdx97kfIZUwZv6mkU3NbVGHJCLyLiWCMhk3tC9Xff4Alq7dytH/+zg/uH8hzW3qVioi0VMiKKODdxvK9WdNZ59RA/jlY4v54rXPsGGragciEi0lgjL76J4NXHvWdH52+lTmLHuHk3/xBPNWbOC11ZvY3JKJOjwRiSE9jyAiJ04ZxfD6Os675XlO/NnfARg5sI67zj2EUYP6RBydiMSJagQROnDCEP74tcO46LhJfP/kfdncnOGs659Vc5GIlJUSQcQa6ms552O78cWDduVXX/wwr7+9hX/6bWO3TyRv2NZGe4euWhaRnlMiqCCH7D6MKz+7P8++vo6jf/QYn/3VU9zRuKzggX71xmYOvfyvXP3Y4jJHKiLVRImgwnxyv1H8+HP7s9eIAWzY2sZ/3Pkip179JBvzXHvwwwdeYVNzhocXrI4gUhGpFjpZXIFOnjqak6eOxt2554UVXHjni5x9/XN8fsY4Jgzrx9Rxg5m3YgN3Pr+cQX3TzF2+gU3NbdTXpaMOXUR2QkoEFczM+NQBY6hNJfna716g8Y31JAyuOHU/rnpkEUP61nDpKR/inJtm88ySdRw1eUTUIYvITkiJYCdwwpSRTJ8wmI3bMlxw+xy+ccdcapIJbvmnGew7eiC1qQRPLH5biUBEekTnCHYSw+vr2H14f649czqH7TGMH31uf6aNH0JdOsmBE4bwxKK3AdjU3MZTi9fq+Qci0m2qEexkGupr+e1XZrxv3BGThvO9P7zMBbfPYc6yd1jStIX/PHYS5x6+W0RRisjORImgCnzp4PG8vbmFnz+ymGH9a/jYng1cfv9Clq3fyoSh/Xh44WpmTBjK+UfvGXWoIlKBlAiqQDJh/McnJnHU3iMYPbgPg/rUcOGdc7lr9nJaMh0M7pvm6SXr+Mjuwzhwgp6SJiLvZztDW/K0adO8sbEx6jB2Oq2ZDt7e3MKgvmk+8ePH6eiAPUb0p74uzdGTRzC4b5rxQ/sxdkjfqEMVkRIws9nuPq2r+VQjqGI1qcS7N7C7/NNTOPem51m1oZmXlm/gD3PfAiBhwQ3wTjlgNAdPHEoqYVzxl1eYt2IDl31qCmOH9MXdeezVJvYZNZCG+tooN0lESkA1ghjKtHfw8sqNNLd18PDC1dz01BtsaW2nTzrJqEF1LG7aQm0qQV06yb8esTvL12/jhieXMnnkAO7+6iHUpZNRb4KIdEN3awRKBEJzWztPL1nLXxeuYc6yd/j8geM4ZLdh/Psdc3l26ToAjp48ggdfXs1Re4/gqL2HU1+XJp00Vm9qoW86yaSR9ewzamDEWyIi2ZQIpFfMf2sDqzY0c+TeI/j5I4u44oFXCs57wpSRfPXw3dh1aD/6176/1bG9w1m6dgvjh/YjmbBShy0iVHgiMLNjgZ8ASeAad7+s2PxKBJVjc0uGjdva2NScoSXTzvD6Ora0ZvjTiyv52V8X0dreAcCgvmn61aRoyXQwrH8NTZtaWLullQnD+nHEpOGs29LKXrvUs9eIepo2t7BmYzPJRIKT9h+lB/OI9JKKTQRmlgReBY4GlgPPAae7+8uFllEi2DksX7+VOcveYfn6bSxfv5VtrR2kk0bTphb616XYb8wg7n5hOa+u2szQ/jWs3ND8gXWYQToRXPA+enAfWjMdrN/aSm0qwS4D+zBxWD8G9EnTvzZJXTrJlpZ2Mh0d1KWTjB7Uh/q6FFta26lNJUgnjea2Dlra2kkmE4wcUEe/2hQ1KaMmmSSdMtLJBAkzEgYJM8yC7ridw8G0YLqZhdN43/TmtnbWbWklmTD61qTok05Sl05gppqPRKuSew0dCCxy9yUAZvY74CSgYCKQncOYwX0ZM7h4V9SzD52Au2NmrNywjRXrtzG8vo7hA2pp2tTC7+esYEtrOx0dzvL126hNJRjSr4bmTDsr1m/j5ZUb2dScYUtLhm1t7fSrSZJKJmhua6cl01GmLe2aGdQkE+TmAuODyaEzsZCTYCxcD3mW4d1pBcrvIrbCyxWeWGi54mX1LBkWjbFgHNsfe7BcsbKKrLMHK+xJWf9zyodKfv1PFIlgNLAs6/1yYEbuTGY2E5gJMG7cuPJEJmXR+YUfObAPIwe+1ww0dkhfzjtij26vpzOhdA43bW5hS0uQHFoyHWQ6nLp0gtpUkrb2DlZuaGZbazut7R20ZTqCv+0duEOHOx0OHR3+3rA7njXc3uHvnzecXpNKMKRfLR0dztbWDNvaOtjWmvlAYspX93bvXGewPuhcb/i34LYX3TNF9lmx/VlsjfknFl+md8sqttLiZRXZH0WX2+4welxWsYn9akvfS69iryNw91nALAiahiIORypQ9i8oM2N4fR3UF55/xIC6MkQlsvOJ4u6jK4CxWe/HhONERCQCUSSC54A9zGyCmdUApwH3RhCHiIgQQdOQu2fM7DzgAYLuo9e5+/xyxyEiIoFIzhG4+5+AP0VRtoiIvJ+eUCYiEnNKBCIiMadEICISc0oEIiIxt1PcfdTMmoA3erj4MODtXgynt1RqXFC5sSmu7VOpcUHlxlZtce3q7g1dzbRTJIIdYWaN3bnpUrlValxQubEpru1TqXFB5cYW17jUNCQiEnNKBCIiMReHRDAr6gAKqNS4oHJjU1zbp1LjgsqNLZZxVf05AhERKS4ONQIRESlCiUBEJOaqOhGY2bFm9oqZLTKziyKMY6yZPWJmL5vZfDP7t3D8JWa2wszmhK/jI4htqZm9FJbfGI4bYmYPmtlr4d/BZY5pr6x9MsfMNprZ16PaX2Z2nZmtMbN5WePy7iML/DT8zr1oZgeUOa4rzGxhWPY9ZjYoHD/ezLZl7buryxxXwc/OzC4O99crZvaJMsd1W1ZMS81sTji+nPur0PGhfN8xDx+3V20vgltcLwYmAjXAXGByRLGMBA4Ih+uBV4HJwCXAv0e8n5YCw3LG/QC4KBy+CLg84s9xFbBrVPsL+ChwADCvq30EHA/8meDxtAcBz5Q5rmOAVDh8eVZc47Pni2B/5f3swv+DuUAtMCH8n02WK66c6VcC34lgfxU6PpTtO1bNNYIDgUXuvsTdW4HfASdFEYi7r3T358PhTcACgmc3V6qTgBvD4RuBkyOM5Uhgsbv39MryHebujwPrckYX2kcnAb/xwNPAIDMbWa643P0v7p4J3z5N8ATAsiqwvwo5Cfidu7e4++vAIoL/3bLGZcFzTz8L3FqKsospcnwo23esmhPBaGBZ1vvlVMDB18zGA1OBZ8JR54XVu+vK3QQTcuAvZjbbzGaG40a4+8pweBUwIoK4Op3G+/85o95fnQrto0r63p1N8Mux0wQze8HMHjOzwyKIJ99nVyn76zBgtbu/ljWu7Psr5/hQtu9YNSeCimNm/YG7gK+7+0bgl8BuwP7ASoKqabkd6u4HAMcB/2JmH82e6EFdNJI+xhY8yvSTwB3hqErYXx8Q5T4qxMy+BWSAm8NRK4Fx7j4VuAC4xcwGlDGkivzsspzO+39wlH1/5Tk+vKvU37FqTgQrgLFZ78eE4yJhZmmCD/lmd78bwN1Xu3u7u3cAv6ZEVeJi3H1F+HcNcE8Yw+rOqmb4d0254wodBzzv7qvDGCPfX1kK7aPIv3dmdhZwIvCF8ABC2PSyNhyeTdAWv2e5Yiry2VXC/koBnwJu6xxX7v2V7/hAGb9j1ZwIngP2MLMJ4S/L04B7owgkbH+8Fljg7v+bNT67Xe8UYF7usiWOq5+Z1XcOE5xonEewn84MZzsT+H0548ryvl9pUe+vHIX20b3Al8KeHQcBG7Kq9yVnZscCFwKfdPetWeMbzCwZDk8E9gCWlDGuQp/dvcBpZlZrZhPCuJ4tV1yho4CF7r68c0Q591eh4wPl/I6V46x4VC+Cs+uvEmTzb0UYx6EE1boXgTnh63jgt8BL4fh7gZFljmsiQY+NucD8zn0EDAUeBl7wZcwLAAAEfUlEQVQDHgKGRLDP+gFrgYFZ4yLZXwTJaCXQRtAe+5VC+4igJ8fPw+/cS8C0Mse1iKD9uPN7dnU476fDz3gO8DzwD2WOq+BnB3wr3F+vAMeVM65w/A3AOTnzlnN/FTo+lO07pltMiIjEXDU3DYmISDcoEYiIxJwSgYhIzCkRiIjEnBKBiEjMKRFIpMzsyfDveDP7fC+v+5v5yioVMzvZzL5TonV/s+u5tnudHzKzG3p7vbLzUfdRqQhmdjjB3SlP3I5lUv7eDdbyTd/s7v17I75uxvMkwYVcb+/gej6wXaXaFjN7CDjb3d/s7XXLzkM1AomUmW0OBy8DDgvv/X6+mSUtuLf+c+GNyv45nP9wM/ubmd0LvByO+7/wpnnzO2+cZ2aXAX3C9d2cXVZ4ReYVZjbPgmcxfC5r3Y+a2Z0W3NP/5vCqT8zsMgvuF/+imf0wz3bsCbR0JgEzu8HMrjazRjN71cxODMd3e7uy1p1vW84ws2fDcb/Kugp2s5ldamZzzexpMxsRjv9MuL1zzezxrNX/geCqe4mzUl0tp5de3XkBm8O/hwP3ZY2fCXw7HK4FGgnuV384sAWYkDVv5xWXfQhuXTA0e915yvo08CDBsw5GAG8S3BP+cGADwb1bEsBTBFd9DiW46rWzBj0oz3Z8Gbgy6/0NwP3hevYguJK1bnu2K1/s4fDeBAfwdPj+F8CXwmEnvAqW4H72nWW9BIzOjR/4CPCHqL8HekX7SnU3YYiU2THAFDM7NXw/kOCA2go868G96zt9zcxOCYfHhvOtLbLuQ4Fb3b2d4MZejwHTgY3hupcDWPC0qvEE9/VvBq41s/uA+/KscyTQlDPudg9usvaamS0BJm3ndhVyJPBh4LmwwtKH925I1poV32zg6HD4CeAGM7sduPu9VbEGGNWNMqWKKRFIpTLgX939gfeNDM4lbMl5fxRwsLtvNbNHCX5591RL1nA7wdO+MmZ2IMEB+FTgPOCInOW2ERzUs+WegHO6uV1dMOBGd784z7Q2d+8st53wf9zdzzGzGcAJwGwz+7AHd9esC2OXGNM5AqkUmwge09fpAeBcC27Pi5ntGd4hNddAYH2YBCYRPLqvU1vn8jn+BnwubK9vIHiEYcE7Xlpwn/iB7v4n4HxgvzyzLQB2zxn3GTNLmNluBDf4e2U7titX9rY8DJxqZsPDdQwxs12LLWxmu7n7M+7+HYKaS+dtjPck2ru4SgVQjUAqxYtAu5nNJWhf/wlBs8zz4QnbJvI/MvN+4BwzW0BwoH06a9os4EUze97dv5A1/h7gYIK7rjpwobuvChNJPvXA782sjuDX+AV55nkcuNLMLOsX+ZsECWYAwd0tm83smm5uV673bYuZfZvgyXIJgrtp/gtQ7HGeV5jZHmH8D4fbDvBx4I/dKF+qmLqPivQSM/sJwYnXh8L++fe5+50Rh1WQmdUCjxE8pa5gN1ypfmoaEuk9/wP0jTqI7TAOuEhJQFQjEBGJOdUIRERiTolARCTmlAhERGJOiUBEJOaUCEREYu7/AfL/VAC8e+gbAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "text": "Train Accuracy: 1.0\nTest Accuracy: 0.8201\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "dd17869015c471e860833495bc33f0a10ed20a75"
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "9f6a905fc6aad06ee7698219281fed3e5b9ed6db"
      },
      "cell_type": "code",
      "source": "predict(test_X[9],test_y[9])\ntrain_y.shape",
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": "(3, 3, 1, 16)\n(3, 3, 16, 32)\n(3, 3, 32, 64)\nINFO:tensorflow:Restoring parameters from ..input/mnist.ckpt\nPredicted class: [0]  Original class: 0\n",
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "execution_count": 21,
          "data": {
            "text/plain": "(60000, 10)"
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": "<Figure size 432x288 with 1 Axes>",
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAEvBJREFUeJzt3W1sneV5B/D/dY6PX2LHSRxixyQZgSxkzaANmxtKy6qurFUIlQIfoERqlU2I9ANMq9ZJQ+mH8RFNox3aJiR3RA2oBSZRRD5EvCxthRgoipMlJJACgZkQ58UJTuKXxC/nnGsfzuPKBT/XfXLenmNf/58UxT73eZ7n9uPz93Ps67nvW1QVRORPKukOEFEyGH4ipxh+IqcYfiKnGH4ipxh+IqcYfiKnGH4ipxh+IqcaanmwRmnSZrTW8pDzQrbTPmepKaPtwliFe3N1ctfE910Dl56GwWT7PheNYwyTOiHFPLes8IvIJgBPAEgD+E9Vfcx6fjNacavcUc4hXTp7/1fN9rZTudi21hf2Vbo7V2Voy22xbdkF9mu089/frHR35r19urfo55b8tl9E0gD+A8CdANYD2Coi60vdHxHVVjm/828EcFxVP1LVSQDPAdhSmW4RUbWVE/4VAD6Z8fnJ6LE/ICLbRaRPRPqmMFHG4Yiokqr+135V7VXVHlXtyaCp2ocjoiKVE/4BAKtmfL4yeoyI5oBywr8fwFoRuV5EGgHcD2B3ZbpFRNVWcqlPVbMi8jCAV1Ao9e1U1Xcq1rM5JL32BrN96NZOs/3SGvtncK7Jnm2pZfP52LbRB+2+DQ61m+35vF2O+6OuIbP9iwuPxLa9sfdmc9vj//oVs73ljH3elh2KvwGi8eX95rYelFXnV9U9APZUqC9EVEO8vZfIKYafyCmGn8gphp/IKYafyCmGn8gpqeWKPe3SofU6pDf9hbVm+//duyy2TeJH1AIAcs32OW66aNfSmz+1t59YHL/9lS5726mOrNnesNCYLABA7lP7lu2Ow/HXl4bxwNfVbl+bJhebzabQ92zVyxfM9vzhY6UfvIr26V4M61BR4/l55SdyiuEncorhJ3KK4SdyiuEncorhJ3KqplN317MPv3eN2d4cP2oWqdDsZKlA5SVQbR1fam9vla3aTtj7ln77JZAet9tTObvzWaMSON5hX3tC5bhQCbTpUnz72HL72CfuWmK2rzxsNs8JvPITOcXwEznF8BM5xfATOcXwEznF8BM5xfATOeWmzp/+03Vme/N5u5aemiz92BKoheea7WO39wcK3sbmDWN5c9P0hN0+1Z422zMjdt8mlsS/xDIn7W1HVmXM9pQ9GhkTi+JPjLWsOQBkRu327Df/3Gxv+PUBewd1gFd+IqcYfiKnGH4ipxh+IqcYfiKnGH4ipxh+IqfKqvOLSD+AEQA5AFlV7alEp6phaIM9PjvbbG/faNSFU1PlTX++8IRd7x7ssX9GN16Kr2fnmuw6/cSacbO98xX72Ge+a09msGRPY2zbwJ321935W7M5OJeAdQNEwxV728ud9r0XVzrtexAWmq31oRI3+fylqhpTXRBRPeLbfiKnyg2/AnhVRA6IyPZKdIiIaqPct/23q+qAiHQCeE1Efqeqr898QvRDYTsANGNBmYcjokop68qvqgPR/4MAXgSwcZbn9Kpqj6r2ZGCv60ZEtVNy+EWkVUQWTn8M4NsAjlaqY0RUXeW87e8C8KKITO/nl6r6ckV6RURVV3L4VfUjAF+qYF+qanSl/Sbn8g32AO/GQ/F1XbGHxKNxxK4pX1lq9+39bU+a7Zt+d1ds23VtQ+a258bbzPYlf3vFbB+4vMhs/9LfD8S25dWupXfeOmy2v/rQ1832S9fH37wx2WYfW+3bI3C50/6ezYU6P0t9RE4x/EROMfxETjH8RE4x/EROMfxETrmZurvhst0uE/bPwcn2+LaWwJjGvD36E62D9tDW9W9+z2xvXxA/LPe94zeZ29687hOz/dcH15vtX/7ih2b7i6/cFtu24fb3zW1ffSZ+WwBY+ckps715cWds28Ri+6W/4Ixdnh271i4VppctM9tz586Z7bXAKz+RUww/kVMMP5FTDD+RUww/kVMMP5FTDD+RU/Omzp/uiq/pAsCkPfIUC04GlqK+LX5obFuvPYBz6EF7veflS+xht+Ojxk0GAP54cfyNBpfGWsxtt3W/abbveON6e/vl/2O2H1q9IrbtxLA9nXrH5vjhwAAwfldg+u3n47+nI2vteysy/2tfF0PDuPPXddlPYJ2fiJLC8BM5xfATOcXwEznF8BM5xfATOcXwEzk1b+r8w39h16NzTXZNuPlTe/9j4/FLTa/bYY9p//joGrM9v+SC2X5x2F7m7J61B2Pb3n3mC+a2/7bom2Z795tZs/0fbrzXbMdHrbFN195+2tx0bMpe4elvVtr3GPx4/Xdj2zRtvx5S9pcdNHpd/NcNAK195e2/EnjlJ3KK4SdyiuEncorhJ3KK4SdyiuEncorhJ3IqWOcXkZ0AvgNgUFVvih7rAPA8gNUA+gHcp6p2sbrK0uN23TYzZs+zPmWXZTF5IX6553tvsYu2h0/9idn+3nn7HoXut+yx5088d39sW2ObfV7OvHWt2Z6xbxNA82/tRQkWDsQXzE+9Y9//0Dxkf92Pd8d/3QDQYOw+3zVpbju+1L63QufBZbOYL+HnADZ95rFHAOxV1bUA9kafE9EcEgy/qr4O4LNTzWwBsCv6eBeAuyvcLyKqslLfvHSp6vS9mWcABOYsIqJ6U/ZvLqqqAGJ/sRSR7SLSJyJ9U5go93BEVCGlhv+siHQDQPT/YNwTVbVXVXtUtScDe6AGEdVOqeHfDWBb9PE2AC9VpjtEVCvB8IvIswDeArBORE6KyAMAHgPwLRH5AMBfRZ8T0RwihV/Za6NdOvRWuaNmx6sk+fLNsW0TS+PvAQCA01+1b6doOWPfgzBhT2+PJuMOi1TO/v6mA3+GSU3Z22dbAuvUG/ufXGRv2zhsH3sisH3ngbHYtswp+7YUvXjJbM8F2pOyT/diWIfsExOZB7cqEFEpGH4ipxh+IqcYfiKnGH4ipxh+IqfmzdTd1ab7j8S2Nbfa44Ent64z2xsv2qXCtD36FKlsfEksn7GrPnl7ZXKk7FG1wf2bpcJAlTlnjxbGVJvd3nBuJLYt23/C3tgBXvmJnGL4iZxi+ImcYviJnGL4iZxi+ImcYviJnGKdf1oqUPDOxxe89cbV5qY6bu9bA4cOTRMteavR3ta6RyC4byBcqzcmbwoNF87Hr4oOoIi+NQROrCX0etDAwWs4VL5UvPITOcXwEznF8BM5xfATOcXwEznF8BM5xfATOcU6/7RQ3daQGr1itjefXmS2Z1vs/WfiZ6AuMGr5Gqjz5xrtJ0igXB2q1Zv7DpzybGtRM1DHS5VxbZsHdfwQXvmJnGL4iZxi+ImcYviJnGL4iZxi+ImcYviJnArW+UVkJ4DvABhU1Zuixx4F8CCAc9HTdqjqnmp1sibKqduet5d7lqnlZnu549ZzTWXWw8sQmmtA0/F9a7gcOOeB5nzg1Wsdm4q78v8cwKZZHv+pqm6I/s3t4BM5FAy/qr4OYKgGfSGiGirnd/6HReRtEdkpIksq1iMiqolSw/8kgDUANgA4DeDxuCeKyHYR6RORvilMlHg4Iqq0ksKvqmdVNaeqeQA/A7DReG6vqvaoak8GxmyORFRTJYVfRLpnfHoPgKOV6Q4R1Uoxpb5nAXwDwDUichLAPwH4hohsQKEY0w/gB1XsIxFVQTD8qrp1loefqkJf5qzchUCdP1CvzoyW2QHrPoBgLTyw7/jlCgrbp+xaurX/0FwDmVH7xE102NunRuLnWSh99ob5g3f4ETnF8BM5xfATOcXwEznF8BM5xfATOcWpu6dJoO5kDPlNtbaam04stQtLTeftn8ESKrcZ5bTQsNdyl+AOkWx8W2goc+NwaPnwwLVrnLeTW3jlJ3KK4SdyiuEncorhJ3KK4SdyiuEncorhJ3KKdf4KkLQ9LlamqjuFdLBWX862oa6H7gMwts83BIYDBy5NqVAZv5wluh3g2SFyiuEncorhJ3KK4SdyiuEncorhJ3KK4SdyinX+CtDA8t6h6bFD49pTU6VvH9p3qI4fml47FZza22oMbBuaVjwg12ksITlwqrydzwO88hM5xfATOcXwEznF8BM5xfATOcXwEznF8BM5Fazzi8gqAE8D6EKhMturqk+ISAeA5wGsBtAP4D5VtdeqnqdSi9rtJwTW6E6P28X0fMbevVXLb7hsbxscr1/m5SE1aRy6zDUDQtunJuMXDQjcnuBCMd/aLIAfqep6AF8B8JCIrAfwCIC9qroWwN7ocyKaI4LhV9XTqnow+ngEwDEAKwBsAbAretouAHdXq5NEVHlX9aZORFYDuAXAPgBdqno6ajqDwq8FRDRHFB1+EWkD8AKAH6rq8Mw2LdzcPutvaCKyXUT6RKRvClw7jaheFBV+EcmgEPxfqOqvoofPikh31N4NYHC2bVW1V1V7VLUng6ZK9JmIKiAYfhERAE8BOKaqP5nRtBvAtujjbQBeqnz3iKhaihnS+zUA3wdwREQORY/tAPAYgP8SkQcAfAzgvup0sUYk8HNQ44tDuWWLzU0bL9r7ThvlMACYXGS3izXkN1DKy4eGzZY767i1fHighBkayhxqt5ZVpyLCr6pvIP4lcEdlu0NEtcI7/IicYviJnGL4iZxi+ImcYviJnGL4iZzi1N2V0GD/DM032vXmqdD82IFydcYYthuq44dq7YHRyEF54xVmtQGA5ANLeIe+tgWhect945WfyCmGn8gphp/IKYafyCmGn8gphp/IKYafyCnW+adpaB7pePnGQME5UMdvGLc3TwdmP8s2G4cuc5nrcqfPttpT8TNrF9WebbHbcy3xNzHwqsdzQOQWw0/kFMNP5BTDT+QUw0/kFMNP5BTDT+QU6/zTypjj/fLywEpEgUHx+Qb7PoBcYFi6VcuXMtei1sDlIdQ3S2je/dCxg/c/LIg/McFuO5jzn1d+IqcYfiKnGH4ipxh+IqcYfiKnGH4ipxh+IqeCdX4RWQXgaQBdKIzu7lXVJ0TkUQAPAjgXPXWHqu6pVkfrWbbZrtNLLjD/fOBHcD5QlE5Nlr7v0Hj8UC0+dvH26eMb9yBo4NWXL2OuAADIN/LaZinmJp8sgB+p6kERWQjggIi8FrX9VFX/pXrdI6JqCYZfVU8DOB19PCIixwCsqHbHiKi6rup9kYisBnALgH3RQw+LyNsislNElsRss11E+kSkbwqB+zGJqGaKDr+ItAF4AcAPVXUYwJMA1gDYgMI7g8dn205Ve1W1R1V7MgjcA09ENVNU+EUkg0Lwf6GqvwIAVT2rqjlVzQP4GYCN1esmEVVaMPwiIgCeAnBMVX8y4/HuGU+7B8DRynePiKqlmL/2fw3A9wEcEZFD0WM7AGwVkQ0olP/6AfygKj2cAy7eaP8MzTXZw0MzgVLgVFtgeGkqvl2ygVpcSBVHtqYCw40bRu2+h5YXz7ZUsdQnoRpn/Q8JLuav/W9g9mquy5o+0XzBuyCInGL4iZxi+ImcYviJnGL4iZxi+Imc4tTdFbDyN/Ya25OL7dOcGbEL3qGpvTVVei1f8nY9WnLl1as1Hd+30JBbydrHnmy31x9fdOhcbFtwRvNUYG3zfJlzotcBXvmJnGL4iZxi+ImcYviJnGL4iZxi+ImcYviJnBKt4bhjETkH4OMZD10D4HzNOnB16rVv9dovgH0rVSX7dp2qLivmiTUN/+cOLtKnqj2JdcBQr32r134B7Fupkuob3/YTOcXwEzmVdPh7Ez6+pV77Vq/9Ati3UiXSt0R/5yei5CR95SeihCQSfhHZJCLvichxEXkkiT7EEZF+ETkiIodEpC/hvuwUkUEROTrjsQ4ReU1EPoj+n3WZtIT69qiIDETn7pCIbE6ob6tE5Dci8q6IvCMifxc9nui5M/qVyHmr+dt+EUkDeB/AtwCcBLAfwFZVfbemHYkhIv0AelQ18ZqwiHwdwCiAp1X1puixfwYwpKqPRT84l6jqP9ZJ3x4FMJr0ys3RgjLdM1eWBnA3gL9GgufO6Nd9SOC8JXHl3wjguKp+pKqTAJ4DsCWBftQ9VX0dwNBnHt4CYFf08S4UXjw1F9O3uqCqp1X1YPTxCIDplaUTPXdGvxKRRPhXAPhkxucnUV9LfiuAV0XkgIhsT7ozs+iKlk0HgDMAupLszCyCKzfX0mdWlq6bc1fKiteVxj/4fd7tqvpnAO4E8FD09rYuaeF3tnoq1xS1cnOtzLKy9O8lee5KXfG60pII/wCAVTM+Xxk9VhdUdSD6fxDAi6i/1YfPTi+SGv0/mHB/fq+eVm6ebWVp1MG5q6cVr5MI/34Aa0XkehFpBHA/gN0J9ONzRKQ1+kMMRKQVwLdRf6sP7wawLfp4G4CXEuzLH6iXlZvjVpZGwueu7la8VtWa/wOwGYW/+H8I4MdJ9CGmXzcAOBz9eyfpvgF4FoW3gVMo/G3kAQBLAewF8AGA/wbQUUd9ewbAEQBvoxC07oT6djsKb+nfBnAo+rc56XNn9CuR88Y7/Iic4h/8iJxi+ImcYviJnGL4iZxi+ImcYviJnGL4iZxi+Imc+n9Wkau+ehJGJQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "ff12549a0e26ea851379f32e8932fcb5e91f7cc6"
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.6.6",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 1
}