MNIST 10 with Fast AI .ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST with Fast AI"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook, I will attempt to replicate all the steps in Chapter 4 of the book 'Deep Learning for Coders with fastai and PyTorch for the full MNIST dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai2.vision.all import *\n",
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "matplotlib.rc('image',cmap='Greys')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(#2) [Path('/storage/data/mnist_png/training'),Path('/storage/data/mnist_png/testing')]"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = untar_data(URLs.MNIST)\n",
    "path.ls()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will view one sample image for each of the 10 classes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_paths = (path/'training').ls().sorted()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(#10) [Path('/storage/data/mnist_png/training/0'),Path('/storage/data/mnist_png/training/1'),Path('/storage/data/mnist_png/training/2'),Path('/storage/data/mnist_png/training/3'),Path('/storage/data/mnist_png/training/4'),Path('/storage/data/mnist_png/training/5'),Path('/storage/data/mnist_png/training/6'),Path('/storage/data/mnist_png/training/7'),Path('/storage/data/mnist_png/training/8'),Path('/storage/data/mnist_png/training/9')]"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_images = [path.ls()[0] for path in training_paths] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAA90lEQVR4nMXPMUtCURjG8ccL2WBpQ2sQ+AWCIMcaiqBNoTVaIqIhauoDqEVLS26tQUWDBHZpDcqgnFxbhaDFQVC5/tOhU3bONTfpnR7e33nhPNL/TmS92v3c+QNzADfT/YX3C6ckKf04M+jwMoCXV3jbD9vsB9zG55+gdRDCE3iPSbHjgNaiYysBXEmSsnDvWTZ2CqWvGH2AJQsXgJTJa3BtVclIpYrJE27PpJTvmPzsoqRaqIAX2gzHyOEAzJic3nafF4A9SdJmFdiwMFGEju9vnftNoJiwT+fKfE/jzDFp8s5Y++jna30dX17drV2o0KgPaTfy6QG/tWoHTrm4GgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<PIL.PngImagePlugin.PngImageFile image mode=L size=28x28 at 0x7FA54E6F0310>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAAhUlEQVR4nGNgGOTAcfNt3JJT/ljglLN+ewiJx4QqGS34A6dG2Tt/nXFK7vo3H5mLYiyfLMM6nBo1/33XwqmTgfHqNZySif+PMuCUDGZA0YgKPvxnxamTjfHKP5ySAbxn/uK2k2EFAx7Jr/gk1XFLHvj2hwE3WPo7RxanpOTzw3J49FILAAA/iyIaFmGe9QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<PIL.PngImagePlugin.PngImageFile image mode=L size=28x28 at 0x7FA54E6F0610>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAA9ElEQVR4nGNgoD2wm/L47xxJrFIcBTf+/tn255EKNknvP3/+nuRZ+PMcGxZJyz9/1ogyMFz+24pFkqXQmoGBgaHmbw1cBCH5p5+BgYGB4T2jDkyECcOA//8ZcEsiAWyST/FIvp+B0yjZ21dwyChM2vfiz2U4lxFKCzF8E0+48vAEAwPDG4sHqFqK3l89+ufPn1t//vz5gwghqIOYrfebT9PTCwg8U9LKcBpNUirLvL382gvhaUbXXjHporlk5Z8NDOGzb/z9U8rgfmsFhuSimN8n/vw6ws3AIBeCJmlx7O+fP3/+7MTuQ7uLf2ZPThfC4f/BAgAnUl0J0X/FDgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<PIL.PngImagePlugin.PngImageFile image mode=L size=28x28 at 0x7FA54E6F0250>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAA8ElEQVR4nGNgGEqAEYUnEMv4P12z8MEmNEl7riD7/6xyjP8ZGBhP+75mYGBgYGBhYGBg4G7UZLDlYmC8//M20+pcbgZNjdcI02b/+fPnz58VOfwMDIy1d//83Yxs1aw/f/6UQZh///z5c04EWZKrXFmZlYGBwT7u5t+985RR5GAg+dO36x5cOLz1908Lbj8f+PtCH6ek36c/L7RwyqZ//HPPkhuXbNKfv3/ccdv7H5+9SSf+vMCt1f7jH1Hcslv/TIAxmdDlFOQZ4dHIgiwxcQeDXYzk/+tYTbz558/fP3+ew/koxq5lYGBgmIHHo0QDAJHBUY+QFhwGAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<PIL.PngImagePlugin.PngImageFile image mode=L size=28x28 at 0x7FA54E6F0910>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAA3ElEQVR4nGNgGLJA6u4+QQYGBvaOciySef9++jAwMFj+vceLIef24V8TJwMD76a/1zAkBR/8PczPwMDQ9fdvLIbGwH//lBgYGHSv/DuMIcd760cWAwOD4Jm/LxwwJNUufjBkYGBo+/e3EYs/zv79G8tp/OrfTkynMjAE/oMACyxyDKwa597+/fv3XzY2SQYGhtN//745K4tdzuXf30s6OPR5Pf33rxCHnPaDv//yWHBITv77dzYuOePvf29L4pCT+v/vGi45hr5/b9xxydl9+J7AiUsy6G8ZLilyAAAHdFVq0RH6+gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<PIL.PngImagePlugin.PngImageFile image mode=L size=28x28 at 0x7FA54E6F0650>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAAxElEQVR4nGNgGO5AodULU9BovgZ/QED3u69/j6FLiZf8/H/7/b9/X57MKzZnYGBgRJJjWRz27ew5hrdHrr7BMFJoyt/LzrgcsePvZRZUEYSxRmf+Xz3DwPj/1uq7mBpr///79+Hd+3ffXrmjSnAxMDAwlAXbczAwMET8nYUqOXk23FWxXx6poUru/bnNS4KBgYHBccffj1Zo9nE3vvv7ece8te++/7sgjekcjdrt77/++7DYCIsfGRgYGBhk7DlwSQ15AADpaUkFjFp+VQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<PIL.PngImagePlugin.PngImageFile image mode=L size=28x28 at 0x7FA56F5A5510>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAA7klEQVR4nGNgoBbI+rcBpxzXv59hOCXn/zuHU87i268gXHL8j//uwamx5d8/b1xyfI//TsGpceHfj7y45Ky//q3DqfHQ3/s8DAwMGrXzajHkbP7/82BgYOv59+/ff3t0yRX/bvMwcE38+/3kjD/m6JLv/vkx8Fz+d8+UoeIjD5qcz+9/9jw7/64XYlB5+w/d2L6/71nW/H3CysAQ+XcvC5pk/78PJf+eKTMwqH35aYRuZe7ffy//NjEwaZ3/24jhE56T//7+Pd9w6e/fScyYYaB77e/fv3//XXdHt5CBgYGBgc37+N8nK2WxSQ0JAABiT2DtzP0uhwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<PIL.PngImagePlugin.PngImageFile image mode=L size=28x28 at 0x7FA54E6F0B50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAAyElEQVR4nNWQPw4BQRSHvxGCRIKEim1Er5K4hRO4gAsgorMOoNkzUDmGRqWVkIiEiE40vBmFLLI7u9H6NfPny3zvzYO/S7r1Wrtaa7373CcBxp0TKEPJACYAi+UyAOoLvJNxZyIisnLXItsQzjrO3HOKDESO9ajW8iLSe58Scb8IQAU0UjHaQy5GO736u2RAq1CKIMw0Mahh1WBMCPZHhCfk17zY2vNfegsqNbXptW3wsWe/hJtVa41lQpHwAPdzlKggMokr9GueEG46Ai3ARaoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<PIL.PngImagePlugin.PngImageFile image mode=L size=28x28 at 0x7FA56F5A5390>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAABBklEQVR4nM3QsSuEcRzH8fc9g1yS0g3qomyU8pTYUQZuMPEPkMlguH+AMyqTWVJkNDCdZNCjy23qBpMOA92xeXp+n4fhLnc9z3c0+Ey/fq++/T6/L/zjZBe34vg7DucMG7hVK/XhlK1UJdWCIHBq+AnLPcoVZ3NAqKPBBJ5KVQB2XfvQlbI0BDD2oV/0utwHxkv9NNeSk+dSVJhYrUvvk6my+fv2R96SVQHyB5Kk1+nOVefN5wcALivGIEsvkqSbrIV7ki5OpG3DCk7a9LxSFBl7L0vHAPu6NrEPYLQZLqTaPkERmNrpzaQr+Q25q427T2nGaHTYWlBtPWNgz/yXdLY8YtAf5wduWYBUnryTJAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<PIL.PngImagePlugin.PngImageFile image mode=L size=28x28 at 0x7FA54F88DE10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAA3klEQVR4nNXPv0pCYRjH8ccjIQq2KAckUMQhEJwiFKJAxDtwE8QhHHQOGr2IhK6gUZwURRBcJWgWBAdxCM7RQ0kE+j1tKp73Xdz6rZ/nr8h/Tm3oPmsoNtjBNq20+BcAHZXd2thPpQ1ThV1vsLPhBfxmvDOXWDnpA/PgqV20WCcl+AO8eBpv2DYl0AXIn5rxhlV4/AD4vPJcyj4Ph47jCp8sRTpjz0qj2BtNq6kZlqn4UsQI+V/hTmki0sBdXWos8Y1zrzFzAWXd0Dq86yzp4EQ0Fp1ARWORNjjqF8/JH9wlaKc3D/dpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<PIL.PngImagePlugin.PngImageFile image mode=L size=28x28 at 0x7FA5612AA950>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for img in sample_images: display(Image.open(img))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pixel Similarity Approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "train_split = 0.8 # train_test_dplit\n",
    "frac_images = 1 # Fraction of images to use\n",
    "image_names = [path.ls() for path in training_paths]\n",
    "[random.shuffle(image_name) for image_name in image_names] #shuffle images in each folder\n",
    "train_image_names = [image_name[:int(len(image_name)*frac_images*train_split)] for image_name in image_names]\n",
    "valid_image_names = [image_name[int(len(image_name)*frac_images*train_split):int(len(image_name)*frac_images)] for image_name in image_names]\n",
    "#Number of images of each type is given \n",
    "n_train_images = [len(image_name) for image_name in train_image_names]\n",
    "n_valid_images = [len(image_name) for image_name in valid_image_names]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[4738, 5393, 4766, 4904, 4673, 4336, 4734, 5012, 4680, 4759]"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_train_images"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now convert image of each digit into a tensor and stack the tensors to get a single stacked tensor for each digit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Convert each image into a tensor \n",
    "train_tensors = [list(map(lambda x: tensor(Image.open(x)),image_name)) for image_name in train_image_names]\n",
    "valid_tensors = [list(map(lambda x: tensor(Image.open(x)),image_name)) for image_name in valid_image_names]\n",
    "#Now stack the tensors\n",
    "train_tensors = [torch.stack(digit_tensor).float()/255 for digit_tensor in train_tensors]\n",
    "valid_tensors = [torch.stack(digit_tensor).float()/255 for digit_tensor in valid_tensors]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As expected 10 train and valid tensors are created , with each tensor of dimensions m_imagesx28x28"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, torch.Size([4738, 28, 28]), 10, torch.Size([1185, 28, 28]))"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_tensors),train_tensors[0].shape,len(valid_tensors),valid_tensors[0].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will also create target labels for each of these training examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y = torch.cat([tensor([i]*n_images) for (i,n_images) in enumerate(n_train_images)])\n",
    "valid_y = torch.cat([tensor([i]*n_images) for (i,n_images) in enumerate(n_valid_images)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([47995, 1]), torch.Size([12005, 1]))"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_y = train_y.unsqueeze(1) # To ensure correct shape of tensor\n",
    "valid_y = valid_y.unsqueeze(1)\n",
    "train_y.shape,valid_y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can compute the mean of each tensor across the first dimension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "digit_means = [t.mean(0) for t in train_tensors]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As expected the length of the list is again equal to 10, and the shape of each tensor is 28x28"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10, torch.Size([28, 28]))"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(digit_means),digit_means[0].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can now visualize the image of the 'average' digit for each of the 10 digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABpgAAACcCAYAAABr5qh0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO3d61dV57XH8ceoEVBAQbkIiIqKt5g2l+ZFR3te9C8/44y0o02aNE3iHUG5I8jdu8l52WHmd9W52Ahs+X5e/rLZ17nm86y14pgHfv311yJJkiRJkiRJkiRlfbTbb0CSJEmSJEmSJEnNxRtMkiRJkiRJkiRJqsUbTJIkSZIkSZIkSarFG0ySJEmSJEmSJEmqxRtMkiRJkiRJkiRJqsUbTJIkSZIkSZIkSarl0Dv++6878i70oTmww69nnWordrpOS7FWtTX2VDUD61TNwDpVM3CPqmZhT1UzsE7VDKxTNYPKOvVfMEmSJEmSJEmSJKkWbzBJkiRJkiRJkiSpFm8wSZIkSZIkSZIkqRZvMEmSJEmSJEmSJKmWQ7v9BiRJkiRJkhr166/NN7f8wIGdnu0uSZK0ffwXTJIkSZIkSZIkSarFG0ySJEmSJEmSJEmqxRtMkiRJkiRJkiRJqsUbTJIkSZIkSZIkSarl0G6/AUnS3pYdlrzdjyM0BDk7GNkBypIkSXtbnX0iPfaXX34J2Zs3b1KPe/36dSqj56P3cvDgwZB99FH8f3wPHYqXZQ4fPpx6vqrnbGTPLEmSVIf/gkmSJEmSJEmSJEm1eINJkiRJkiRJkiRJtXiDSZIkSZIkSZIkSbV4g0mSJEmSJEmSJEm1xGmS+8huDa7f7oH0DutUKfUG4m4n62/vo9qoqhcaWkzZq1evQvby5cuQvXjxIpXRa1Bt0cDjlpaWkLW2tqb+loYq06Dkqvdj/e8d2Trf7jWdNLJ+u/Zrp1CNZ+uq0T2H9bu/7ETffR/sx3sL1QftH1+/fp3Knj9/HrLNzc2QLS8vh2x9fT31GrSnbGtrC1l3d3fITpw4ETLa35bCe9yq/exvWb+SdtNOnHNJev/8F0ySJEmSJEmSJEmqxRtMkiRJkiRJkiRJqsUbTJIkSZIkSZIkSarFG0ySJEmSJEmSJEmqJU43bzLZAd7ZwfU0fJ6GeGYzeo2PP/44ZO3t7SE7duxYKjty5EjISskPr3cw3t5BtfvLL7+EjIbIUu1S9urVq5BRDVCt0ABZqrPs40op5eDBg6n3Y53mZGuI6qAUrpnV1dWQPX78OGSTk5Mhu3//fshmZ2dDtrGxETKqmY6OjpCdP38+ZKOjoyEbGBgIGQ1VbmlpCVkpXNdUv3q3Rtbuqvzly5che/r0achogDdldCzQe6Q6PXr0aMho/abH0R6hTu3ZP/eORoYWN/K3WbQ2ZI/N7Purqr3svsN96/uX/X2pXqgXZx/XSJZFfZIyqrM6f5/NrN2oTn/J1hadJz1//jxka2trIaP97czMTMiWlpZCRvsQWvtpP0qPo89W9d3sxJoh6cOR7RmNXFulXkzXIOhx2XOu7LUoWuerzqWye0/XdOlt/gsmSZIkSZIkSZIk1eINJkmSJEmSJEmSJNXiDSZJkiRJkiRJkiTV4g0mSZIkSZIkSZIk1RInou0BdYZU0rDP7GDPlZWVkE1NTYXs559/Dtndu3dDNjc3F7Jnz56FjIZ4Dg8Ph+zq1ashu3LlSshOnz4dslJKaW9vDxkND3c43e5oZKgyDZ9/8uRJyBYXF0NGw+zJkSNHQka129HRETKqvZaWFnwdqr+qgYt6W7ZeaJjm06dP8TlpuDH1xR9//DFkd+7cCdn9+/dDNjY2FrLV1dWQ0ec7ceJEyKhX/v73vw/Zl19+GbJLly6FrLe3N2SlOIB+q+h3pHWaMlpDS+F+R0O4szU5PT0dsoWFhZDRsUR9sa+vL2RUa9evXw/Z2bNnQ3bq1KnU65aSH2xrnb5bdp1udCB9tpeTRoYO0zFHQ+opo/dMr0v1WArvR+mxlNEewXp+W52aopqkXkd7z/X19ZDR+RWt8ZTRHpXqjz4L1UVra2vIOjs7Q0b71lJ4j0t/T4+j17Z2cxrtqdl9L50n0X6AMno+6msnT54MGdULrel0LkbreSnW0W6oc81qJ55zJ2rAOms+dfppdl9I6/fs7GzI6Jx/YmIiZHT9gd4LrdVnzpwJ2ejoaMjommlXV1fISuH1u5H9qMfN+9fItVXKsr04+3tnz8WbvX78F0ySJEmSJEmSJEmqxRtMkiRJkiRJkiRJqsUbTJIkSZIkSZIkSarFG0ySJEmSJEmSJEmq5dBuv4Hs8KyqwbQ0xJOGgs/Pz4fs7t27Ifvuu+9C9vPPP4fs5s2bqdeggbg0AJSGzk1NTYWMBpt/9dVXISullOHh4ZDRUFEaWNdMg8Q+dNnhdM+fPw/Z8vJyyGhwPaEhxlQXbW1tIcsO/9b2o9qgPrm2toZ/T/VBAzonJydDRgM6aQA49ZyjR4+GjIaKUp3Te6H6PX78eCqjmi6Fe/ehQ3EZ3c+1Tsc+DeCmwa0bGxshowHcpZRy//79kP30008hu3PnTsjGx8dDRustDbCl+qPfm+rv4cOHIaPj7YsvvgjZtWvXQkb7hlJ4UHjVUHD9RyPDYanGS+E6p30h1VXVc/4WDYGnXkXodWltoIzeH70XqseqnHovDVCmbD/L1inVYym81tLaTec51NeoP09PT4eMzmmoJumzUN+l+qPznhMnToSst7c3ZKWUMjAwkMqoH1OdtrS0hGw/7Rmy5/2NDqWnawF0TjQ3NxeymZmZkNF+gPayVEdnz54NWV9fX8hoP5rdd5bS2ADx/Wy7B8NX5bRmUlbndbYT1U92/aWarNp3WqfvV509Kl0f2NzcDBn1Sbo++v3334eMriFMTEyEjPaZVGtdXV0hox5LffzGjRshu3DhQshKKaWnpydk7e3tIaMeTe/ben63qnW/kWsLT58+DRntPaleqE5pH0F7T9onDg0Nhay/vz9k2TorZW9eh/KKgyRJkiRJkiRJkmrxBpMkSZIkSZIkSZJq8QaTJEmSJEmSJEmSavEGkyRJkiRJkiRJkmrhKZG7jAZ5VQ2mpWGwNIT29u3bIfvmm29CRgPr7t69GzIaDkbvm4a80YBnGn5LA7poMGLVQPrW1taQ0RCyvTgcTPXRoMaVlZWQ0VD57BDPU6dOpR7X6LBP5WSH0FJtUO+seiw5evRoyGi4MQ0tzg6wpV5Jg5bpcTS0nAYzUi+nwZ6llNLZ2Rmy7MDq/aKRYfNUkzRsthT+zbPDYGkwNw2IzQ4LXVpaCtnGxkbIsj2aBo3SsFD6bKXwnoB+F9f5d2uknkvheqEelq0Xqt1jx46FjPozHR/ZeqZ9NQ3Y7e7uDlnVGt/S0oK5tkedQd/UZx8/fhwyOh/66aefQkaDkamG6HWpL1Ht0jkOHQt0zNDf0rFQCh+blNHfu2fIoe+k6nuiGqb9Hv1Gi4uLIaP1NnuOT3te2kucPn06ZLQ3zg6Lr+qprulvy9ZVdlg87VHX19fxtWmdz56DUPbs2bPU+6H3TRmtD1R/1MMGBwdDNjw8HLK+vr6QlZLv01T71vjbGrkOUArX7+zsbMjo+ui//vWvkNHaT8cCXU8aGhoKGV23zF6zotel663Ui0vhfUIjPdrzsLdla7eU/HUs2mfSHvXrr78O2bfffhuye/fuhYx6cUdHR8g+/fTTkH3xxRch+9Of/hSykZGRkNH5VSn5PfNO1ppXdiVJkiRJkiRJklSLN5gkSZIkSZIkSZJUizeYJEmSJEmSJEmSVIs3mCRJkiRJkiRJklSLN5gkSZIkSZIkSZJUy6HdfgPkzZs3IXv58iU+9smTJyEbHx8P2XfffReyO3fuhOzhw4eZt1gGBwdD1tnZGbKPP/44ZPRZlpaWQvbs2bOQjY2NhayrqwvfY39/f+qxra2tIfvoI+897hUHDhxIPY6Om+Xl5ZDNz8+HjGrg1KlTITty5Ejqb6nuq2qKPl/2Myv69ddfU9nBgwfx79va2kJG/Y5+9zNnzoSMfvfDhw+HjOqX+vv09HTIZmZmUs/39OnTkG1sbITs9evXISullF9++SVk9N3qbfQd0TFOddHR0YHPSbV24sSJkNFvTuj9ZHvqxMREyB49ehSyzc3N1OtSndWpPWtya7K9k36LV69e4XNSf3n8+HHIqK7oOWmfST22paUlZFRr2f0o1TO97qFD8bTi+PHjISuF1yDK3I++W7Z2q9Y26pNUB5OTkyGj9ZfqmX5b2mcePXo0ZLTfoMcdO3YsZO3t7amsqk67u7tTr121p9LWUJ8thWuYzpepBmn/SP2Ynu/kyZMhGxkZCdnAwEDIqLbofIp6nedDW5ftgbTWrq+vh2xxcTFktP8rha8xPXjwIGR03YnqdGVlJWQvXrwIGX0W+sy0v6Vz99OnT4fss88+C9kf//jHkP3hD38IWSmlDA0NhYxq3+Phbdn9KP3eVddR19bWQjY3NxcyqnOqSeprV65cCRnVAK21hN4z9XH6zM+fPw8Z7dNL4XWAjq+qtUr/0ei5FPXjqampkH399dep7IcffggZnaPTdQW6/kDXLwhdg6V1ga7X076zFD7v2u3e6VmbJEmSJEmSJEmSavEGkyRJkiRJkiRJkmrxBpMkSZIkSZIkSZJq8QaTJEmSJEmSJEmSaolToXYYDf2iwYM03KsUHoRIg7vu3bsXMhpWS4OyaJgXZX19fSGjgcw0YI6GPNKASBpsR99BKaWMjY2F7Pz58yGjAWb0PWjvyA7rpqGHs7OzIevp6QkZDWrMDl/ODu8uZX8P7Hwf6Puk47lqWCD9PQ3O7u3t3fJrUy3QYEeqXxoKScOcac2gY4TWG20d1QANm6S6aGtrSz2uFK5J+i1pj0GormjAKw20pzqlz0yyxyZlNJC56rXts1uTHUxL+7pSSlldXQ0Z1RANfs3+jjQMlgbO0vPRkHAa8Ezvj9b+wcHBkFXVKe0xssNq9W51BijTuQXtFSl78uRJ6v2cOnUqZLSPoP0onae0t7enMuqdLS0tIaN6rHosZfQ6VM/24ohqtWpvRvs4GtK+sLAQMqpf2j/SXoR628DAQMioVqkHUh3QekNZVQ259r8tu35na2ppaSlk4+Pj+Np0DYau69DfUz+m9011lR02T2s/nTc9ffo0ZNTrzp07F7LNzU187devX2fe4r6u3axsz6AaL4V/I1rTqQ6o/mg/evXq1ZBdvHgxZHReR++Pjhnaa1ON07pCx1YpXH/ZTG+jmqQ+QHVWCq/df/vb31IZ1QvV35UrV1KPo/0fnTfRfQbag6+srISM6r5q/04a3Us0yrM2SZIkSZIkSZIk1eINJkmSJEmSJEmSJNXiDSZJkiRJkiRJkiTV4g0mSZIkSZIkSZIk1cLTs98TGi5Fw9ZoWDIN0S6llNu3b4fs/v37IZuens68xTIyMhKyy5cvh4wGgZ09ezZkNAiRBoffvXs39f5+/vnnkFUN2KXPTI+lIaXaO7ID2Oi4oRp49OhRyE6ePBkyGtTY0dERMhooevDgwcr3qe1DtUFDfmnQNPWmUnjQNQ1izA4LzNYCDXakjD4fDeikjN4f1S99X6XwZ3G457vRb0ao9qoGFre2toYsu8egYbc03Jj2HbQG0+Bm6rP0+YaHh0OWHRxO30Ep9t/t1Mi+tRQeCj4xMREyGvJ6/PjxkNHxQIORqTao7mmQLO0bFhYWQtbX1xcyGn7b1tYWslL4eMj22P3cd7PDeymjodelcP+jAcVUGzTcuLu7O2TUw6j/0bkU7VHb29tDRnVPdUbDyeus+7Sm0ePoOfdz7VbJ9tlSuNdS/5yamgrZ/Px8yKgvDg4OhuzcuXMho5qm35w+S52B3b9VtcZTnq3fZq9LqqFGZI/xqvWNzpeHhoZCRufa1J86OztTr0FrMJ1L0T7k22+/DRkdW/RdZ7+vqrzZ628vod+HzomrcjrnJ7QfpXqmvSLVM9UQ7VloP1p1LfS3aD9A76UUvk5Cf589x90vsus5rYHUb0rh6+R0Pk7n7aOjoyH785//HLKvvvoqZLTG03nTTz/9FLKZmZmQLS8vh4w+M+1zqta47V77toNHhCRJkiRJkiRJkmrxBpMkSZIkSZIkSZJq8QaTJEmSJEmSJEmSavEGkyRJkiRJkiRJkmrhiaY7iIZ+ra2thYwGZpfCQ79osCcNiaPhstevXw/Zl19+GbLz58+HjAYy07DPZ8+ehYyG7NEw3dnZ2ZBVDbajoWGUZYf5aW+j3/bevXsho4HMNKSUhirTwEMa8uhQ7t1DwybpN6oavkrDZauGg24V9RwaUkiDaWkYOQ1DzH4PVPs0xLOU6u9M/112oDQ9rs5gWhoYSjVEPZAGdH7//fepxz148CBk9Fk++eSTkNHw2/7+/pDRMGeq56rXtv9uTbbOVldX8e/Hx8dDNjk5GTLaC3d3d6ey7N6TjgXaU9J+m/ru4OBgyNrb20NWNQR9vwyf3wnZod50LlRKKUtLSyGjOqChynQOkl0rqU6z+xVap2mPSvVHr1v1nrP72Uay/SRbq9QTS+EB29TH6ByazpNoiDf1NhpeT/1qY2MjZHTc0TpCtUF78tbW1pCVwscEHU/Z87Zmkj3WGjk3oPX33Llz+H6ox9B+j36z3t7ekNG+kNZbel3q0f/3f/8Xsvn5+ZBR7dKxQMdRZ2dnyErhmqb+2+w1uVvoe6ta3yin44HqKnveTvVHPYyuj9Ke486dOyGj6wDUx4eGhkJGx1spfN6Vrd39vJeluqD1nHrL4uIiPuf9+/dD9vjx45D19PSE7I9//GPI/vSnP4WM6oA+C53vzczMhIz20LTXpnM42hNRTZWyN/ee/gsmSZIkSZIkSZIk1eINJkmSJEmSJEmSJNXiDSZJkiRJkiRJkiTV4g0mSZIkSZIkSZIk1RIntr1HNCjr5cuXIaMBWHfv3sXnnJqaCtny8nLIaCDhxYsXQ/bll1+GbHR0NPV8VQO3f4uGdNEgRBqYSANsq4ZL03dLg5pfv34dMvqt9stwut2UHcpGv8/Dhw9DdvPmzZDRgMJTp06FjIaZ0jDS/TzIcC/KDvesGhZIgwWzv2d2UDMN8qQhoFTTtD7QoMiWlpaQ0UBcGq5If1tKvtb3c//c7u+j6nFUV2trayGjwaBff/11yL799tuQffPNNyGjQZ70+WiYM+0RqCap/uhv6wz81NvoN8tmNEyYhs2WUsrY2FjIqIaoD1FG+0Lq77Svo73xrVu3QjY+Ph4yGnZM+1HKqvbG2Trdz/20EdmhyqVw/VKdPnjwIGQ01Jv2ijRwm/ofve+BgYGQUY2T7MDyqppqZPi8dZpDx3hVrdI6Pzk5GbL5+fmQ0Xkx9VRav2l/S69Be1nKXr16FTJa++l6A52fVT2WenK2ppu9fun9076J+kFbW1vIqAYoK4V7ID2WeiD9jrQGU2+j/pn9HekaEb1GT09PyIaHh0NG77nqOZu91nZLtsbp2k8p3B+o/qhOqa9NT0+HjM756dxscXExZI8ePQoZ9c5z586F7NKlSyGja790DawU7gHUK6rOxfaDRs6laI2n9b0UvvZDtU97RbqOf+zYsdT7WVlZCdm///3vkNE1BMpaW1tD9vnnn4eMjreqY3gv1t/ee0eSJEmSJEmSJEna07zBJEmSJEmSJEmSpFq8wSRJkiRJkiRJkqRavMEkSZIkSZIkSZKkWuKksveIBnzRcNipqamQTUxM4HPSEFoawDY0NBSyGzduhIyGv508eTJkNGiLhmzRQEcarkkZvQZ9NspK4eFnNPyRMu1tNGT89u3bIVtYWAjZhQsXUhkN52xk2LG2XyND0ikrhXsW9Qgaskl1ubGxETIaKH737t2Q3bt3L2RU0zTsmwaX0iDPzs7OkNGg5VL4+676HjP2y7HTyPdWNUCZhnXTgNiffvopZNkBnbS/oGOBao0GPFOd0pBcGnJKxxa9Rin26a2imqTfm36fBw8e4HPSgGKqKxr8Sq+9ubkZMqp7Gqr8/fffh+yHH34I2fj4eMiuXLkSMqozUrXHpO+bjnfaW1vPb8v206rHUW+inkPrPqFzu9nZ2ZDReQ4NWqZ9BA1fprqnmqLzOhroXQrXmj1267LHPdVBKTzse25uLmRUHzTY+/jx4yGjnkV9cWxsLGR0DYPq8vDhwyGj/ejp06dDNjIyErKq58xeS8j282ZH6wl9dvqOqEdU7VFpf0avTftHqlP6bannUN/O7lnm5+dD1tXVFbLBwcGQUZ3SZytl/9TadqPfmzKqM6qfUrjW6Fy5p6cnZJOTkyF7+PBhyH788ceQ0XGzvLwcMupfn3/+eciuXbsWsk8//TRkdeqUvrPsfnQ/7weya3x231kKr8lUG3QuRftRql1ap+na6jfffBOy//3f/w0Znf/RvYf+/v6QUd+lz1vK3jxH8l8wSZIkSZIkSZIkqRZvMEmSJEmSJEmSJKkWbzBJkiRJkiRJkiSpFm8wSZIkSZIkSZIkqZY4wfA9ogFfNHhwYmIiZDQctpRS1tfXQzY0NBSyixcvhuzy5csh6+7uDhkN1coOKKQhW9lhpjSUjAai0SDKUnjQWdXweu1dNNiOBiHevXs3ZFQvNExueHg4ZFT3uz00Tm9rZJDiy5cv8TlpyGF2wPbjx49DNj09HTIarkjZzMxM6r3QYOSOjo6Q0ZBSGu5JAxOrZIfSZ3mMva1qKD31RRpAT98n1cbVq1dDRnuJ7PBcehwdh1T31MtpEHnVep4dQG+tvY1qjX6z1dXVkFXtUamHLSwshKy9vT1kt27dChntmWmYOL1HGrT817/+NfUaNHiXMtq30v62FH7f1unWZAd9V/UM2heOjIyEjOpqc3MzZFTPNLSY3jftQWi/kd3/UH+m/W3VuRR9j9k13trNofW8atj3/Px8yKinUi+i4fW0XtJw7vHx8VRGxwP1OjpG6Hug90d7mFJ4n0CPpfdDx1Oz12/2/dPxTN879Qg6hyiFv2P6e+pP9Dj6LFQvVH+09v/rX/8KGV1To73x2bNnQ3bixImQVfXUrA+xJrdbdu2vum7Z1tYWMuoj1Dvp96XrAHfu3AkZ9Xfan/zud78LGV2/pcf19vaGjK6NVn039D1af1uTrVPqm6VwnVLvpL3iP//5z5DRb05/Ozc3F7KbN2+GbGpqKmS0H6XeeenSpZDRul3VT/diTfovmCRJkiRJkiRJklSLN5gkSZIkSZIkSZJUizeYJEmSJEmSJEmSVIs3mCRJkiRJkiRJklRLY9P3aqJhhMvLyyGjocg0NK4UHhBGg9xpgBYNrKPhYtmB2TSMkD4zDR6l74E+Mw1LrhraSwNE6wwN086jGnr58mXIJiYmQkbHDf22Fy5cCFl3d3fIHG6492WH0tPQ7CdPnuBz0rDkycnJkFEN0pDD6enpkI2NjaXeD32Wzs7OkFH9Uv+jjOqc+nbV+6H1gX6X7DryIaLvI5tVoeGeNGT4+vXrqcdtbGyEjAZ0Ztd0Wr9nZ2dDRkNFaXD4mTNnQkbHQik8wL7O0F/9x6tXr0JG+7CqnkHfO6HaoLX//v37qeejfkp/S6+bHYJMn42OYTqOqh6b7QEO/34bfXb6zY4dO4Z/f+7cuZD9+c9/Dtnw8HDIaFh3tt9keyxl2X0N9WLqnUePHg1ZKbyPtp9uXbZH0O9bCv/G6+vrqeek343WfuqLjx49Sr3H48ePh4z2HLSHofWG9hdra2shq/r7qrVJ/5EdQE+9oGrdod83+zrZvkh7hIcPH4bsb3/7W8ju3bsXMroGRmsDZW1tbSHbz2vyTqHvOFtnVXl2zaNzYupNtEcgdF12dHQ0ZCMjIyGjHps9F7JOt1e2/qjf0PpZSimnT58O2ebmZshovaTrUITqmXpx9vyjq6srZHQNdnBwMGR0HlZ1DO/F+vVfMEmSJEmSJEmSJKkWbzBJkiRJkiRJkiSpFm8wSZIkSZIkSZIkqRZvMEmSJEmSJEmSJKmWOMXtPaIBhcvLyyGjodc0tKsUHirY29sbsr6+vtTfZgex04Av+nw0DJoGlNKwRRoySs9XNRCtp6cn9VgaRKndQQPmVldXQzY2NhYyGu5KAxMvX74cspaWlpDtxaFx+xn1HKoXGjpMvWRiYgJf5/bt26nH0nPScFnKlpaWQkafhXo01Wq2b9MxQkMiq3oiDV2kIaKNDPdtdtkB3vS47ODMUvg77u7uDhkNbqdB9dn3TTVEexbq0VTjMzMzqeebn58PGQ0GLaWUjo6OkNH3Vef73q+ot1Bfqvotrl+/HjLao1JdZdGgb9oz06BlqgvaL9Mxc+bMmZB1dnaGjHp2KfzdZgcwW6dvy/ZYWq9K4Zqk3nn16tWQ0bkP1TOdv2R7HZ03bWxshIzqPntOSe+vFF736ft2ePjWUa1SXyuFfzva92ZrkK5D0PNRH+vv7w/Z0NBQyGiY+ePHj0O2sLAQMnrPVXtHyrP74/0i+9mz+/Oq58vucRvZey4uLobs73//e8j++c9/hoz2A59++mnIrly5EjJaL5p9KH2zovohVFOl8HkJnRdPT0+HjM7l6belvSKdr50/fz5k1GOzazLtTbJ7parc2n1bdn9OvYDWVLp2XQqfS9F+ltZV2kscOhRvgVBG+wM6bz9x4kTILly4ELJLly6FjK7N03tppmtGzfNOJUmSJEmSJEmStCd4g0mSJEmSJEmSJEm1eINJkiRJkiRJkiRJtXiDSZIkSZIkSZIkSbXECVLvEQ1boyGtNFyThtCVwgPCaDAtPY6GZdFAN3rflNEgThpWe/PmzZDdunUrZDS8kYZ1Vg1Eo6HMXV1dIaNBYg6x217ZuqIaooF1lNHgcaqBs2fPhox+7+wwUmtlZ2SHvtIw7ImJiZD9+9//xtehx1K9zc3NpR5Hg2SzA7JpAD0NSyY0CJreMw2JrBouTQPsab2hY5E+H/XeZpIdTpzNsj2nCv2WlDUyFJf2IlSna2trIaO6oDWdXoPWhqo6pbWlzve4H2SH0FL90J7rs88+w9ehNZhqg/pkdkg97RWzQ2Op1q5duxayzz//PGSjo6MhO3nyZMio7kvh48aB9O+W7UvUH6g3lMK/BQ0tpox6C70fWpeBpTcAABITSURBVJPp3IzeIx0z2TUk2zurzjPrDAXX1mQHhVeh345qZmVlJWQ0MJ72madPn05l1Htpb0zXB2j/3tvbG7Jjx46FrBTej9Ia1kzDwndLdo9QVafZfS+h/kLXxiYnJ0NG53Z07kP7GFrnb9y4ETK6lpRdz0tp/HjXf5fdI5RSyurqasgePnwYsrGxsdTf0p53aGgoZFRDtB+gfkp7Yzo+aO2n3l51Lu51sK2h74N6Aa1NtMcshXsvPXZ9fT1kVPv0Huk87IcffghZR0dHyKjG6VzqzJkzIaOabHSN3u2adIchSZIkSZIkSZKkWrzBJEmSJEmSJEmSpFq8wSRJkiRJkiRJkqRavMEkSZIkSZIkSZKkWnZ0wjgNncsO+qsaqJodhk3DZSkj9Nr0twsLCyG7detWyL777ruQjY+Pp15jcHAwZCMjIyErhQeO0ZD6Zh80v5dU1Wl24CINp6MB3jSIrr+/P2Tnzp0LGQ1WJNkhxtnH7fbAuWZHNUS9joYE08DO6elpfJ2lpaWQ0YBieh0anE1osCMNiKWBxTQMMfs9PHr0KGT0njc2NkJWCg9bpqH21FNpeOmHKNvrKKsaQE+yQ0Qpyw7PpNeg95gdSk9DaKl/0vujrKr32pO3hr4P6lW0hlKvKoX3bFQvdNzQIGOqK+rvJDvcmPaUNKz21KlTIaPvoWqPmT02rdO3Ua3QOkZrYFWPpTqneqHHZXtT9nH0WSijfTC950brJ9tPlUO/B9UG/ZallHLs2LGQUY+hvSz1VOpDtNejOsjuMymbn58PGQ0tpz7b19cXslJ4+Djtrekz7+c+m/3sda5PZa9vZfcDtPb//PPPIZuamgoZ9e3f//73Ifv0009DNjAwELLsMVi1125kD663Zc+5qq550vn92NhYyOhaFPWbK1euhOzChQsho5qka2B0fNHno/0A9fvs8VZKvk71tuwaTz2j6pinx1L90V6RfnO6bkRrMr3v9vb2kI2OjoaMzqVojX8f1+HpuNnJfuqRI0mSJEmSJEmSpFq8wSRJkiRJkiRJkqRavMEkSZIkSZIkSZKkWrzBJEmSJEmSJEmSpFq2f6rUf0HDpWhQYFtbW8iqBmCtrKyEjAYe//jjjyGjAVg0oJgGhtFQvNu3b4fs3r17IZuYmAgZDRuj4dAXL14M2eXLl0NWCg8BpUHzDrHbGqqfqmGfNGCOBhLSEM/l5eWQ0fFA9XL69OmQ0WBFen/Zz+cQzp2RHYb97NmzkK2uroaMBiCXwv1zbm4uZDSMk/oYvW/q8Z2dnSHr7u4OGQ1IpGHQ9N3QsZQdxFsKv28aMlnnOT80NCyVBq3SwFnqiVVD6en7bGRQPaHXpj3H5ORkyG7duhUyGiBKw2ppnabao+HdpTjAe6voO6Lvkn6fqprKrqN03FAPIzRwtqurK/W69FnOnDkTMurP9Jnp+6raY1qT75atH+qdtMZT/yqF95RUV9nh7lTPtGegnjg9PR2yJ0+ehIyOj2y/rzOQfj+v5+8DfXfZAd6l8LnOqVOnQkZr64MHD0JG+2PqgcePHw8Z7bcXFxdTj6PPMTw8HLJPPvkkZAMDAyErhY9Z2idQrVvTb6Pvo9Fz4OxzUr3Mzs6GjM7XNjY2Qnb+/PmQUV3RoHqq+/dRU9bfu2X3A9T7aP0tpZSFhYWQUQ+jvd3Zs2dDRjVEfY32LLRvoHNFOqekjL6H7PUuvX91egY9lvYNtAekOsiec1G90HUo2hsPDQ2FjO4zfIjrsXcWJEmSJEmSJEmSVIs3mCRJkiRJkiRJklSLN5gkSZIkSZIkSZJUizeYJEmSJEmSJEmSVEucjvU+XwyGcdFgzr6+vpDRQK1SShkbGwvZ3//+95Ctra2FjIZw05AuGvpFA/BmZmZC9vjx45DR9zAyMhKyq1evhuyzzz4LGQ0jLYWHMtNrN/sgsd1CQwGrBtLT8MHNzc2Q0bBZGoRIA9/puKFB3zSokd43DcCrGoL8W9lBplWP1dbR70vfMQ09LIV729zcXMioL1J/oQGx2eHGVL80lJ7qkuqNhj/SsFp6jVJ4OCMdi9kBuM2ukYGz1P+Wl5dDVjWYlvoiDXOn36fq9828Ng1VvnnzZshob0L7kJMnT4aM9kU9PT0ho/1KKfun/rYb9cns91b1ODoeaG3NPq6qb/8WDXun90jHB/Vd6p3W1O7I1ikNir9//z4+58rKSsioj1DPoVqj90j9j86R5ufnQ0aD62m/Qec9VOOU0fpRSn5P5V42h76n7N6xlFIuXrwYshs3boRseno6ZP/4xz9CNjExEbLJycmQ0fFA77ujoyNkdD7/1Vdfhex//ud/Qnbp0qWQdXd3h6wU7tPZ+tW71Tm3za7pdC61tLQUMurdT548CRkdNzRsno4j2ntmz7nq9ETrb/tk94lPnz7Fv6d1mZ6ztbU1ZHRtlvYIdI2J1nR6j3SuR++FrrPR52iUa//7VfVdZntOtu/S/pj2wYTW3+y5PO0zP8T68exQkiRJkiRJkiRJtXiDSZIkSZIkSZIkSbV4g0mSJEmSJEmSJEm1eINJkiRJkiRJkiRJtcTplO/zxWAYJg3AunbtWsimpqbwOWdmZkJGQ7jHx8dDRkNeaYg7Dd+igXU0SJM+3+joaMhoQOkXX3wRsrNnz4bMYZ/vX3aYPdVFKTx8kAbMvXjxImT0m9FgRXptGnhMaCBkdhB5tn6ss8bQ90fDV2nA6+DgYMiod5ZSypkzZ0JG9UH1RsPhz507l3qN3t7ekFGd06DlbP3SGkSv0d/fH7Kq90gDnWmIY3ZAZTPJDjulz069hIa+Vg2lp/rd3NxMvQ71VOrRq6urIaMB9LOzsyGj44OOQxq+THsEqkmqvVK4zpu91nZLo99bI3sHGv5NewR6j1QbtCekYcm0rtAx06hG9hP7Rbaf0prT3t4esqqh19RPadA8rbVUG/Qes/tgGupNtUs9kdbzvr6+kNHxQXVfCvfTD3E93yn0PVENVf0etH/8y1/+EjIaNk9r8I8//hiyxcXFkNExRvvbK1euhIzO5z/55JOQUU3T56B9cCme479v2fW8FO61tKbTYHnqx3Nzc5m3iP2Orh3RsUDXxex/zYdqr+r6VFX9/hat/QsLCyGjNZ3qhdZ5ymgdoGu19BpUu9Qjq/a31vn2oe8tW3tVqM6pTp8+fRoy6rvr6+sho/qjvfXJkydDRv00u0Y3+t3sNv8FkyRJkiRJkiRJkmrxBpMkSZIkSZIkSZJq8QaTJEmSJEmSJEmSavEGkyRJkiRJkiRJkmrxBpMkSZIkSZIkSZJqObSjL3YovtyJEydCdu3atZA9f/4cn/Ojj+I9sq+//jpkDx48CNmTJ09SWWtra8h6enpCduXKlZBdv349ZL/73e9CdvXq1ZD19/eHrKOjI2Qff/xxyEop5eDBgyE7cOAAPlb/3a+//rrlrJRSXr16FbI3b96EjOq5vb09ZIcPH8bX+a2WlpYt/22WNbUz6Him3/fUqVMhu3HjRsiOHTuGrzM4OBiyxcXFkFGtU3/q6+sL2cDAQMiOHz8eMvp8hN4LHV+0BtFrtLW14evQd5Y9xujY/hBRndL3Qd8xrbVV39vy8nLIxsfHQzY5ORmylZWVkD179ixk1Lfp81Ht0ppOx+FXX30VsosXL4aM9hxVxwfVuX16d1Bv+uWXX0JG/Yrqj/6W9oDUi1++fBmyI0eOpDKqe1JnX6StoWOZfjPqGXR+VQr3RFr3Hz58mPpbqjVCtdvV1RWy3t7ekI2MjISMzsMa3W9QP90v6/lOoe+zzrktrZn0u//lL38JGe0laD9A75H2hHRdg/oxHbPZWqtaz13nt092LaM1uRRevzc2NkJGfXZmZiZkdB2Mrg9QrZ0+fTr1ODrm7HV7Gx3z9JtRvymFz8Wox9L1UarTzc3N1Huk16U9y/DwcMiOHj0aMqpneo06NW4/3XlV5wvUZ1+/fh2yFy9ehGxtbS2V0WtQrXV2doYs208buTbfTOdSrhqSJEmSJEmSJEmqxRtMkiRJkiRJkiRJqsUbTJIkSZIkSZIkSarFG0ySJEmSJEmSJEmqJU6TfI9oiBoNVaUh81VD14eGhkJ2/fr1kN27dy9kCwsLIaNByzTM68yZMyGj4bLnz58PGQ2rzQ4HqzO82+F02yf7XVY9jobcU+3T4GEafE91SsPfaJgc1Rq9Fzpes8M+rb3tlx2CTP2K+icNfS2llM8//zxkNLCb6o36E9U+ZY0M0m6k3urUOeV0jDUyxLGZZIfL0u9Nw4mpJquGWjZSf5OTkyGjod503NDA2XPnzoXsxo0bIRsdHQ1Zf39/yBoZCF7Kh1lre12jw1ezf091QH9LawPtG6iGqO6zPZtqzz3q9qLvjX4L2k+OjIzgc2b/noZwz87Ohmx9fT1ktC6eOHEiZAMDAyG7fPlyyOici9YQ2hPV6ae0plm771/Vd0y/E9UWnddQvdGa3kg/z/bARs8rtX3o96YsO2i+FD5v2tjYCNnKykrqcbQGUz2fPHkyZNS3aZ1vdE3X+5U958peGyiF18ylpaWQLS8vh2xxcTFkdG2VdHd3h6yvry9kFy9eDBntB+hzZK+t0vpRimv/bqhae+n8hXrv06dPUxn1Z9oXHj16NGR0/YL+lnp2tn7q7EHosbtdp/4LJkmSJEmSJEmSJNXiDSZJkiRJkiRJkiTV4g0mSZIkSZIkSZIk1eINJkmSJEmSJEmSJNVy4B1DpBqbWrxF9J5ouFcpPODr1atXIaNhXvQ4GuBIw99oSFx2CHJ2mH0TD1bc6Te5Z+q06niiuqKapsdlB71lH0f1nB2a2yT1l7UbH2bP12r27/eS7a7LRp/vPRwnTdNTs7WWHdj54sULfJ3Nzc2Qra2tpTL6W+q9tM7TcE8ankuPa21tTb0G9d4mGTbbNHW63eqs/VTnjexls89H7zG7v80OsM3uJUrhmib7uZ+mXyA5kJ5qqpRSnj9/HrLV1dWQ0aDv9fX1kFHfpt+bBih3dXWFjPop/W22Juv0013ss/tmj6qm19Q9tZF9K63JpXBffPz4ccgePnwYsqmpqZA9e/YsZLQu9/T0hGxgYCBkvb29ITt+/HjI2traQmZPTdsza39VnVJdPXnyJGTz8/OpbGVlJWRULydPngwZ1eSpU6dCRudcdCxYp2m7su5TnVJWCu9daZ9J9Uf7Vro2QOdNjZwP0Tn/sWPHUln2PkMp+eu1O3ku5b9gkiRJkiRJkiRJUi3eYJIkSZIkSZIkSVIt3mCSJEmSJEmSJElSLd5gkiRJkiRJkiRJUi0H3jG83WGf2op9MZxOTc8BymoW+6KnZgct13lsnefMyA7O3O6sSeyLOq0jW39Vg2238/kaqb/sYOQmqed9UaeN9tPsAOZG+mm2rhqpP7IHa5K4R1Wz+OB6KvW6N2/ehOzly5f49+vr6yGjYfOLi4upx9EAehoi39nZGbITJ06kHpcdNn/o0KGQZXv0f8t3wAdXp1m7eS5Fj9ut8ybXfvTe67TRcyHqf9R7Nzc3Q/b06dOQPXv2LGRUG9T/Dh48GDLqf4cPHw4Z9ewjR46EjHosvW7Va+/QuVjlE/ovmCRJkiRJkiRJklSLN5gkSZIkSZIkSZJUizeYJEmSJEmSJEmSVIs3mCRJkiRJkiRJklTLgXcMaNszw+nUVD644XT6IDlAWc3CnqpmYJ2qGVin2yg76LtJhmvvJe5R1Sw+uJ6aHUD/5s0b/PtXr16F7MWLFyF7/vx56nH0frJD6WmIfHawfHbY/A4NlW/UB1en+iDtizrN7h2rHkv9uJEs268aedxHH8V/20NZnX66i3228oX9F0ySJEmSJEmSJEmqxRtMkiRJkiRJkiRJqsUbTJIkSZIkSZIkSarFG0ySJEmSJEmSJEmq5cA7Bmw5nE5bsS+G06npOUBZzcKeqmZgnaoZWKdqBu5R1Sz2bU+tM6g++/fZ52zktbND5OsMm28C+7ZO1VSsUzWDyjr1XzBJkiRJkiRJkiSpFm8wSZIkSZIkSZIkqRZvMEmSJEmSJEmSJKkWbzBJkiRJkiRJkiSplgONDieUJEmSJEmSJEnS/uK/YJIkSZIkSZIkSVIt3mCSJEmSJEmSJElSLd5gkiRJkiRJkiRJUi3eYJIkSZIkSZIkSVIt3mCSJEmSJEmSJElSLd5gkiRJkiRJkiRJUi3/D4Y2Xnl7bgZsAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 2160x216 with 10 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_images(digit_means)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The below function return the L1 distance for each image from the given mean image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mnist_distance(inputs,means):\n",
    "    '''\n",
    "    Arguments:\n",
    "        inputs: Tensor of shape Nx28x28\n",
    "        mean: mean tensor of shape 10*28x28\n",
    "    Outputs: \n",
    "        Tensor of shape Nx10\"\n",
    "    '''\n",
    "    distances = [(inputs - m).abs().mean((-1,-2)) for m in means] \n",
    "    return torch.stack(distances).transpose(0,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(distances):\n",
    "    '''\n",
    "    Arguments:\n",
    "        distances: Nx10 distance matrix\n",
    "    Output:\n",
    "        prediction corresponding to the lowest distance; Tensore of size N\n",
    "    '''\n",
    "    return (distances.min(dim=1)[1]) \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using these two functions, we can get the predictions for the validations set. Note that the true label for each input is the index of the parent list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_predictions = [predict(mnist_distance(valid_input,digit_means)) for valid_input in valid_tensors]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(valid_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy = np.array([(prediction==i).float().mean().item() for (i,prediction) in enumerate(valid_predictions)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The classwise accuracy of the model is as follows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:0.8245\n",
      "1:0.9993\n",
      "2:0.4077\n",
      "3:0.6161\n",
      "4:0.6732\n",
      "5:0.3060\n",
      "6:0.7475\n",
      "7:0.7757\n",
      "8:0.4082\n",
      "9:0.6924\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    print(\"{}:{:.4f}\".format(i,accuracy[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The  overall accuracy of the baseline model is equal to 0.6451\n"
     ]
    }
   ],
   "source": [
    "print(\"The  overall accuracy of the baseline model is equal to {:.4f}\".format(accuracy.mean()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this section, a simple linear model is created to classify the digits. The simple linear model will just have an input layer with 784 units corresponding to each pixel and an output layer with 10 units corresonding to each image class."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We first define a function to intialise the weights and bias terms of the model that can be optimized iteratively."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_params(size,std=1.0): return (torch.randn(size)*std).requires_grad_()\n",
    "weights = init_params((28*28,10))\n",
    "bias = init_params(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The linear model is defined as shown below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_model(xb):\n",
    "    return xb@weights + bias"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The softmax function defined below, takes a list of values and normalizes it so that each value is between 0 and 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(inputs):\n",
    "    exps = torch.exp(inputs.float())\n",
    "    return exps/exps.sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The loss function is the cross entropy loss which is the appropriate loss function for multi class classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy_loss(inputs,targets): \n",
    "    inputs = inputs.softmax(dim=1)\n",
    "    return -(inputs[range(len(inputs)),targets]).log().mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can create the equivalent of a Dataset in Pytorch that return a tuple of the form (x,y).First we stack all the tensors into a single tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([47995, 784]),\n",
       " torch.Size([47995, 1]),\n",
       " torch.Size([12005, 784]),\n",
       " torch.Size([12005, 1]))"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x = torch.cat(train_tensors).view(-1,28*28)\n",
    "valid_x = torch.cat(valid_tensors).view(-1,28*28)\n",
    "train_x.shape,train_y.shape,valid_x.shape,valid_y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now create a dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dset = list(zip(train_x,train_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([784]), tensor([0]))"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x,y = train_dset[0]\n",
    "x.shape,y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can now create a Pytorch dataloader with this dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([256, 784]), torch.Size([256, 1]))"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dl = DataLoader(train_dset,batch_size =256)\n",
    "xb,yb = first(train_dl)\n",
    "xb.shape,yb.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Also create a data loader for the validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_dset  = list(zip(valid_x,valid_y))\n",
    "valid_dl = DataLoader(valid_dset,batch_size = 256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([47995, 784]), torch.Size([784, 10]))"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x.shape,weights.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample run "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below we step through the training process with a sample batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([256, 784]), torch.Size([256, 1]))"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch,labels = first(train_dl)\n",
    "batch.shape,labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([256, 10])"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = linear_model(batch)\n",
    "preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(10.8375, grad_fn=<NegBackward>)"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss = cross_entropy_loss(preds,labels)\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([784, 10]), tensor(-3.6493e-10), tensor([-1.1176e-08]))"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss.backward()\n",
    "weights.grad.shape,weights.grad.mean(),bias.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Carry out one step of gradient descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 1e-3\n",
    "weights.data -= lr * weights.grad\n",
    "bias.data -= lr * bias.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "See if loss reduced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(10.7605, grad_fn=<NegBackward>)"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds = linear_model(batch)\n",
    "cross_entropy_loss(preds,labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights.grad = None\n",
    "bias.grad = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can put the above steps into a single function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_grad(xb,yb,model):\n",
    "    preds = model(xb)\n",
    "    loss = cross_entropy_loss(preds,yb)\n",
    "    loss.backward()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check if this works"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(-1.8246e-10), tensor([-1.8626e-09]))"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc_grad(batch,labels,linear_model)\n",
    "weights.grad.mean(),bias.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And write a function to carry out model training for one epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_epoch(model,lr,params):\n",
    "    for xb,yb in train_dl:\n",
    "        calc_grad(xb,yb,model)\n",
    "        for p in params:\n",
    "            p.data -= p.grad*lr\n",
    "            p.grad.zero_()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we also write a function to calculate validation accuracy for a batch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_accuracy(preds,yb):\n",
    "    predictions = preds.max(dim=1)[1].unsqueeze(1)\n",
    "    yb = yb.squeeze()\n",
    "    return (predictions==yb).float().mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can now create a function to calculate the accuracy for each epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate_epoch(model):\n",
    "    accs = [batch_accuracy(model(xb),yb) for xb,yb in valid_dl]\n",
    "    return round(torch.stack(accs).mean().item(),4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1053"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validate_epoch(linear_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now examine if accuracy improves by training for one epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1859"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr = 0.1\n",
    "params = weights,bias\n",
    "train_epoch(linear_model,lr,params)\n",
    "validate_epoch(linear_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Try running for multiple epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2785 0.3548 0.4105 0.4512 0.4807 0.5047 0.5239 0.5398 0.5513 0.563 0.5727 0.5799 0.5856 0.5901 0.5946 0.598 0.602 0.606 0.6091 0.6106 "
     ]
    }
   ],
   "source": [
    "for i in range(20):\n",
    "    train_epoch(linear_model,lr,params)\n",
    "    print(validate_epoch(linear_model),end = ' ')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating an Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([10, 784]), torch.Size([10]))"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linear_model2 = nn.Linear(28*28,10)\n",
    "w,b = linear_model2.parameters()\n",
    "w.shape,b.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BasicOptim:\n",
    "    def __init__(self,params,lr): self.params,self.lr = list(params),lr\n",
    "    def step(self,*args,**kwargs):\n",
    "        for p in self.params: p.data -= p.grad.data *self.lr\n",
    "    def zero_grad(self,*args,**kwargs):\n",
    "        for p in self.params: p.grad = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.1\n",
    "opt = BasicOptim(linear_model2.parameters(),lr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Updated training loop is now given by"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_epoch(model):\n",
    "    for xb,yb in train_dl:\n",
    "        calc_grad(xb,yb,model)\n",
    "        opt.step()\n",
    "        opt.zero_grad()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1082"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validate_epoch(linear_model2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model,epochs):\n",
    "    for i in range(epochs):\n",
    "        train_epoch(model)\n",
    "        print(validate_epoch(model), end= ' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1011 0.1834 0.2842 0.3441 0.3866 0.4137 0.4381 0.4557 0.4715 0.4824 0.4917 0.5005 0.5081 0.5146 0.5206 0.5251 0.5296 0.5334 0.5379 0.5412 "
     ]
    }
   ],
   "source": [
    "train_model(linear_model2,20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now using the SGD optimizer provided by fast ai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1011 0.1784 0.2805 0.344 0.3854 0.4135 0.4375 0.4545 0.4699 0.4829 0.4908 0.4981 0.5056 0.5131 0.5184 0.5241 0.5281 0.5337 0.5382 0.5417 "
     ]
    }
   ],
   "source": [
    "linear_model3 = nn.Linear(28*28,10)\n",
    "opt = SGD(linear_model3.parameters(),lr=0.1)\n",
    "train_model(linear_model3,20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now create a DataLoaders class by passing in both the training and valdiation sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "dls = DataLoaders(train_dl,valid_dl)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can use the Learner class to avoid writing any boiler plate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(dls,nn.Linear(28*28,10),opt_func = SGD,\n",
    "                loss_func = cross_entropy_loss, metrics = batch_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>batch_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.367332</td>\n",
       "      <td>4.238232</td>\n",
       "      <td>0.099125</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.281678</td>\n",
       "      <td>3.116744</td>\n",
       "      <td>0.167177</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.259381</td>\n",
       "      <td>2.690326</td>\n",
       "      <td>0.276394</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.251433</td>\n",
       "      <td>2.460329</td>\n",
       "      <td>0.340662</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.247416</td>\n",
       "      <td>2.311152</td>\n",
       "      <td>0.384212</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.245038</td>\n",
       "      <td>2.203323</td>\n",
       "      <td>0.411633</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.243515</td>\n",
       "      <td>2.120255</td>\n",
       "      <td>0.435468</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.242472</td>\n",
       "      <td>2.053680</td>\n",
       "      <td>0.454908</td>\n",
       "      <td>00:08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.241703</td>\n",
       "      <td>1.998873</td>\n",
       "      <td>0.469478</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.241097</td>\n",
       "      <td>1.952861</td>\n",
       "      <td>0.481165</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit(10,lr=lr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adding a Non Linearity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "simple_net = nn.Sequential(\n",
    "    nn.Linear(28*28,300),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(300,10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = Learner(dls,simple_net,opt_func=SGD,loss_func=cross_entropy_loss,metrics=batch_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>batch_accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.752031</td>\n",
       "      <td>2.590733</td>\n",
       "      <td>0.099125</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.216744</td>\n",
       "      <td>2.557050</td>\n",
       "      <td>0.100554</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.899474</td>\n",
       "      <td>2.280073</td>\n",
       "      <td>0.215216</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.736832</td>\n",
       "      <td>2.078324</td>\n",
       "      <td>0.312774</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.646674</td>\n",
       "      <td>1.946730</td>\n",
       "      <td>0.356592</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.590854</td>\n",
       "      <td>1.852450</td>\n",
       "      <td>0.383172</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.552992</td>\n",
       "      <td>1.778626</td>\n",
       "      <td>0.404523</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.525687</td>\n",
       "      <td>1.717293</td>\n",
       "      <td>0.424119</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.505122</td>\n",
       "      <td>1.664572</td>\n",
       "      <td>0.443226</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.488980</td>\n",
       "      <td>1.618474</td>\n",
       "      <td>0.464506</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.475960</td>\n",
       "      <td>1.577802</td>\n",
       "      <td>0.481939</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>0.465254</td>\n",
       "      <td>1.541339</td>\n",
       "      <td>0.499504</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>0.456265</td>\n",
       "      <td>1.508550</td>\n",
       "      <td>0.513989</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>0.448556</td>\n",
       "      <td>1.478963</td>\n",
       "      <td>0.528326</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>0.441856</td>\n",
       "      <td>1.452126</td>\n",
       "      <td>0.540860</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>0.435917</td>\n",
       "      <td>1.427778</td>\n",
       "      <td>0.552100</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>0.430628</td>\n",
       "      <td>1.405506</td>\n",
       "      <td>0.561430</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>0.425827</td>\n",
       "      <td>1.384878</td>\n",
       "      <td>0.570112</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>0.421518</td>\n",
       "      <td>1.365782</td>\n",
       "      <td>0.579636</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>0.417600</td>\n",
       "      <td>1.347943</td>\n",
       "      <td>0.587805</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>0.413949</td>\n",
       "      <td>1.331389</td>\n",
       "      <td>0.595636</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>0.410551</td>\n",
       "      <td>1.315955</td>\n",
       "      <td>0.602979</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>0.407346</td>\n",
       "      <td>1.301578</td>\n",
       "      <td>0.609633</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>0.404322</td>\n",
       "      <td>1.287936</td>\n",
       "      <td>0.616356</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>0.401483</td>\n",
       "      <td>1.274906</td>\n",
       "      <td>0.623330</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>0.398803</td>\n",
       "      <td>1.262360</td>\n",
       "      <td>0.629566</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>0.396288</td>\n",
       "      <td>1.250299</td>\n",
       "      <td>0.634948</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>0.393818</td>\n",
       "      <td>1.238869</td>\n",
       "      <td>0.640489</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>0.391454</td>\n",
       "      <td>1.227755</td>\n",
       "      <td>0.645517</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>0.389166</td>\n",
       "      <td>1.217013</td>\n",
       "      <td>0.650609</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>0.386982</td>\n",
       "      <td>1.206528</td>\n",
       "      <td>0.655969</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>0.384854</td>\n",
       "      <td>1.196366</td>\n",
       "      <td>0.660713</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>0.382770</td>\n",
       "      <td>1.186411</td>\n",
       "      <td>0.666102</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>0.380672</td>\n",
       "      <td>1.176713</td>\n",
       "      <td>0.669561</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>34</td>\n",
       "      <td>0.378664</td>\n",
       "      <td>1.167043</td>\n",
       "      <td>0.674265</td>\n",
       "      <td>00:06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>35</td>\n",
       "      <td>0.376654</td>\n",
       "      <td>1.157752</td>\n",
       "      <td>0.678219</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36</td>\n",
       "      <td>0.374640</td>\n",
       "      <td>1.148710</td>\n",
       "      <td>0.682188</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>37</td>\n",
       "      <td>0.372689</td>\n",
       "      <td>1.139665</td>\n",
       "      <td>0.687388</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38</td>\n",
       "      <td>0.370734</td>\n",
       "      <td>1.130794</td>\n",
       "      <td>0.692103</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>39</td>\n",
       "      <td>0.368828</td>\n",
       "      <td>1.122074</td>\n",
       "      <td>0.696841</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit(40,0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fa54e89dbd0>]"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD7CAYAAACRxdTpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO3deXxddZ3/8dcnTdqm2dMspU3TdElbutOmbAWpgAiKFKkKAyIgWC04jr8ZdZxxcBDHFUedmQfiDxVBdnGKooggqxSErqRtSptA26TpkqVJbrZmu/nOH/e2xpg0t+lNzl3ez8cjD8j3njTvx7fpu6ffc873mnMOERGJLQleBxARkfBTuYuIxCCVu4hIDFK5i4jEIJW7iEgMSvQ6AEBOTo4rKiryOoaISFTZvHlzvXMud6DXIqLci4qK2LRpk9cxRESiiplVDvaalmVERGKQyl1EJAap3EVEYpDKXUQkBoVU7maWbWZPmlmbmVWa2bWDHPdjM2vt89FpZi3hjSwiIkMJ9W6Zu4EuIB9YAjxtZqXOubK+BznnPgN85tjnZnY/0BueqCIiEqohz9zNLAVYDdzunGt1zq0HngKuD/HrHghHUBERCV0oZ+6zAb9zrrzPWClwwRBftxqoA/400ItmtgZYA1BYWBhCDBGR2NDU3sX2Az62H/CxaEom5xXnhP17hFLuqYCv35gPSBvi624AfuEG2TDeOXcvcC9ASUmJNpUXkZjka+8+XuQ7DvjYdqCJ/Q1Hj7++duVMz8q9FUjvN5YODHqh1MymEjiz/9Two4mIRBdfezc7DgaKfHt14L9VDe3HX5+anczCKRlce+Y0Fk7JYMGUdDInjB2RLKGUezmQaGbFzrmK4NhioOwEX/MJ4HXn3J5TDSgiEonaOnvYVu2jtLppwCIvyAoU+dXLp7KoIIMFkzPIShmZIh/IkOXunGszs3XAnWZ2C4G7ZVYB557gyz4BfCc8EUVEvOWcY099G1urmthS1cjWqiZ2H26mN7ig7HWRDyTUWyFvBe4DaoEjwFrnXJmZFQI7gXnOuSoAMzsHKACeGIG8IiIjqqPbzzu1rVTUtrD7cCu7DjeztaoJ39FuANLGJbKkMJP3XVjMGYWZLCnI9LzIBxJSuTvnGoArBxivInDBte/Yn4GUsKQTERkhHd1+9tS1UVHbQnlNC+U1rVTUtFDV0H78jDxpjDEzN5XLFkzijMJMlhZmMTM3lYQE8zZ8CCJiy18RkZHS2eNnb33b8fIur2mhoqaVfUfajpd4YoJRlJPCvMnprFoyhdn5aczOT6UoJ4WkMdG5S4vKXURiRke3n92HW9h2wMeO4EXO8poWeoItPibBmDZxArPz07h80WkU56cxOz+N6TkpjE2MzhIfjMpdRKKOv9dxoPEoe+pb2VffxtuHWv6myDMnJLFwSgafmjODuZMCJT4jN4VxiWM8Tj86VO4iErG6enopO+hj56Fm9tW3sTf4UdXQTrf/L88+Zk1IYsGUDNbMmRG8fzyDgqxkzCJ/bXykqNxFJGI0tHWxpbKRTZWNbKlspLS6ic6ewN6D4xITKJqYQnFeGpfMn8T0iSlMz02haGIKOalj47rIB6JyFxFPdPb42RVcTind38Tmqkb21LUBgQuc86dk8PGzp7FsWhaLCjKYnJEcFXepRAqVu4iMuM4eP+WHW9l2oCmwv0p1YH382NJK1oQklhZm8ZFlBSwrzGLx1EzGJ8XH2vhIUbmLSNjVtnSwpbKRzcGPHQea6fIHllcykpNYVJDBLecH1scXan18RKjcReSU9Ph72V3TwpaqpuB6ecPxXQ/HJiawuCCDm1YUsaggk0UFKvLRonIXkZNy2NfB1qpG3trfxNb9gU2zjnb7AchNG0fJtCxuOKeIpdOymD85PW5uPYw0KncRGVRbZw/bD/h4a38Tb1U18db+Jg43dwAwdkwC8yanc/XyqccfzddZeeRQuYsIEFheqahtPV7kpdVNlNe0HH9EvzB7AmfNyGbJ1EyWTM1kns7KI5rKXSROtXR0s7WqiU37Gti4L7DMcmx5JXNCEosLMnn//EksmZrJ4qmZZEfgzocyOJW7SJyoae5g474GNu1rZOO+Bt4+FNiPPMFg/uTAXuTHzsqnTZyg5ZUop3IXiUH+Xseuw83Hn/bcXNlIdWPgDpbkpDEsnZbJ319YzPKibJYUZpI6TlUQa/Q7KhID2rt62FzZyKZ9gSLfWtVIW1dgiSUvbRwlRVnceG4RZ07P5vTT0qN2G1sJncpdJAr1+Hsprfbx2jv1vPZOPVuqGun2OxIM5k5KZ/WyApZNy9IdLHFM5S4SBZxzvFPbyvpgmb+xp4HWzh7MYP7kdD553nRWzMxh6bQsLbEIoHIXiVgd3X7e2HOEl3bV8sKu2uNr5tMmTuCKJZM5b1YO58yYGJHv3yneU7mLRJCa5g5e3FXLi7tqWV9Rz9FuP+OTEjhvVg5rV87kPcW5TM2e4HVMiQIqdxEP9fh72bq/iZd31/Ly7jrKDjYDMCUzmY8sK+DCuXmcM3OidkiUk6ZyFxlltc0dvFxexyu763i1oo7mjh7GJBjLCrP44vvncNHpeczJT9NFUDklKneREdbt72VLZSOvlNfx8u46dh4KnJ3np4/j0gWTWDknjxWzcshITvI4qcSSkMrdzLKBnwGXAPXAvzjnHhnk2BnAfwMXAJ3Afc65L4Unrkh0ONB0lFd21/FKeS2vvXOE1s4eEhOMpdOy+NKlc1g5O4/TT9PZuYycUM/c7wa6gHxgCfC0mZU658r6HmRmY4E/Bo+/GvADs8MXVyQy9fh72bivkRferuGV8joqalsBmJwxng8tnswFs3NZMWsiaeN1di6jY8hyN7MUYDWwwDnXCqw3s6eA64Ev9zv8RuCgc+77fca2hSmrSERp7+rhT+V1PLezhhd31dLU3s3YMQmcNSObq5dPZeWcXGbmpursXDwRypn7bMDvnCvvM1ZKYNmlv7OBfWb2DLAc2AH8vXNue/8DzWwNsAagsLDwZHOLeKK+tZMX367luZ2HebWins6eXjKSk7jo9DwumZfP+cW5pOghIokAofwUpgK+fmM+IG2AYwuA9wJXAC8A/wD8xszmOue6+h7onLsXuBegpKTEnWRukVFT29LBszsO8/T2Q2zY20CvC9yqeO1ZhVwybxLLi7JI1F4tEmFCKfdWIL3fWDrQMsCxR4H1zrlnAMzse8C/AacTONsXiQq1zR08Eyz0jfsacA5m5aXy2QuLuXT+JF0MlYgXSrmXA4lmVuycqwiOLQbKBjh2G7AiXOFERlNdSye/336Ip7cdYmNloNBn56fyDxcV88GFp1GcP9A/VkUi05Dl7pxrM7N1wJ1mdguBu2VWAecOcPhDwD+Z2cXAS8DnCNw6+Xb4IouEj6+9mz+UHeKp0oP8+d0j9DqYk5/G5y+azQcXTWJWngpdolOoV35uBe4DaoEjwFrnXJmZFQI7gXnOuSrn3G4z+zjwYyAP2AJc0X+9XcRLbZ09PP92Db8tPcgr5XV0+x1FEyfw2ffO4kOLJ+sMXWJCSOXunGsArhxgvIrABde+Y+uAdWFJJxImzjne3NvAoxuqeLbsMB3dvUxKH8+N5xZxxeIpLJiSrjV0iSm6Z0tiWlN7F7/aXM2jG6p4t66NtPGJrF5awBWLJ7O8KJuEBBW6xCaVu8Qc5xybKxt55M0qfrf9EF09vZxRmMldH1nE5YsmkzxWOyxK7FO5S8yob+3kqbcO8vjG/eyuaSF1XCIfKyng2jOnMW9y/7t5RWKbyl2iWke3nxd31bJuSzUv766jp9excEoG37pqIVcsnqynRSVu6Sdfoo5zji1VTazbUs1vSw/S3NFDXto4bj5vOlctLWDOJN3tIqJyl6hR29zBE5ur+dXmavbWtzE+KYFL50/iqqUFrJiVwxhdHBU5TuUuEc3f6/hTeR2PbqjihV21+HsdZ03P5taVM7ls4WmkatlFZED6kyER6UDTUX65cT9PbNrPQV8HE1PGcst507l6+VRm5KYO/QuIxDmVu0QM5xx/fvcIP3l1Dy+X1+EcnF+cw79dPo+LT89nbKJ2XhQJlcpdPNfb63hhVy13v/QOb+1vIid1HLetnMXVy6cyNXuC1/FEopLKXTzT4+/l6e2H+NFL77K7poWCrGT+48oFfGRZAeOT9KCRyKlQucuo6+zx87+bD/DjV96lqqGd4rxUfnD1Yi5fNJkkvemFSFio3GXUdPt7eWJTNf/9QgWHmztYVJDBVz64jPednq89XkTCTOUuI6631/H7HYf4z+fK2VvfxtLCTO766CLOm5WjnRhFRojKXUaMc44/VdRz17O72HGgmTn5afz0EyVcdHqeSl1khKncZURsqWrku3/YxRt7GijISub7H1vMqiVT9BSpyChRuUtYVTe2842n3+aZHYfJSR3L166YzzVnTmVcou5+ERlNKncJi66eXn66fg///UIFhvH/Lp7NLedP166MIh7Rnzw5Za+/W8/tv97Bu3VtXDIvn69+aB4FWXr4SMRLKncZttrmDr7x+7f5zVsHmZqdzH03lnDh3HyvY4kIKncZhh5/Lw++Ucn3nyuns6eXz11UzK0rZ+qpUpEIonKXk7K1qpGvPLmDnYeaOb84hztXLWB6TorXsUSkH5W7hMTX3s13nt3FoxuqyEsbx93XLuUDCyfpfnWRCBXSRh5mlm1mT5pZm5lVmtm1gxx3o5n5zay1z8fKsCaWUeWc4383V3Phf77MYxuquOnc6Tz/jxfwwUWnqdhFIlioZ+53A11APrAEeNrMSp1zZQMc+2fn3HnhCijeeae2hX/79Q7e2NPAGYWZ/OLmM5k/OcPrWCISgiHL3cxSgNXAAudcK7DezJ4Crge+PML5xANHu/z8z4sV/OTVPUwYm8g3P7yQa5ZP1eZeIlEklDP32YDfOVfeZ6wUuGCQ488ws3qgAXgQ+JZzrqf/QWa2BlgDUFhYeFKhZeRU1LRw2yNbKK9pZfXSAv7lA3PJSR3ndSwROUmhlHsq4Os35gPSBjj2T8ACoBKYDzwO9ADf6n+gc+5e4F6AkpISF3pkGQnOOZ7YXM1Xf7ODlLGJPPDJM7lgdq7XsURkmEIp91Ygvd9YOtDS/0Dn3J4+n243szuBLzJAuUvkaOvs4fZf72Dd1gOcM2Mi/3XNEvLSx3sdS0ROQSjlXg4kmlmxc64iOLYYGOhian8O0EJtBHv7UDO3PbKFvfVtfP7iYv7+wmLt3CgSA4Ysd+dcm5mtA+40s1sI3C2zCji3/7FmdhmwxTlXY2ZzgduBJ8KcWcLAOccjG6r42m93kpGcxMO3nMW5M3O8jiUiYRLqrZC3AvcBtcARYK1zrszMCoGdwDznXBVwEXC/maUCNcBDwDfDH1tORUtHN/+ybju/23aI84tz+MHVS3TRVCTGhFTuzrkG4MoBxqsIXHA99vkXgC+ELZ2E3bbqJj77yFYONB3li++fw9oLZuoWR5EYpO0H4oRzjvte28e3n3mbnNRxPLbmbJYXZXsdS0RGiMo9DjS2dfHFX5Xy/Nu1XHx6Pnd9ZBFZKWO9jiUiI0jlHuM27mvgc49upb61k69ePo+bVhRpTxiROKByj1H+Xsc9L7/DD56voCArmXVrV7CwQPvCiMQLlXsM8rV3c9sjW1j/Tj1XLJ7MNz68gLTxSV7HEpFRpHKPMYd8R7nhvg3sq2/n21ct5OrlU7UMIxKHVO4xpKKmhRvu20BzRw/337Scc2fpoSSReKVyjxGbKxv45P2bGJuYwOOfPlv7rovEOZV7DHh+Zw23PbKFyZnJ/OKTZzI1e4LXkUTEYyr3KPf4xir+9ckdLJiczn03LmeithEQEVTuUcs5x90vvcP3nivnPbNzuee6paSM02+niASoDaJQt7+Xr/22jIfeqOLDZ0zhO6sXMTYxpPc6F5E4oXKPMg1tXdz28Bb+vOcIn37PDP750rna+EtE/obKPYrsPNjMmgc3UdvSyX9+dDGrlxV4HUlEIpTKPUo8ve0QX3iilIzkJJ749DksnprpdSQRiWAq9wjn73V8/4+7ufuld1k2LYt7Pr6UvDS9v6mInJjKPYI1d3Tz+cfe4sVdtVyzfCpfWzWfcYljvI4lIlFA5R6h3q1r5VO/2ETVkXa+vmo+Hz97mvaIEZGQqdwj0J/K67jtkS2MHZPAQ7ecxdkzJnodSUSijMo9gjjnuP/1fXz9dzuZnZ/GTz5Roq0ERGRYVO4Rotvfy1d/U8ajG6p437x8fnj1Ej1xKiLDpvaIAI1tXax9eDNv7Glg7cqZfPGSOXowSUROicrdYxU1Ldz8wCYON3fwg6sX8+Ez9GCSiJy6kDYkMbNsM3vSzNrMrNLMrg3ha140M2dm+gtkEC/truWqH71Oe5efx9acrWIXkbAJtXjvBrqAfGAJ8LSZlTrnygY62MyuO4lfOy498Po+vvbbMuZOSuenN5QwOTPZ60giEkOGPHM3sxRgNXC7c67VObceeAq4fpDjM4B/B74UzqCxwjnHXc/u4t+fKuOi0/P51dpzVOwiEnahnF3PBvzOufI+Y6XABYMc/03gHuDwiX5RM1sDrAEoLCwMIUb06/H38q9PbueXm6r5uzML+fqq+SSO0Va9IhJ+oTRLKuDrN+YD0vofaGYlwArgf4b6RZ1z9zrnSpxzJbm5uaFkjWpHu/x85qHN/HJTNZ+7qJhvfniBil1ERkwoZ+6tQHq/sXSgpe+AmSUAPwL+wTnXo0fl/6KpvYubH9jElqpGvn7lAq4/e5rXkUQkxoVS7uVAopkVO+cqgmOLgf4XU9OBEuDxYLEf2+Gq2sw+6px7NRyBo83BpqPccN8GKo+0c/e1S/nAwtO8jiQicWDIcnfOtZnZOuBOM7uFwN0yq4Bz+x3qAyb3+XwqsAFYBtSFJ250qahp4RP3baC1o4cHPnkm58zUHjEiMjpCvV3xVuA+oBY4Aqx1zpWZWSGwE5jnnKuiz0VUMzu26XiNc64njJmjwo4DPq776ZuMTUzg8U+fw7zJ/Ve2RERGTkjl7pxrAK4cYLyKwAXXgb5mHxCXC+/7G9q58ecbSR2XyGNrztbmXyIy6vSgUZg1tnVxw8830O3v5bE1Z6nYRcQTKvcw6uj2c/MDG6luPMrDt5zFrLy/uVtURGRU6EbrMPH3Oj736Fa27m/iv65ewvKibK8jiUgcU7mHgXOOO54q47mdNXz18nlcptsdRcRjKvcwuOeVd3nwjUo+/Z4Z3LRiutdxRERU7qdq3ZZqvvuH3VyxeDL/fOlcr+OIiAAq91PyakUdX/rVNs6ZMZG7PrpI754kIhFD5T5Mh3xHWfvQFmblpfL/P7GMcYljhv4iEZFRonIfph/+sYKunl7uvb6E9PFJXscREfkrKvdhqKhp4YnN+7n+nGkUTtRDSiISeVTuw/DdZ3eTMjaR2947y+soIiIDUrmfpE37Gvjjzho+s3Im2SljvY4jIjIglftJcM7x7Wd2kZc2jptWFHkdR0RkUCr3k/D827Vsqmzk8xfPZsJYbcsjIpFL5R6iHn8v3/3DLmbkpPCxkgKv44iInJDKPUTrthygoraVL106R29sLSIRTy0Vgo5uP9//YzlLpmby/vmTvI4jIjIklXsIHnh9H4ebO/jyZXMJvvm3iEhEU7kPwdfezd0vvcOFc/M4e4be4FpEooPKfQg/euUdWjp7+NKlc7yOIiISMpX7CRxsOsrPX9vHh8+YwtxJ6V7HEREJmcr9BH74fDk4+Mf3zfY6iojISVG5D6Kj28+vtx7kY8sLKMjS5mAiEl1CKnczyzazJ82szcwqzezaQY67xsx2m5nPzGrN7AEzi8r1jNL9TXT5e1k5O8/rKCIiJy3UM/e7gS4gH7gOuMfM5g9w3GvACudcBjADSAT+IxxBR9uGvQ2YwfKibK+jiIictCHL3cxSgNXA7c65VufceuAp4Pr+xzrn9jvn6vsM+YGo3Bd3w74G5uSnkTFBb8QhItEnlDP32YDfOVfeZ6wUGOjMHTM7z8x8QAuBvxR+OMhxa8xsk5ltqqurO8nYI6vb38vmykbOmq6zdhGJTqGUeyrg6zfmA9IGOtg5tz64LFMA3AXsG+S4e51zJc65ktzc3NATj4Kyg820d/k5c7oeWhKR6BRKubcC/S+KphM4Mx+Uc+4A8AfgseFF886GvUcAWD49y+MkIiLDE0q5lwOJZlbcZ2wxUBbC1yYCM4cTzEsb9jYwIyeFvLTxXkcRERmWIcvdOdcGrAPuNLMUM1sBrAIe7H+smV1nZoUWMA34BvBCuEOPpN5ex4a9DZyp9XYRiWKh3gp5K5AM1AKPAmudc2XBIm81s8LgcfOA1wks5bwG7AY+FebMI2p3TQvNHT0qdxGJaiG9V5xzrgG4coDxKgIXXI99/hXgK2FL54ENexsAVO4iEtW0/UA/G/Y2MCUzWVsOiEhUU7n34ZzjTa23i0gMULn3sbe+jfrWTm05ICJRT+Xex8Z9Wm8Xkdigcu/jzb0NTEwZy8zcFK+jiIicEpV7H8fub9ebYItItFO5Bx1oOkp141EtyYhITFC5B23U/e0iEkNU7kFv7m0gbXyi3ghbRGKCyj1ow94jLC/KZkyC1ttFJPqp3IH61k7erWvTkoyIxAyVO1pvF5HYo3InsN6enDSGBZMzvI4iIhIWKncC97cvnZbJ2ERNh4jEhrhvM9/Rbt4+3Kz9ZEQkpsR9uW+ubMA5rbeLSGyJ+3J/c28DSWOMM6bqzbBFJHbEfblv2NvAooJMkseO8TqKiEjYxHW5H+3ys73apyUZEYk5cV3uW6sa6el1KncRiTlxXe5v7m0gwWDZNK23i0hsiety31TZwOmnpZM+PsnrKCIiYRW35d7b69i238eSqZleRxERCbuQyt3Mss3sSTNrM7NKM7t2kONuMLPNZtZsZtVm9l0zSwxv5PDYe6SNls4eFheo3EUk9oR65n430AXkA9cB95jZ/AGOmwB8HsgBzgIuAr4Qhpxht73aB8DCAu0nIyKxZ8izajNLAVYDC5xzrcB6M3sKuB74ct9jnXP39Pn0gJk9DLw3jHnDprS6ifFJCRTnpXodRUQk7EI5c58N+J1z5X3GSoGBztz7ew9QNpxgI21btY8FkzNIHBO3lx1EJIaF0mypgK/fmA9IO9EXmdlNQAnwvUFeX2Nmm8xsU11dXShZw6bH30vZQZ+WZEQkZoVS7q1A/zcWTQdaBvsCM7sS+DZwmXOufqBjnHP3OudKnHMlubm5oeYNi4raVjq6e3UxVURiVijlXg4kmllxn7HFDLLcYmaXAj8BPuSc237qEcNvW3UTAIt05i4iMWrIcnfOtQHrgDvNLMXMVgCrgAf7H2tmFwIPA6udcxvCHTZctlX7SBuXSNHEFK+jiIiMiFCvJt4KJAO1wKPAWudcmZkVmlmrmRUGj7sdyAB+HxxvNbNnwh/71GyrDqy3JySY11FEREZESA8YOecagCsHGK8icMH12OcRedtjX509fnYdbubm82Z4HUVEZMTE3X2Auw610O13Wm8XkZgWd+Wui6kiEg/isNx9TEwZy5TMZK+jiIiMmLgs90UFGZjpYqqIxK64Kvf2rh4qaltYqIeXRCTGxVW5lx1sptfBYq23i0iMi6tyL90fuJiqPWVEJNbFVblvq/ZxWsZ48tLGex1FRGRExVW5bz/g0y2QIhIX4qbcfUe72VvfxiJdTBWROBA35X7sbfV05i4i8SBuyn3bgeCTqVN05i4isS9+yn2/j6KJE8iYkOR1FBGRERc/5V7dpIeXRCRuxEW517V0ctDXoYeXRCRuxEW5bz+23q4zdxGJE3FR7qX7fSQYzJ/c/32+RURiU1yU+/YDPmblpZIyLqQ3nhIRiXoxX+7OObZVN2lJRkTiSsyX+0FfB/WtXXp4SUTiSsyX+/ZqXUwVkfgT8+VeWu0jaYxx+mlpXkcRERk1MV/u26qbmDMpjXGJY7yOIiIyakIqdzPLNrMnzazNzCrN7NpBjltgZs+aWb2ZufBGPXmBi6k+LcmISNwJ9cz9bqALyAeuA+4xs/kDHNcN/BK4OTzxTs2+I+20dPToyVQRiTtD3vhtZinAamCBc64VWG9mTwHXA1/ue6xzbjew28xmjUTYoXT19NLS0U1zRw/NR7t5aXctoIupIhJ/QnmqZzbgd86V9xkrBS4YmUihe3l3LV//3U6aO3po6eimo7v3b47JSE6iOC/Vg3QiIt4JpdxTAV+/MR9wSrefmNkaYA1AYWHhsH6N9OQk5k5KJz05kfTxSaSNTyQ9Ofjf8UmkjU9i2sQJJI6J+evGIiJ/JZRybwX6b8qSDrScyjd2zt0L3AtQUlIyrIuvSwuzWHpd1qnEEBGJSaGc0pYDiWZW3GdsMVA2MpFERORUDVnuzrk2YB1wp5mlmNkKYBXwYP9jLWA8MDb4+XgzGxfmzCIiMoRQF6NvBZKBWuBRYK1zrszMCs2s1cyOLZpPA47yl7P6o8DucAYWEZGhhbQHrnOuAbhygPEqAhdcj32+D7BwhRMRkeHRbSQiIjFI5S4iEoNU7iIiMUjlLiISg8w5zzdvxMzqgMphfnkOUB/GOOGkbMMTydkgsvMp2/BEa7ZpzrncgV6IiHI/FWa2yTlX4nWOgSjb8ERyNojsfMo2PLGYTcsyIiIxSOUuIhKDYqHc7/U6wAko2/BEcjaI7HzKNjwxly3q19xFRORvxcKZu4iI9KNyFxGJQSp3EZEYFLXlbmbZZvakmbWZWaWZXet1pr7M7GUz6whuidxqZp5sfWxmnzWzTWbWaWb393vtIjPbZWbtZvaSmU2LhGxmVmRmrs/ctZrZ7aOcbZyZ/Sz4s9ViZlvN7LI+r3s2dyfKFiFz95CZHTKzZjMrN7Nb+rzm9c/cgNkiYd76ZCwOdsdDfcZOft6cc1H5QWBf+ccJbDl8HoH3dZ3vda4++V4GbomAHFcR2K75HuD+PuM5wTn7KDAeuAt4I0KyFQEOSPRw3lKAO4JZEoDLCby1ZJHXczdEtkiYu/nAuOD/zwUOA8u8nrchsnk+b30yPge8CjwU/HxY8xbSfu6RxsxSgNXAAudcK7DezJ4Crge+7Gm4COOcWwdgZiVAQZ+Xru1owYMAAAMUSURBVALKnHNPBF+/A6g3s7nOuV0eZ/OcC7wD2R19hn5nZnsJFMFEPJy7IbJtHunvPxTnXN+34HTBj5kE8nn9MzdYtiOj8f2HYmbXAE3A68Cs4PCw/qxG67LMbMDvnCvvM1ZK4G/lSPItM6s3s9fMbKXXYfqZT2DOgOOF8S6RNYeVZlZtZj83sxwvg5hZPoGfuzIibO76ZTvG07kzsx+ZWTuwCzgE/J4ImbdBsh3j2byZWTpwJ/BP/V4a1rxFa7mnEvhnSl8+IM2DLIP5Z2AGMIXAQwi/NbOZ3kb6K5E8h/XAcgJv27iMQKaHvQpjZknB7/9A8EwpYuZugGwRMXfOuVuD3/t8Au/B3EmEzNsg2SJh3r4O/Mw5t7/f+LDmLVrLvRVI7zeWTmDdMSI45950zrU45zqdcw8ArwEf8DpXHxE7h865VufcJudcj3OuBvgscEnwzGZUmVkCgTeD7wrmgAiZu4GyRdLcOef8zrn1BJbc1hIh8zZQNq/nzcyWABcDPxjg5WHNW7SWezmQaGbFfcYW89f/LI00jsh6f9kyAnMGHL+OMZPInMNjj1GP6vyZmQE/A/KB1c657uBLns/dCbL158nc9ZPIX+Yn0n7mjmXrb7TnbSWBi7pVZnYY+AKw2sy2MNx58/rK8ClcUX6MwB0zKcAKIuhuGSATeD+BK9uJwHVAGzDHgyyJwRzfInCWdyxTbnDOVgfHvsPo37kwWLazgDkETj4mErgr6iUP5u7HwBtAar/xSJi7wbJ5OndAHnANgaWEMcE/B23AKq/nbYhsXs/bBGBSn4/vAb8Kztmw5m3UfhhHYDKygV8Hf3OqgGu9ztQnWy6wkcA/m5qCfwjf51GWO/jLXQHHPu4IvnYxgYtKRwnculkUCdmAvwP2Bn9vDwG/ACaNcrZpwTwdBP5ZfOzjOq/n7kTZvJ674M/+K8Gf+2ZgO/CpPq97OW+DZvN63gbIegfBWyGHO2/aOExEJAZF65q7iIicgMpdRCQGqdxFRGKQyl1EJAap3EVEYpDKXUQkBqncRURikMpdRCQG/R99QSkdefKZUwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(L(learn.recorder.values).itemgot(2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The final accuracy is given below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6968408226966858"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn.recorder.values[-1][2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now use the resnet model architecture with the fast AI learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqwAAACwCAYAAADKSTz5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAS10lEQVR4nO3deYyVZZbH8fMgawE6yA4jAUTtFJuILAqylGK3hYiACoqoaFhEBNTuGM3AjI4aooCAiOCCDIoWRJDGUgEFSdEwpShDKFIqJSqbtAMljMhmgHf+wJ4xnHOL93Kr7n3ee7+fxJj86l0O5qmqw+s97+OCIBAAAADAV5VSXQAAAABQFhpWAAAAeI2GFQAAAF6jYQUAAIDXaFgBAADgNRpWAAAAeI2GFQAAAF6jYY2Tc66Sc26Sc+4b59xR59xO59xM51zNVNcGlIW1i6hyzv3FOfefzrkDzrmDzrm/Oef+lOq6gLNxzuU65zY754475753zj2c6pqiqnKqC4igR0TkLyJyj4h8ISKXicjrIlJNREalrizgrFi7iKocEZknIhtF5KiIjBCRfOdczyAI1qe0MiAG59yVIvJXEZkqIreLSBcRmeOcOxIEwZyUFhdBjp2u4uOcWyYiJ4MgGPS7bKqI5ARB0CF1lQFlY+0inTjnikRkVRAEj6S6FsDinHtLRJoHQXD177LnROSWIAhapK6yaOIjAfH7m4h0c861ExFxzrUUkVwReT+lVQFnx9pFWnDOVRKR2iKyP9W1AGXoJiIrzshWiEhz59w/p6CeSOMjAfGbKiI1RGSTcy6Q0/8NXxGRiSmtCjg71i7SxeMi8k8i8kaqCwHK0FhE/n5G9vfffW13csuJNp6wxu8WEblfRIaLyBUicquI3CAiT6WyKCAE1i4izzk3Rk43rLcEQcAvfEQVn8eME09Y4zdVRGYEQfCPv9kXOedqiMg859y/B0FwLIW1AWVh7SLSnHN/FpEnROSmIAg+TnU9wFnsFZFGZ2QNf/v3mU9ecRY8YY1fTRE5dUZ2UkTcb/8AvmLtIrKcc0+KyL+KSC7NKiJivYj88YzsTyKyg/87ED+esMZvmYj82Tn3jYj8l5x+NdBTIvJhEARHU1oZUDbWLiLJOTddTr967XYR+do594+nVkeDIPif1FUGlOl5EdngnHtaTn/eurOIPCgiD6W0qojitVZx+u0l6/8mIoNEpImI/LeI5IvIvwRB8FMKSwPKxNpFVP02JGj5jyAI7klmLUA8nHN9ReQZEfmDnP4YwIwgCKaltqpoomEFAACA1/gMKwAAALxGwwoAAACv0bACAADAazSsAAAA8FqZr7UqYzITCC0IgqS/45O1i/LA2kVUJXvtsm5RHspatzxhBQAAgNdoWAEAAOA1GlYAAAB4jYYVAAAAXqNhBQAAgNdoWAEAAOA1GlYAAAB4jYYVAAAAXqNhBQAAgNdoWAEAAOA1GlYAAAB4jYYVAAAAXqNhBQAAgNdoWAEAAOA1GlYAAAB4rXKqCwAApIeaNWuq7MYbb1RZx44dE7rP559/rrLFixcndE0AfuMJKwAAALxGwwoAAACv0bACAADAazSsAAAA8BpDV0AGaNy4scrmz5+vsuuvv15lQRCY15w5c6bKJkyYEH9xiKQ6deqobNmyZSrr3r27ymKtqUT07t1bZY8//rh57IEDB8r9/sC5OP/881XWoUMH89jc3FyVHT9+XGWTJk1KvDAP8YQVAAAAXqNhBQAAgNdoWAEAAOA1GlYAAAB4LaOGrpo3b66ye+65xzz266+/VtmUKVNUVlpaqrLq1aurLNYwygcffGDmwLnq1auXymbNmqWyFi1aqOzbb78NdZyISMuWLeMvDpEzaNAgM7cGO1q3bl3R5cQ0cuRIlTVq1Mg8dsCAARVdDqB06tRJZe+9957KGjRoEPqaJ06cUNnWrVtVVlhYqLKdO3eGvo8PeMIKAAAAr9GwAgAAwGs0rAAAAPAaDSsAAAC8lrZDV6NGjVLZM888o7KsrCzz/B07dqisdu3aKrM+1G/t4nLllVea92HoCuVt+PDhKsvOzlbZww8/rLJVq1apzPoAv4g9mIhoe+qpp1Q2duxY89hatWqFuubcuXNV9tJLL4WuyRrkevXVV1VWo0YNlV1++eXmNa2d3/bu3Ru6JuBsOnfurDJrJ7h4Bqw2b96ssvbt26ssLy9PZSUlJSrr37+/eZ+vvvoqdE3JxBNWAAAAeI2GFQAAAF6jYQUAAIDXaFgBAADgNWcNCP3fF52L/cUUsHaqEhHp2LGjyhYvXqwy68/66aefmte0dqvavn27ytasWaMya1ehI0eOmPeJlZ8p1o4tURAEgUv2PX1bu8lkDUNdcsklKrv00ktVZu389sgjj5j3sa65e/fuEBVGRzqv3dzcXJVZPzetn4WxtGvXTmXFxcXxFRbC8uXLVda3b1+Vxfr9dsMNN6jso48+SrwwjyR77WbCz9xKlexnfJMnT1bZsGHDVNawYUOVWT3I1KlTzftY675nz54qs4YSL7roIpUdPHjQvM/VV1+tsmQNYpW1bnnCCgAAAK/RsAIAAMBrNKwAAADwGg0rAAAAvEbDCgAAAK9FamvWWG8JWLRoUajzlyxZorInnnjCPDbsZOvgwYNDHRdrC9hY+ZmsqcFY09vIbD/99FOo46ytNa3J1v3795vnp9sbATKN9UYAa3vTWEpLS1W2b9++hGoKq6CgQGX9+vVT2alTp5JRDjLE888/b+YPPvhgqPPfeustlVnbyB8+fDh0TdbbLay3YFjbwDdr1sy8pvXneeCBB0LXVFF4wgoAAACv0bACAADAazSsAAAA8BoNKwAAALwWqaGreLRt21ZlO3bsUFk8H262WB/+X7Vqlcrq1Kljnn/hhReq7OKLL1bZkCFDVDZt2jSV7dmzx7wPMkd+fr7KunTporKRI0eqzNq+b8aMGeVTGLzSo0cPlWVnZ4c+//3331fZgQMHEqopEdaAVVlbjwNlsbYZtgakYikpKVHZiBEjVHb06NH4CgvBGhq/6667VLZ27VrzfGuL7meffVZlVk9VkXjCCgAAAK/RsAIAAMBrNKwAAADwGg0rAAAAvBapoavGjRubeU5OjsrC7lSVqL1796osNzc3oWtaA2NbtmxRWatWrVTG0BXCGj16tMqs9fPkk08moxwk2aZNm0JlQLpr06aNylavXq2yqlWrmue/++67KrvzzjtVVhEDVmH9/PPPoY+1dryrXDn17SJPWAEAAOA1GlYAAAB4jYYVAAAAXqNhBQAAgNdS/ynaOFx22WVmbu24EmVFRUUqs3ZxASyzZs1SmTUAYH0/WcN9qdy9CGjZsqXKrF3aLNu2bTPzwsLChGpCepkwYYLK6tatG/r8lStXqiyVA1bpiiesAAAA8BoNKwAAALxGwwoAAACv0bACAADAazSsAAAA8Fqk3hLQvHlzM8/Ly1NZotujAlGVlZWlsipVqoQ613pzQLVq1cxjjx8/Hl9hwDl44403VGa9OcCybt06Mz906FBCNSG6evfurbIBAwaEOtfail1E5OWXX06opmSoV69eqktIGE9YAQAA4DUaVgAAAHiNhhUAAABeo2EFAACA1yI1dDV//nwztz5EHWVNmzZV2Weffaay4uLiZJSDiLn33ntVFnZIxTou1kDB3XffHV9hwFlYwy9du3YNde7hw4dVNmPGjIRrQjTVrl3bzKdMmaKyOnXqhLpmlIe5rd8LsZSWlqrs2LFj5VnOOeEJKwAAALxGwwoAAACv0bACAADAazSsAAAA8JoLgiD2F52L/cUUqFmzppn37dtXZfXr11fZsmXLVLZnz57ECytnY8aMUdnWrVtVVlBQkIxyEhYEgUv2PX1buxXhmmuuMfP8/HyVbdmyRWXjx49X2fr161VWubI9mzlo0CCVLV++3Dw2qli7FaNZs2ZmvmHDBpU1atRIZfv27VOZNZT72GOPxV9cmkj22vVt3Xbu3NnMCwsLQ52/dOlSlQ0ePNg89uTJk+ELS4Lu3burzPrzxNr9auDAgSqz+qeKUNa65QkrAAAAvEbDCgAAAK/RsAIAAMBrNKwAAADwWqR2urJ2MhER+fLLL1X29NNPq8waMpk1a5Z5zZ07d6rM+tBx9erVVXbeeeep7NSpU+Z9Fi1apLITJ06obPbs2eb5yFx33HGHmVtrcsGCBSrbtGmTyoYPH66y1157zbzPc889p7KPP/5YZUeOHDHPR2aoVq2ayiZOnGgeaw1YWaz1nMkDVtDuu+++0Md+9913KrPWk2/DVbEMGTJEZbEGrCw+DqOL8IQVAAAAnqNhBQAAgNdoWAEAAOA1GlYAAAB4LVJDV7EUFRWpLCcnR2Vvv/22ymJ9uNja3WfNmjUqs3bfmjt3rspiDV3t2rVLZdOnTzePRebKzc1VWb9+/cxj582bp7JXXnkl1H3y8vJUFmvtWsfedtttKrN2IELmsIZdreG+eFhDV8Dv1a1bN/Sx1u5XJSUl5VlOhenVq5fKwn5/xeo1Nm/enEhJFYYnrAAAAPAaDSsAAAC8RsMKAAAAr9GwAgAAwGs0rAAAAPCaC4Ig9hedi/3FCOrYsaPKxo4dax5rTVX36dNHZZMmTVLZ8ePHVZafn2/eZ9iwYaHOj7IgCFyy7xnltVu1alWVrVu3TmWtWrUyz7feHrBhw4ZzricrK8vMf/nlF5VZ06VXXHHFOd871Vi78enUqZPKrLer1KhRI/Q1r7vuOpUVFBSoLNbbLDJVstduKtdtpUr62dvSpUvNY2+66SaVWW8QGjp0aOKFnaMqVaqYeffu3VW2cuXKUOdv3LhRZV27djXvk8rvpbLWLU9YAQAA4DUaVgAAAHiNhhUAAABeo2EFAACA19Jia9awvvjiC5XF2sKsadOmKhsxYkSo+1hDM1u3bjWPTbcBKyTu0UcfVZk1zDJhwgTz/EQGrICwrMEpa7i0Vq1aKos11DFx4kSVrV27Nv7ikuyCCy4wc+u/UXZ2tsrq1aunssWLFydeWIZo2bKlyqzhqqi4//77zdzaStU5PaNkbZttDYhHbVCRJ6wAAADwGg0rAAAAvEbDCgAAAK/RsAIAAMBrGTV0ZWnbtq2ZFxUVqaxnz54qs3bTaN26tcpiDXctXLhQZdu3bzePRWYYOHBgqOPeeeedCq7ktA4dOoQ+tri4uAIrgU+swZC6deuqzBrsiLXD4v79+1U2ZcoUle3YsUNl1mBiWTs5nitryOWqq64yj23RosU534ehq/RjDYdZ30dhB7xF7O8Fa8CqpKQk9DV9xRNWAAAAeI2GFQAAAF6jYQUAAIDXaFgBAADgtYwfurKGq2KxhqHat2+vstdff11lQ4cONa85btw4lY0fPz50TchcBw4cKPdrduvWTWXLly8Pff6SJUvKsxx4LCsrq9yvOXv27HM+1xqGStbQVUXcB+H98MMPKlu/fr15rPUzrmHDhiqrXr26yo4dO2Zes1q1airr37+/yl544QWV1a9f37ymZdeuXSrr06ePytJhwMrCE1YAAAB4jYYVAAAAXqNhBQAAgNdoWAEAAOC1jBq6atWqlcq++eabcr+PtavVr7/+ah7bpk2bcr8/MkOsD+vv3LlTZQ0aNFDZrbfeqrJp06aprFIl+++1kydPVtmHH35oHov08+KLL6qsSZMmKhs1alQyyvHS6tWrVbZ3716VzZkzJxnlpK0jR46orKCgwDzWGrrKyclRWWFhocpi/R6vUqWKyqyB7LCsn8MiIjNnzlSZ9fM+XfGEFQAAAF6jYQUAAIDXaFgBAADgNRpWAAAAeI2GFQAAAF5zZW0p55yL7H5zP/74o8p2796tsljbTlrT+3l5eSrr0aOHygYPHqyyWNsYbty4UWXXXnuteWxUBUGg9zKsYFFeuzfffLPKFi5cqLJVq1aZ5xcXF6vMenNFo0aNVFZaWqqyhx56yLzPm2++aebphLWbuBYtWqhs9OjRKajk/1lbCH///fehzo1na9aDBw+qLNakeXlL9tr1bd1aW6uKiHzyyScq69KlS0WXIyJ2X2L1C9YbCkSSt3ZSqax1yxNWAAAAeI2GFQAAAF6jYQUAAIDXaFgBAADgtbQdutq1a5fKGjduXO73CfsB/A0bNpjnz58/X2Xz5s1LuC6fMLiSuOnTp6ts3LhxCV1zwYIFobI1a9YkdJ8oY+0iqjJ96CqWdu3aqez2229X2ZgxY1RWu3Zt85rWz8iVK1eqzNqC99ChQ+Y1MxVDVwAAAIgsGlYAAAB4jYYVAAAAXqNhBQAAgNfSdugqOztbZStWrFBZkyZNErqP9QFuy7Zt28z8xIkTCd0/ChhcQVSxdhFVDF0hihi6AgAAQGTRsAIAAMBrNKwAAADwGg0rAAAAvJa2Q1fwB4MriCrWLqKKoStEEUNXAAAAiCwaVgAAAHiNhhUAAABeo2EFAACA12hYAQAA4DUaVgAAAHiNhhUAAABeo2EFAACA12hYAQAA4DUaVgAAAHiNhhUAAABeo2EFAACA12hYAQAA4DUaVgAAAHiNhhUAAABec0EQpLoGAAAAICaesAIAAMBrNKwAAADwGg0rAAAAvEbDCgAAAK/RsAIAAMBrNKwAAADw2v8CYI3bgCgb/tUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x216 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dls = ImageDataLoaders.from_folder(path,valid_pct=0.2)\n",
    "dls.valid.show_batch(max_n=4,nrows=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = cnn_learner(dls,resnet18,pretrained=False,loss_func=F.cross_entropy,metrics=accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.097582</td>\n",
       "      <td>0.056914</td>\n",
       "      <td>0.984071</td>\n",
       "      <td>00:51</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit_one_cycle(1,0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learner Class"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is my own implementation of the learner class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyLearner(object):\n",
    "    def __init__(self,dls,model,opt_func,loss_func,metrics):\n",
    "        self.dls = dls\n",
    "        self.model = model\n",
    "        self.opt_func = opt_func\n",
    "        self.loss_func = loss_func\n",
    "        self.metrics= metrics\n",
    "    \n",
    "    def calc_grad(self,xb,yb):\n",
    "        preds = self.model(xb)\n",
    "        loss = self.loss_func(preds,yb)\n",
    "        loss.backward()\n",
    "\n",
    "    \n",
    "    def train_epoch(self,opt):\n",
    "        for xb,yb in self.dls.train:\n",
    "            self.calc_grad(xb,yb)\n",
    "            opt.step()\n",
    "            opt.zero_grad()\n",
    "    \n",
    "    def validate_epoch(self,model):\n",
    "        m = [self.metrics(self.model(xb),yb) for xb,yb in self.dls.valid]\n",
    "        return round(torch.stack(m).mean().item(),4)\n",
    "    \n",
    "    def fit(self,n_epochs,lr):\n",
    "        opt = self.create_optimizer(lr)\n",
    "        for i in range(n_epochs):\n",
    "            self.train_epoch(opt)\n",
    "            print(self.validate_epoch(self.model), end= ' ')\n",
    "    \n",
    "    def create_optimizer(self,lr):\n",
    "        return self.opt_func(self.model.parameters(),lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "dls = DataLoaders(train_dl,valid_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_learn = MyLearner(dls,nn.Linear(28*28,10),SGD,cross_entropy_loss,batch_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.237 0.3283 0.3803 0.4098 0.4278 "
     ]
    }
   ],
   "source": [
    "my_learn.fit(5,lr=0.1)"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}