Homework3_p2_complete.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.8"
    },
    "colab": {
      "name": "Homework3_p2_complete.ipynb",
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ybt9uI5VdASV"
      },
      "source": [
        "## Homework 3: Back-propagation Implementation: Part II\n",
        "## 2021 CS 474/574: Deep Learning\n",
        "## Due: 10/29/2021\n",
        "\n",
        "\n",
        "### Tasks:\n",
        "\n",
        "1. Implement the sofmax function. (Part I)\n",
        "\n",
        "2. Implement the Layer class. (Part I) \n",
        "\n",
        "3. Complete the NN class. \n",
        "\n",
        "    - implement the 'add' function. (Part I)\n",
        "    \n",
        "    - implement the cross-entropy loss. (Part I)\n",
        "    \n",
        "    - implement the forward propagation process. (Part I)\n",
        "    \n",
        "    - implement the prediction function. (Part I)\n",
        "    \n",
        "    - compelte the NN class. 70 points\n",
        "    \n",
        "        - transform function from part I to class methods. 10 points\n",
        "   \n",
        "        - complete 'train' function. (Part II). 20 points\n",
        "    \n",
        "        - implement the BP algorithm. (Part II). 30 points\n",
        "    \n",
        "        - implement the accuracy method (Acc) (Part II). 10 points \n",
        "    \n",
        "4. Create and evaluate a neural network. 30 points"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M7woCAlrdASd"
      },
      "source": [
        "import numpy as np\n",
        "from matplotlib import pyplot as plt\n",
        "from tensorflow import keras\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 285
        },
        "id": "OqBkF1k6dASj",
        "outputId": "0e133a28-93f9-42f7-f33e-893193e63695"
      },
      "source": [
        "## load the digits dataset\n",
        "def load_digits(show_sample = True):\n",
        "    (x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
        "    \n",
        "    #show first 100 images\n",
        "    if show_sample == True:\n",
        "        nImg = 4\n",
        "        for i in range(nImg*nImg):\n",
        "            plt.subplot(nImg, nImg, i+1)\n",
        "            plt.imshow(x_train[i], cmap = 'Greys_r')\n",
        "        plt.show()\n",
        "        \n",
        "    x_train_1 = np.reshape(x_train, [x_train.shape[0], x_train.shape[1] * x_train.shape[2]])\n",
        "    x_test_1 = np.reshape(x_test, [x_test.shape[0], x_test.shape[1] * x_test.shape[2]])\n",
        "    \n",
        "    # data normalization --> [0, 1]\n",
        "    x_train_1 = x_train_1/255\n",
        "    x_test_1 = x_test_1/255\n",
        "    return x_train_1, y_train, x_test_1, y_test\n",
        "\n",
        "x_train, y_train, x_test, y_test = load_digits()\n",
        "print(x_train.shape, y_train.shape)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVkAAAD7CAYAAADAUeeKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOy9eXBb15Xn/7nYAQLgToI7uIqLRIqSqF2yFju2HLfbzuIsHSfT6XQyU9Xd2X49Trp7amaqq6ZSvc1MJdVJOYunk1HF7pSXOHYcZyJbtrZoISWSoiSu4k6CKwgQKwG83x8UXkRZsjZio96nCkUSeMA77/Di++4999xzhSRJKCgoKCjEBlWiDVBQUFBYyygiq6CgoBBDFJFVUFBQiCGKyCooKCjEEEVkFRQUFGKIIrIKCgoKMeS+RFYI8ZgQolsI0SeE+NZqGaWwjOLf2KH4NnYovl2JuNc8WSGEGugBHgFGgbPAZyRJurR65j24KP6NHYpvY4fi2w+iuY/3bgX6JEkaABBCvAj8MXBLZwohUnrlgyRJIo6nuyv/Kr69K5S2GzsU397A/YQLioCR6/4evfbcCoQQXxZCnBNCnLuPcz2I3Na/im/vGaXtxg7FtzdwPz3ZO0KSpOeB5yH171jJhuLb2KL4N3Y8SL69n57sGFBy3d/F155TWB0U/8YOxbexQ/HtDdyPyJ4FqoUQ5UIIHfBp4PXVMUsBxb+xRPFt7FB8ewP3HC6QJCkkhPgL4G1ADfxEkqSuVbPsAUfxb+xQfBs7FN9+kHtO4bqnk6V47CXOM+B3RTx8q1arMRgMWK1WLBYLOTk5AExPT7O4uIjL5cLv9xMOh+/6s5PZt6C03Viy1n0b84kvhbWD1Wpl/fr1PPzww+zYsYNt27YBcPr0aU6cOMHvfvc7urq6cDqdCbZUQSF5SHmRFUKg0WhQq9Urni8vL6e4uBidTodWq8VqtdLf3097ezvbt2+nqKiI8vJy1Go14XCYwcFBhoeHOX36NB6PJ0FXk5yoVCpycnJobm7mk5/8JA0NDRQUFGAwGJAkiZqaGsxmM0VFRfzrv/4rFy5cSLTJa5b8/Hxyc3P5yEc+wvT0NL/+9a/xeDz4/f5Em5ZymM1mWlpasNvtVFRUIITA5/Px6quvMjMzw9TU1KqcJ6VFVqVSoVKpSEtLQ6fTrXht586dbNq0CZPJhNlsprS0lLfffpvR0VGefPJJduzYwcaNG2WR7ejo4P3336e7u1sR2RtQq9XY7XZ27NjBn/zJn6DRaFCp/jBnWlJSQmFhIRs3buTo0aOKyMaQyspKGhsb+da3vkV/fz8dHR2MjIwoInsPZGZm8sQTT3DgwAGampoQQuD1enE4HFy4cOHBEVmdTkdmZiZqtRqNRkN9fT05OTlUVFRgMBjQ6/Vs3ryZjIyMFe/LyMjAZDIhhECSJEKhEI8++ig5OTk8+uij5ObmEolECAaDuFwuurq66OzsxOfzJehKk5PS0lIqKyv5u7/7O+x2O1qtFiGWQ1DhcBhJkohEIqhUKrRaLeXl5dTX1zMwMEAwGCQSiST4CmKL0WiktrYWs9mMyWTi3LlzzM7Oxux8drudsrIypqenGR8fj9l51jIqlYoDBw7Q2NjIJz7xCTIyMrh+bkqSJFZzriqpRVaj0ZCRkUF9fT06nQ69Xk99fT35+fk0Nzej1+vR6/VUV1eTlpZ2y88JhULMzc1htVqpr68nLS0NtVqNz+fD4/HgcDi4evUqIyMjBIPBOF5hciKEQKVSoVarqayspKGhgfXr12OxWOQerCRJhMNhwuEwfr8fg8GA0WikrKyMpqYmFhYWcLlcuN3uBF9NbNHpdFRWVmI2mzGbzVy5ciWmIpuenk5WVpbcdhXuHpVKRWlpKdXV1eTl5aHR/EEGY5EIkLQiq9FoaGho4NFHH+Wv//qv0el0qNVqVCqVLAKwLAg3xmOvR5IkPB4PP/zhD3G73fj9frRaLQAzMzP4fD7m5ubo6elhcnKSpaWluFxfsiKEwGAwkJ+fT3V1Nd/+9repq6uTRxPX43a7mZyc5MUXX+Sxxx5j165dPPvss3ziE5/gZz/7Ga2trfz0pz9N0JXEB7PZzJYtW8jPzyc/P5+2tjaGhoZicq5obLysrIycnBxyc3MRQsgjC4U7QwhBbm4uhYWFsqbEkqQVWYBgMIgQAqPRiE6nW3HHuRFJkuThaSQSkY+XJAm/309XVxeLi4uyiIbDYWZmZvD7/Xi9Xubm5pReLJCWlsauXbsoLy+npqYGu92O1WpFrVZ/4MscHV14PB6cTic+nw+9Xo/JZKK0tJSrV68m6Crih8FgoK6uTh4dfdgN/37QaDTo9XpKS0spLi4mGAzidDpxOp0EAoGYnHMtYrFYyMrKoqamhtLSUoQQ8ohscnKS8fFxxsfHmZ+fX7VzJq3IhsNhHA4HMzMzBAIBOSZ7KyKRCNPT0/j9fnw+H8XFxVitViKRCG63m1/+8pdKvPUOKCgo4Jvf/CY1NTWUlJR86LEWi4VIJILRaGR+fp6RkRFKS0vRaDQUFRWRn58fJ6sTR3Z2Nlu3bmVxcZHR0dGYncdgMJCTk8P27duprq6mra2Nvr4+BgcHY3bOtYjdbqepqYlHHnmE3NxcVCoVfr+fhYUF/uEf/oHf/OY3TE1NrWqHK2lFVpIkFhcX6enp4a233qKqqorc3FxmZ2cxmUzU1tbK3Xyfz4fL5eL5559ndnYWn8+H3W6nuLiYzZs3Mz4+fk8J8g8SQggKCgpobGykurqazMxMuecaiURYWFhgcXGRyclJqqqqyMjIYGlpSRYXv9/P4uIin/zkJ8nIyHighrHREFYs2bJlC48//ji5ublIksTMzIySj3wXpKWlUVJSwtNPP82BAwfIyMiQRx2zs7OcPHmS7u5uuVO3mhO2SSuysBwuGBgY4Le//S2Tk5NUVlbS1dUlxwujX2Sv18vk5CSHDx9mYmICv99PYWEhJSUlBAIBvF5vTALaa4VoXDs6c52bmyunxEUnuKamppiYmOD3v/89VqsVq9Uq9wBGR0eZnZ1lfn6ej370o3L8NhrvWqsZBtG5gXjcUCoqKnjssccwm82yyK71ScXVxGw2U1tby969e9m8eTMGgwGVSiX78sSJE/T398fEp0ktsgAOh4Nf/epXHD9+HKvVyuzsLFVVVdTV1VFeXk5BQQHnzp3j6NGj8l0IYGpqCqfTyXe+8x3C4TChUCjBV5K82Gw2SktL+ed//mfsdjt6vV5ugFNTU4yPj/Nf/+t/JRQKUVxczJtvvklxcTFvvfUWV69epbW1FSEEOp2Or33ta5SUlFBdXY3f76epqYnh4eGYzrgnAiEE5eXl1NbWkpaWFvN4vk6nw2QyyTets2fPcvHixZiecy1RUlLCU089RVVVFUajESEEoVCIyclJjh49ys9//nMWFhZicu6kF9lQKMT8/DwejwetVovP50On0zE5OYnNZgOQczWvz28LhUKEQiG8Xm8izU9qVCoVOp2OkpIS6urqKC0tJSsrC/hDDuzMzAz9/f2cP39eTuuSJInR0VFOnjzJ1NSUnEoUnUQQQqDX68nKyqKhoQGXy7UmRbaoqIjc3NyYz1BHJ710Oh1CCCKRCHNzc8zNzcXsnGsJtVpNTk4OdXV1chpiJBJhaWmJ4eFhhoaGmJmZidn5k15kowSDQbm3EA6HcbvdcqZAY2MjFouFn/70p3g8HiU0cIdEswAef/xx9u/fL8epAoEAwWAQr9fLu+++S1tbG/Pz8/h8PsbGxuThcVSIb4ZarcZms/HFL36RYDBIf39/nK8utqhUKlpaWti6detNl3WvFhqNhpycHPLz88nMzESj0RAKhXA4HKs6A75WUalUZGRksG7dOjZv3iwvTlpaWmJ2dpYf/vCHtLe3x9SGlBHZ63G5XBw7dgyz2UxhYSFGo5HCwkLWrVuHEILJyclEm5j0qFQqCgsLeeKJJ9i9ezfl5eWoVCp8Ph9tbW1MT08zMTHB+fPnGR4eZmlp6a5WwkTjlRkZGRgMhhhfTfwRQpCZmUl2djaw3AlwOByrnsGSnp7ORz/6UZqamtBqtYRCIXkhwoOe0307TCYTGRkZfPKTn2T//v1yCEySJCYnJ+nt7aWrqyvmepGSIut0OnnppZcwmUysX7+e0tJSbDYbjY2NSJKEw+FQerO3QaPRUFtby6c+9SnKy8tJT09naWmJ+fl5fvzjH9Pf309vby96vZ5IJHJP2RlqtRqLxYJer4/BFSQWIQTZ2dmkp6cjhMDv9zM6Orrq4ans7Gw+/elPU1NTg06nw+1243Q6cbvdSn7sbUhPT6eqqoqvfOUrcmgRlrNlOjo6OHnyJO3t7TGfr0lJkY1EIvj9fo4ePUogEOC5556jqKiIr3zlK0xMTHDmzBmmpqaYmZnhrbfeUoZVN6DRaKipqaG+vp7q6mpZSF977TXOnTvHu+++i9vtxuPxyMPgtZohsFq43W6uXLlyX7PTarUavV5PZWUlFRUVtLS0UFNTw6ZNmzAajQCyOExNTSlFYW5DU1MTGzZsoKCgAJPJBCz/n2ZnZ3n11VdpbW2NS2pnSoosLMdlR0ZGCAQCPPvss+Tk5MgTN2azmeHhYUZHR2lra2NpaQmfzydPjj3oaDQa1q1bR1lZGWlpaXKM6ve//z2nTp1ibGxMFtX7HZI+KLmy0UJDN8syiE4YAvKkYLSCXPS56KpGs9lMU1MT5eXl7NmzB5vNhsVikd8/MDBAV1cXXq9Xyf2+BdGbld1up6amBoPBIFfbczqdDA0NcfHiRQYGBuKiBykrsrAcm11cXOR73/sejY2N/MVf/AXZ2dnk5OSwadMmwuEwGRkZtLe389prr8nDrAcdq9XKV7/6VcrLyxFCsLCwgMPh4NixY7S3t99XrzU6Kfag3cx0Oh3p6ekfKLkJy6vobDYbarWazMxMNm/eTHp6OhkZGWg0GiwWC7t370av18u+i4a9ADljQ6VScfXqVTo6OpR47Idgt9vZvn07X/nKV6iqqkKr1RIIBHA4HPzbv/0bL7/8Mv39/XEbCaS0yEYT5S9fvgxAZ2en3KC1Wi16vZ5NmzaRlpaGw+Ggu7ubnp6eVV/RkUpkZWVht9spKCjAarUCy0OosbExnE7nfcenrhfX6JLmtVoTIlonA5b9unXrVtRqNdPT0yuOKy4ulpdwGo1GGhoa0Ov1GAwGlpaWUKlUhMNhvF4vgUBADtVcuXKFzMxMiouLZb+6XC6mp6cf2Pb7YahUKgwGA01NTTz88MPYbDb5xhVti9PT04yNjREMBuPWEUhpkY3S2dnJlStX6O3tZffu3XzlK1+Raxds3LiRmpoarFYr77zzDvPz80xPTz+wdQyamprYuHEj+fn5GI1GJEliYmKC48ePr1rpvGjjDQaDDA0NxSzJO9EEg0FCoRCSJFFYWMif/Mmf8Md//McfuKlEaxtHiYpzdPjqcrlobW1lbGyMnp4eLl26RE9PDzMzM7S0tHDw4EE5F3dmZoaxsQd6h+1botFosNlsPPLII3zhC19Y8Vq0INTMzEzc52jWhMjCshOHh4c5fvw4kUiExsZGamtr2bJlC1qtlk2bNpGZmYnNZuOFF17g0qVLiTY5IUS344nGA4PBID09PZw9e5bFxcV7+ky1Ws26devYunUr2dnZRCIRnE4n3d3dvPjiizHPQ0wEoVCI119/nb6+PoQQWK1W0tLSCAQCH4iVhkIhtFotIyMjLC4uyoWPHA4Hbrcbn8/HzMwMi4uLTE9PMz09jcfjoaWlhZaWFvR6vVxNbq2OClaDzMxMPvaxj7F+/foP9FKdTie//OUv5VFvPFkzIhuJRJidnWVhYYHBwUH6+/tpaGhgw4YNGI1G8vPzMZlMZGZmcvToUa5cufJADrk0Go0sstEdI4aHhzl//vw9xaiiQ+D169fzyCOPyJW5pqenaW9v580331yTo4ZIJMLRo0fp6enBZrNRUVFBRUXFTY+dn58nHA7zzjvvMDk5SV9fH/39/QwMDNzy86/PANFqtSwtLeH1epVY7C2I3uh27dpFYWHhitckScLpdPLuu+8mZBSwZkQ2SigUwuVycfz4cS5fvsxXv/pVOZcxWrm/paWF2dlZWltbH+hGG41pLywsMDU1ddfxWI1GQ2FhIc8++ywHDx6kpaUFWO41vPjii3R2duLxeNb0zWxqaorvf//76HQ6Oc3qZkQiEVwuF0tLS/KKug9DpVLR2NjI+vXrEUIwPz8vhxQUViKEoKKigq1bt9Lc3ExmZqb8WnST1DNnztDX15eQkcCaElmj0Yher8dqtcpr569f7hid+X5Q0opuRyQSwePx4PV670pgtVotOp2OoqIi6urq2LZtm1xlfmJigvHxcTlGvpYFFpZv6hMTEzH5bIvFIgu31+ulp6cHl8sVk3OlMiqVii1btrB+/foVGR6hUIhgMMjly5fp7u5O2IhqzYhsNPeztraWXbt2UVVVhc1mIysrSxbVaDX5zs5Ozp49+8DnGXq9Xs6cOXPXvaOCggIqKir44he/SH19PY2NjTidTi5evMh//+//nSNHjrC0tLTmBTaeOBwO3nnnnVXbQXUtodfr+du//Vt5c9Xo993pdDI2Nsa3vvWtmBZUvx23FVkhRAnwUyAfkIDnJUn630KILOAlwA4MAs9IkhTXaTuDwUBaWhoNDQ0UFRWxZcsWSkpKqK2txWq1YjKZ5J5sdGjs9Xrx+/1JIQCJ9q1Op8Nut5Oenn5Hx2dmZrJhwwa2bt1KXV0dLS0tmM1m5ubmuHz5Mm+88QaXLl1KmpVIifbvahKNnydL/nGy+dZkMslVyqKMj49z/vx55ubmEjovcCc92RDwTUmS2oQQFqBVCPH/gP8AHJEk6TtCiG8B3wKei52pHyRa7fyjH/0ozc3N8kzsjQVJosnd0Ur+gUAgWRpr3H17fbhEr9dTUVFBTk7OHb2vqKiIQ4cO8Ud/9EdUVlbKSd7nzp3jlVde4bvf/e5qmLiaJG3bvReSLMyVNL6NluyM7ukXpbOzk+PHj+NyuRJaT/q2IitJ0gQwce13txDiMlAE/DGw79ph/wYcJQ4NNbol8t69e2loaGD79u3yZn9Go3FFXU9JknC73czPz3PkyBGuXLlCe3s7ra2tsTbzjkiEb6+vpKVSqdDr9TzxxBOo1Wref/99uT5sbm4uZWVlNDc3U1hYiN1uJzs7m9LSUvR6PcFgkMnJSRwOB6+88goXLlxYDfNWlWRru/dCVFhNJhPFxcV0dXUlRW3eZPHtU089xc6dO+XRWNRfkiQxODhIZ2dnwie37yomK4SwA83AaSD/mqMBJlkeNtzsPV8GvnzvJv5h7bdGo6GkpEQunrF582bq6uowmUzyJovRsEB0N4TJyUlGRkZ49913uXjxYtLmbCbCt0IINBoNpaWl7Nu3D6fTydTUFLOzs5SUlFBeXs6BAwcoLCyUV9HpdDq51N7g4CBXrlyhs7OTkZGRezUjLiSq7a4WWq12RQ2DZCKRbbeqqordu3fLoYLo9z8UCjE3N4fD4Uh4aPCORVYIYQZeBr4mSZLr+qGLJEmSEOKm429Jkp4Hnr/2GXc9RtfpdGRmZlJSUkJZWRnf+MY3KCkpkZcp3liVPhgMMj09zcmTJzlz5oy8+2Qy199MlG+vvY+SkhKKiopoaWkhGAzi9/sxm81yFfnoIxAIMDY2xvDwML29vXz3u99lcHAw6dO0Eunf++X64W8yZsYkyrdWq5Xq6mq2bdtGXV2dHCqIbu3d1tZGR0cHs7OzCQ8N3pHICiG0LDvysCRJr1x72iGEKJAkaUIIUQCs2rSnWq3GYDCwadMmsrOzycvLo7CwkPz8fEpLS8nMzFxRozQYDMqVzufn5+nq6uLMmTNcvHiR4eHhpN4tId6+nZ2dlXf0ValUaLVaedNDs9lMJBIhFAqh0+nQ6XTyEtDomvmTJ08yNDQkjxCSfcfUePs3Vuj1enJycpKqNm8ifatWq7FarWRkZKworBMdvb733nuMjo7GtUbBrbiT7AIB/Bi4LEnSv1z30uvAF4DvXPv5y9UyymQyUVRUxN/8zd9QUlKCzWYjLS3tlg3M6XTicDj4yU9+Qk9PD++//z6BQCDpN09MhG+jq4xGR0cpKCiQK/tHS/DByt5TKBQiEAjw+9//nvfee49/+Id/SHijvVMS4d9YYTab2bBhAxkZGYk2BUi8b7VaLVarFbPZLItsOBzG7/dz6tQp/uVf/uX2HxIn7qQnuwt4FugUQkRnN/6GZSf+uxDiz4Ah4Jl7NUKn01FcXMz69eupra2VdzpoamrCZDKh1+vlmCv8YeNEl8vF2NgYb7/9Nj09PZw5c4aZmZmkSdG6A2Lu2xtxu90MDg7yox/9iJqaGnbv3v2BNC4hBC6Xi/HxcS5cuMDQ0BBtbW1xq7+5isTdv6tNsoUHriPhvr3V5pXJ1kbvJLvgOHCr//TB1TBCp9NRXV3Nli1bOHToEMXFxZjNZkwm04r6mlHhjA5pozVQX375ZTo6OvD5fEnn4A8jHr69EZ/PRyAQ4Be/+AUbNmwgFApx6NAheZvka3YxPT3N6dOneeONN7hw4QKDg4Mpt3gjEf5dTUKhkOzzZBPbRPs2Gn+NFuJPNv9cT1Ks+EpPT+fAgQM89NBDrF+/Xp7MEkLIua1jY2M4HA45sP3aa68xPj7OxMSEHGNMJYFNJJFIhJmZGU6dOsWlS5f4yU9+IteWjbK4uIjT6cTpdOLz+VJOYFOdSCTC+fPn0ev11NfXK237BjweDxcvXqSrq4vS0lLy8vJiui37/ZAUIhuJROQScNFq8FECgQAzMzNcuXKFkZERucLT+++/L6/eUrh7QqEQi4uL8g1MIbmQJImRkRGsVitDQ0OMj4/T29t7z+Uo1xpLS0u43W66u7spLS3F5/OhVquZn59P6BLamyHieYf8sFQNjUaDSqVaEXsFVoQKouGC6FAh3kiSlLRjkkSlGK0WyexbSIx/o6lzWq12Rf7nvXxnk9m/9+NbjUaDRqNZEeoKhUJxnfS+nW+TRmRTgbXaUJOBZPYtKP6NJWvdt8kZxFBQUFBYIygiq6CgoBBDFJFVUFBQiCHxzi6YATzXfiY7Oay0syxRhtwhim9jyyLQnWgj7pBU8++abrtxnfgCEEKckyRpS1xPeg+kip3Xkyo2p4qd15NKNqeSrVFSxeZ7sVMJFygoKCjEEEVkFRQUFGJIIkT2+QSc815IFTuvJ1VsThU7ryeVbE4lW6Okis13bWfcY7IKCgoKDxJKuEBBQUEhhtyXyAohHhNCdAsh+q7tTKmwiij+jR2Kb2OH4tsbiBZgudsHoAb6gQpAB7QD9R9y/GMs5xn2Ad+61/Ou9gMoAd4FLgFdwFevPf/fgDHgwrXH43G2S/Gv4lvFt2vAt/djxA7g7ev+/jbw7ds4XkrlR5z/yXfr34T7Zw37Vmm7im/v2bf3Ey4oAq7fB3r02nMruLb170WWV0oo3Dm39a8Q4stCiHMs+1fhzlHabuxQfHsDMZ/4kpa3/v0vwC9ifa4HDUmSnpeWV5/8l0Tbcr8IITITbcONKG03djxIvr0fkR1jOW4Rpfjacwqrw4Pm33+O47keNN/GE8W3N3A/InsWqBZClAshdMCnWd4O+Gbc6HiF23O3/k11tsbxXErbjR2Kb2/gnqtwSZIUEkL8BfA2ywHsn0iS1HWLw88C1fd6rgeRe/BvqhO3uHIytF2LxUJOTg65ubkYDAa8Xi/z8/OMjIwQCoVSZUv7D5AMvk024rbiSwjxOPBmXE4WIyRlC49YUihJ0kSijbgZsWi7Tz75JJ/61Kd4+OGHSU9PZ3Z2lrNnz/KNb3yD6elp3G73ap4uadvug6ALcasnK0nSr5N5b3SF2yOEkDeu02g0lJWVYTabMZlMDA0NMTg4eM8bXCarwMLqtl2VSoXFYqG0tJSmpibMZjNarZb09HRsNhtFRUV4PJ5VF9lkZTV9q1arSU9PRwiBWq0mLy+PvLw8du3ahV6v53YdyomJCWZmZnjjjTfw+Xy3Pf5OSYotwRVSA41GQ1paGgaDAaPRyJ49e8jJySErK4uzZ88yNjaGJEkpO9SNBxqNhuzsbCorK6moqECn0yGEwGQykZ2dTXFxsbJF+z2i0+koLCxEq9Wi1WrZsmUL1dXV/Kf/9J/uSGQHBwfp6+vj+PHjBIPBVdvxVhFZhQ8lKyuLzMxMnn76aWw2G1VVVeTm5mKxWLBarWi1WtRqNZs2baK8vJwXX3yR/v7+RJudtITDYZxOJyMjI4yOjlJUVITRaEy0WSlLdnY2WVlZ7N+/n5KSEvbu3YtGo0GlUpGXl0daWhoajeaOeqU2mw2z2cxzzz1He3s7L730EoFA4L7FNqVFVqVSoVarMRgMaLVadDodKpUKjeYPl+VyueSJBLVajUajwe/3EwqFWFpaSqD1yYtGo0Gn02EwGKioqMBut7N3716KioooKirCYrHIPQMhBEIIIpEIHo+HY8eO3VfYYK0TiUTw+/0sLCwwNzeHzWZLtEkpTVZWFnV1dWzdupWamhoaGxtRqZaTpgwGwwotuJ3Q6vV61Go1O3fuJBwO88YbbxCJRB5skc3Ly6OmpoYDBw5QVVVFY2MjVquVrKwsYLnXcPjwYQYHB5mdnaWgoICKigree+89uru7aWtrW7UhwVrBYDBgt9vZuXMn+/btY9euXeTk5KDX61GpVKhUKoLBIB6Ph6WlJdRqNVarlczMTHbs2MGmTZuYmJhgYGBAEdqbIEkSPp8Pp9PJxMQENTU1iTYppXniiSf4+Mc/zoYNGzAYDKjVavm1e4n1ajQaGhsb0ev1vPXWW3R1deHz+e7LxpQUWa1WS0FBAVu2bKG5uZmdO3eSl5dHYWGhPFSI9mi3bNlCeXk5TqeT3NxccnNzmZ+fR6fT0dnZqYgsyL6qq6ujuLiY9evX09DQwMaNG8nNzcVoNKJSqZAkiVAoxMTEBCMjIywtLZGRkUFzc7M8ojCbzZjN5ntq4A8C0UkZvV6PxWKRe10K98bS0hKBQECejAXkeQGv10swGLzp+6K9XL1e/4HXos/rdLoVon2vpKTIWq1WHn30UZ544gkOHjwoO4dHeaIAACAASURBVEMIgdfrZWRkBKPRiF6vp7m5ecWQQQhBWloaNTU1vPrqq/d9l1oL6PV6rFYrX/rSl9i0aZPssxsbWCgUwu12c/jwYX72s5+h1WrZsWMH3//+91Gr1fLsbmZmpiKytyCaXZCXl0dxcTFarTbRJqU0IyMjnD17ls2bN2MwGIDlEWwwGOT8+fMr5gei4QKNRkNTUxNZWVkUFxfHvK2mjMgKIVCpVBQVFbFx40a++MUvyo3U6/Xi8Xhoa2tjaGiIjo4ODAYDJpOJL3zhC+Tl5ZGRkSE7c25ujqGhIWU4ew273c769evZvXs3xcXF8mggiiRJBAIBxsbG+PnPf86xY8eYnp4mNzd31dJcHhSEEOj1etLS0j7Qk9Xr9ZSXl+NwOFhYWMDpdCpt9Da0t7fjcrkIBoNkZGQAEAwG8fv9XL58mZmZD+4yLoTgxIkT1NbW8h//439Eq9XKHbFIJMLMzAzd3d1cunQJp9N53zamjMhGh1g1NTXU19fT1NQki8HCwgJjY2O89NJL9PT0cP78eXQ6HVarVc6RS09Plz9rfn6esbExpQFfo6ysjLq6Oux2O1ardcVr0aHX4uIi3d3dvPDCC8zMzODxeMjOzlaGu3dJNFyg0+nkOHcUvV5PWVkZw8PDjIyM4Ha7lTZ6G4aHhxkdHaW7u1seFUQnq+bn5/H7/Td9X3p6Ort37+ZP//RPV4zYIpEIExMT9PT0MDExcctww92QMiK7efNmmpqa+PrXv47NZkOn0+HxeJienuZ73/seJ0+epK+vD5/PRygUorm5maamJjZt2kR2djZCCEKhEMFgkCNHjvDmm28qoYJr1NXV8ZGPfGRFfCoa65qZmWFqaoof/OAH9Pb2yjcnIQRVVVUUFhYm0PLUIxQK4XA46Ozs5J133uHgwYNkZi4XIMvMzOQzn/kMBQUFZGRk8POf/5zZ2dkEW5zcRCIRIpEIU1NTK4b9kiTd8galUqnkSdobJ8skSWJ+fp75+XkCgcCqjNSSXmSNRiO5ubk0NTWxdetWOa/Q7/czNTXFxYsX6ejooKurC6/XKzulrKyM+vp6eUUNQCAQwOFwMDY2xuTkpJI0f41oStHCwgJGo5FAIIDH42FhYYGBgQGGh4c5ffo0U1NTctqbWq3GYrGQlpaWYOtTi+jk4eDgIK2trezcuVMWWbVaTVpaGjk5OWRkZKyYS1D4cO50AjsapqmqqsJut6NSqT4gzv39/UxOTj44K77WrVvHn//5n/Poo49SVlaGSqXC5/Px/vvv89vf/paf//znzM/Pf6Bbv2PHDj7zmc/IwXCAvr4+fvCDH3DixAnm5+fjfSlJyy9+8QuOHj3Ks88+i06n4/Tp00xOTjI6Osrc3Bxer5dQKLSi0QkhsFqtK2LdCndOa2sr7e3tPPPMM8poII5s3LiR5uZmnnnmGXlO5/r2u7S0xPe//336+vpW7ZxJK7LRmeuWlhb27t0rD/ndbjcOh4O33nqLzs5OFhYW5LuYWq3GbDZTUFBASUkJaWlpqFQqlpaWmJiY4Ny5c1y4cIGFhYUEX11y4fF4iEQi/P73v0ej0XDp0iVcLhdutxuv13vTYZcQgry8PAoKChJgceoT7dHeeONSWF00Gg01NTWUlJRQV1dHXV0dVVVVlJeXY7FYVvg8GAzidrvxeDyrmtqZtCJrMpk4dOgQu3btoqGhgUgkQjgcZmJiggsXLvCzn/1MToiH5Qaq0+nIyclh+/btlJaWyhMLPp+PI0eOcPToUc6cOZPgK0s+fD4fPp+P11+/VdnPD6JWqykpKaGwsFARh/sgumJOydJYHW5si3q9nl27drFlyxaefPJJebXizSZsPR4PMzMzBIPBVZ1wTEqRNRgM5OTkcOjQIYqKlrcHcrlcTE9P80//9E+0t7ezuLgoOyI7O5uCggL+6q/+isrKSsrLy8nJyUEIwdjYGFevXuXw4cPKmvp74MCBA5SUlMiFTFQqFZFIBK1Wy8aNG8nJWd6iye/343K56OrqoqOjQ5kVv0Ou21RQ4R6wWCxkZGTw0EMPYbPZqK2tXTGRZTAY2Lp1KxaLBYvFglqtXiGw0cU1ra2tnD17lpGREaanp1d1yX1SiqxGo8FsNssFGyRJYmFhgf7+ft5//32uXr2K0WiU6xXY7XYqKyt56KGHyM3NldO1JEliamqKS5cuceHCBRYXFxN8ZclPtJyhXq/HZDLR2NhIVVWVvOor2utSq9Xk5uZiMpnkpaKjo6OMjY0xNzenTCreA4rY3jnRVDibzYbdbmfz5s1UVlbS0NCwQkTVajUFBQWo1eoP+DcSibC0tMT4+Djvvfcep0+fZnh4eFXLHEISi2x0MYFOpwPgyJEj/OAHP8Dr9VJbW8tTTz1FTU0NDQ0N2Gw20tLSMJlMKxwciUR49913OXfuHG63W1lCexuiPq+pqWHPnj08/vjjNDQ0YLFYgA8OxaIN1+/3c+rUKf7xH/+Rzs5ORWAVYo7ZbKayspK//Mu/5JFHHiErK0vOm79ZO70Z0SyPt99+mx//+McEAgE5JWw1SUqRhT8kwUciEVQqFdXV1Rw8eJClpSWsViu7d+8mLy8Pm82G0WiUy5lFK0NFE5KHh4fp6+tThq+3IFrLNC0tjb1795Kbm0tpaSnr16/HbrfLN65AICDXJ4i+D5D9qtFo5NKHSoxRIdZEFydlZGRgtVrlClofxo3iK4TAYrGQlZVFeno6c3Nzt1y8cD8kpciGQiF8Ph9er1cu1LBnzx727Nlz0y+v3++XU7iiveBQKEQgEOD06dO0trbG+xJSguiQq6ioiNraWv7H//gfFBYWYjQa5Rvc0tISXq+Xvr4+0tPTKS0t/UBcSwiBzWZjz549jIyMMD8/r5SRVEgaouJ6o3bodDpKSkrYvn07+/fv55133mFiYvU36EhKkfX7/TgcDg4fPszWrVvZtm2bPBQIBAIEAgEmJiaYmpqip6cHp9OJWq3mqaeeIjMzE4PBgNPpZGhoCJfLlejLSUrUajUZGRlUV1fz9NNPs3fvXmw2G5FIhM7OTubm5hgZGWFwcFBO52pqauKzn/0sBoNBDuOoVCq0Wi12u52nn34ah8NBWloa58+fl4dfCndGVAwyMzOpqalZkeOtsBKv18vVq1d55ZVX6OvrY8eOHWg0GgYGBgiFQitCg9eLbE1NDTabjdLSUrkNx5qkFNnouuNXX32VxcVFmpqa5ILci4uLOJ1OTp8+zYULF3jvvfdwuVxkZWWxb98+eQWS0+nkypUrytLZmyCEwGAwkJuby8aNG/nIRz5CY2MjkiThdrs5duwYfX19XLlyhXPnzuF0OsnPz0eSJJ566im5cUYFNFpT1mq10tLSIn8BXC6X7P/rexFKKOEP3CyFy2q1Ul5ejtFoVEIvtyAYDDI1NcVvfvMbzp49y/z8PBqNhhMnTuDz+W45ktq/fz9btmyhoKAgbhXQklJkYVloL1y4wMDAAL/5zW/kUobRnqzL5cLj8eDxeFi3bh21tbVUVlZiMpkQQuByuRgZGVmVAg9rCZVKRXp6Ol/+8pfZvHkzBw4cIC0tDUmSGB8fp7+/n//zf/4PU1NTzM/Po9Vqqays5H/+z/8pbz0jhCAcDtPa2orX68Vms5GXl0dWVhZPPPEE+/fv52Mf+xidnZ28+eabeDweOda1uLjIwMCAIhzXuFkKV05ODhaLhc2bN+Pz+RgeHlb8dQucTidut5sXXngBWG5fH5YWNz09zalTp3jooYcwmUxxsTFpRVaSJLxeL16vl8nJyQ89VqfTyZkIWq0WSZJYXFxctSo6a4n8/HxKS0tpaWmhvr6ejIwMAoGAnCLX1tbG4OAggUAAgMrKSvkmlp2dDSwP1VwuF2fPnmVhYYGKigqqqqqQJAmDwUBaWhrV1dUYjUbm5+fx+XwEg0GWlpaYnZ1Vtqe5Drfbjc/nWxEa0Gg0CCEoKiqirKyMkZERRWRvQTgcJhwO43A47uj4xcVFXC5XXMNYSSuyd4Ner5crSEUbY3t7O6+99pqyhPYa0YUEzzzzDLt27eLxxx9Ho9EQCoVoa2vjyJEjvPrqq3KeazSx++tf/zobNmygqKiIcDiMy+Xil7/8Jb/97W/53e9+h8fjkYe39fX1PPzww1RVVdHQ0EBBQQHbtm0D/nDT7O7u5le/+pUisiz75N1330UIIccUowghOHToEFarlVOnTimx7VWirq5Orr4VL9aEyA4PD5OVlSU3xOhwQfki/4H8/HwqKyvZvXs3TU1NqFQqWfSOHTvGxYsXMRqNrFu3DrvdTllZGevWraOhoQGr1crY2Bizs7O0t7dz7NgxOjo68Hg8cuhmeHiYQCBAOBymo6ODQCBAbm4u+fn5wHL8tq2tjXPnzin/l+twOp1MTU3dtKd646agDzrRhTIlJSVyFb5wOHxHvfz09HSqq6t5+OGH2bNnT3KJrBCiBPgpkA9IwPOSJP1vIUQW8BJgBwaBZyRJSkhpq6GhIYCUu9vH07d2u52DBw+yc+dObDYboVCIqakpfvazn8nbdGzfvh273c7HP/5xSktLKSgokAvsvPnmm1y4cIFXX32VmZkZ3G63/Nl+v5+JiQm5roTJZGJ6epra2loefvhhYHmi4gc/+AHnz5+P26KQVGi78/Pzq1pWL14kwrdqtRqTyURLSwtzc3M4nU75xn47cnJy2L9/P0899RT19fVxvXndyZlCwDclSWoTQliAViHE/wP+A3BEkqTvCCG+BXwLeC52pt6a2tpampubU7FKf9x829jYyOc+9zl5iw4hBDk5OXz2s5/l4x//OKFQiOLiYtLS0khPT0er1aJSqejt7aWnp4cXXniBoaEhJicnbxvn9vv9vP3225w6dYrf/va3wHLsrKurC7fbHU9BSfq2e+TIEcbGxvj4xz8upx/Cci+2ubmZ9PR0fvjDHzIzM5Nsoa+4+TYnJwe73c6jjz5KdXU1O3bs4Pz583R1deF0OvF6vTd9n9FoJC0tjW3btrF582Y+//nPk5ubu2LRQjgcZmlpiWAwGLNO2m1FVpKkCWDi2u9uIcRloAj4Y2DftcP+DThKghpqQUEB+fn5KSey8fRtZmYmNptNTluJ7jVlt9vlnT6j9QmiCxG8Xi+XLl3izJkztLW14XK57kggI5EIY2NjjI2Ncfny5fsx+75IhbY7Pj6OJEl4PB7MZvOKFXUWi4X8/HwyMzNXjBySgXj6Nisri4aGBvbs2UN1dTUlJSUsLS3JNQlule+alZUlF/zfvHkzRUVF8qRitKpfIBBgfn6excXFmI2w7qrPLISwA83AaSD/mqMBJlkeNtzsPV8GvnzvJt4em81GUVFRSpfci7Vv/X4/i4uLaLVaecXW9YsKJEliZGSEmZkZ2tvb6e3tpbOzkytXruB0Ou9YYJOVZG27sNybGhsbQ6fTrdhOPbq0PNn9HmvfHjhwgG984xsUFBTIy2fz8vL4y7/8S3le4GZs2bJF3hVYo9HIHQxJkpicnGRiYoLXXnuN8+fPc+TIkZitUrxjkRVCmIGXga9JkuS6YcsGSQhx05YgSdLzwPPXPiMmrcXtdq8YSqWa2MbDtyMjI5w8eZJ169aRnp6O0WiUJwfdbjdut5uOjg4cDge9vb0MDw9z6dIleTO6ZP+ifxjJ3HZhuRr/lStXsFgslJSUyPU3UoF4+Fan05GWlrZim3qj0UhTUxNLS0u3LCpfWFiIxWLBYDDIud3RnWxbW1vp7u6mvb2dvr6+mKZ63pHICiG0LDvysCRJr1x72iGEKJAkaUIIUQBMxcrI23H58mV5vX2qrZCJl2/feOMN3nnnHXkbn8bGRsLhMHNzcxw/fpxTp06xsLAg57OuFZK97QJyUfloHDZVSKRvo2U479BOuZbJ0NAQvb29/O3f/i0DAwMxKQhzI3eSXSCAHwOXJUn6l+teeh34AvCdaz9/GRML74CZmRkGBgZwu92oVCr0ej0WiwWbzbZie5pkI56+DYVCeDwezpw5Q19fH11dXUQiEXw+H1evXpVnalPpBnU7UqHtwnK4ILq1tcfjQafTxW1d/b0ST992dHRw+PBh9u7dS2FhoZz1citCoRBLS0ty3HVmZoaZmRnOnj1Lb28vIyMjcV2odCc92V3As0CnEOLCtef+hmUn/rsQ4s+AIeCZ2Jh4exYWFhgfH5d3W9XpdFitVoqLi+XVS0kqHnHzbXQy6+rVq1y9evV+Py5VSPq2C8v/m8nJSRwOB3NzcysyQKJikYTpiXHz7eXLlwkGg6hUKjZt2kROTo6c/RIl+v2WJIlgMIjL5ZKzBjo6Orh06RKvv/46g4ODzMzM3K9Jd4WIp/jEMq5lsVj45je/yd69e3nooYcYGRmht7eX//W//hcXL15clfXfkiQlbaAslr6NB8nsW4i9f7VaLRaLRd6bLprH6fF46O7uvu99p5LZv7fzrUajQafTkZubS0FBAZ/61KfYtGkTW7dulbMFopO2HR0d9Pb2cunSJaampvB4PCwuLuL3+3E6nau+fxfc3rdrZjlJKBSiu7ub/Px8duzYgdlspqKigurqagKBAGNjY3e8OkRBId4sLS0xNzfH3Nxcok1JOqKlC0dHR3E6nbS2thIIBMjJyZE3Rezs7GRkZISOjg4GBgbo6OhImrrGa6YnK4QgIyODRx99lL/7u7+jqKgIs9lMV1cXra2t/PVf/zVut/u+nJ7KvYFkJ5l9C4p/Y8nd+jaagnj9qq1oByqa8hbP8MoD05ONbuZ3+fJlXn75Zfbt20dFRQXZ2dnU1taSmZlJOBxOtlUzCgoKd0m08lYy9FLvhDUjsrCccN/V1SUHtx955BH279+PSqWisLAQn8+niKyCgkJcWTPhguvOIe9bVVBQgN1ux+/389577+Hz+e4rL24tDbmSjWT2LSj+jSVr3bdrTmRjidJQY0cy+xYU/8aSte7beIcLZgDPtZ/JTg4r7SxLlCF3iOLb2LIIdCfaiDsk1fy7pttuXHuyAEKIc5IkbYnrSe+BVLHzelLF5lSx83pSyeZUsjVKqth8L3amVm1ABQUFhRRDEVkFBQWFGJIIkX0+Aee8F1LFzutJFZtTxc7rSSWbU8nWKKli813bGfeYrIKCgsKDhBIuUFBQUIghisgqKCgoxJD7ElkhxGNCiG4hRN+1nSlX5dh4IoQoEUK8K4S4JIToEkJ89drz/00IMSaEuHDt8XgCbFP8Gzu7FN/Gzi7Ft9cjSdI9PQA10A9UADqgHai/32Pj/QAKgE3XfrcAPUA98N+A/y+Bdin+VXyr+HYN+PZ+jNgBvH3d398Gvv1hxwJSKj/i/E++W/8m3D9r2LdK21V8e8++vZ9wQREwct3fo9eeW4FY3vr3/wKb7uNcDyK39a8Q4stCiHMs+zelEUJkxvF0StuNHYpvbyDmE1/S8ta/z5HgzerWIpIkPS8tL/F7LtG2rAL/nGgDbkRpu7HjQfLt/YjsGFBy3d/F1567k2MVbs/d+jfV2RrHcyltN3Yovr2R+4i9aIABoJw/BK0bbnNswuMn9/OIc9zwbv2bcP/c5+PFJPat0nYV396zb++51KEkSSEhxF+wHLhWAz+RJKnrNse+ea/ne9C4B//G1b47wWKxYLfbgeUtQwYGBj6saPrX42VXItuuEAKDwYDBYMBkMpGVlYVer2dxcZFgMEggEIiel+np6ZTZYiWKogsfRCnafRdISuHju+Kxxx7jO9/5DkIIgsEgn//857l8+fJNj01m38Lq+ddoNFJTU8P69evZsGEDhw4dIjs7m4GBAaampujt7QWWd6/90Y9+xPj4OKFQ6L7Pm8z+Tca2ezfczrcpv8eXSqUiNzeXzMxMdu7cicPh4M031/SNMenR6XTs37+fnTt3UlBQwPz8PC6Xa9X3u08lhBDU1dVRXV3NRz7yEUpKSiguLqaoqAiDwUBlZSVFRUVUVFQAEAwGOXPmDJIkMTo6Sjw7QwqrS0qLrEqlQq1WU15eTm1tLV/72tdobW1VRDbBpKWl8bGPfYxNmzaRlZXFwsICU1NTKTf0XU3UajW7du2ipaWFZ599Fq1Wi1qtll9PS0tbcXwoFKKpqQmv18vY2JgisilMSousRqPBbDbzV3/1V2zfvh2r1cr4+DjZ2dl4PJ772jRR4d74xCc+QUNDA4cOHcJqtRKJRJiYmKC1tRWv15to8xJGOBzm+PHjAHz6059GrVavENkbUavVfOlLX5J3W+7t7WVoaChe5iqsIiktsmq1Gp1OR0NDA2VlZfj9fsxmM2azWZ5AUIgvFRUVbN26ldzcXFQqFeFwmKmpKYaHhx/o/4kkSYyPjzMyMoLH40GtVqPRaIhOWEZ7qtG/hRAUFhai0+koLy/H6XQqIhtjVCoVQgiEEGg0GvR6PQaDAY1mWSYlSSIQCODz+e6qw5DSIns90VnbzMxMCgoK8Pl8LC4uJtqsBwohBJWVlVRXV6NWqwkGg4yPj/Puu+/y8ssvP/AjC5fLRVdXFy+88AIHDx6kublZ/gJHhddoNMrHa7VarFYrDz30EH6/n3PnziXK9AeCnJwcTCYTRqOR6upqdu3axd69e+UMGa/Xy/nz5/n1r3/NT37ykzv+3DUlskIIOU6bjClNqYTRaMRisVBRUYHRaOTMmTP4/f5bTl5ZLBZycnIoLCzEbDYTDodxu920tbUxOjqKz+eL8xUkH5Ik4Xa7aW9vp6KigvXr18u9p3A4/IE2K4QgEokwPDzM/Px8gqxeu2g0GrKzs7FareTm5lJXV0dmZiYmk4m8vDw2btxIWVkZGRkZwHLcvKKigpycnLs7TyyMV0h9oo3s85//PHl5efz5n/85Y2NjuN3umx5fUVHBnj17qK6uJj09Ha/XS09PD9/97nfp6+uLs/XJi9Pp5KWXXiI/P589e/aQnZ2NVqslGAze9Hiv18vLL7/MyMjITV9XuHfS0tLYvXs3tbW1bN26la1bt5KRkYFOp7tpJ02tVlNcXIzNZrur86w5kRVCYDQaUamUeuT3Q0tLC5///OdpbGxEpVJRVVWF3++/pcjW1dXx9NNPk5eXhxCC48ePc/r0afr6+nC5XHG2PvkZGBjgyJEjstCazeZbToRptVqlPa8S+/btY8OGDeTm5pKdnc3mzZtJT0+XHxqNhkgkgt/vZ2JiApfLhdvt5sKFCywsLDA/P097e/tdnXPNiGx04kClUq0IVivcGyUlJXLGRigUknORBwcHP3BsdJJm/fr1pKWlyTPpFy5cwOFwxN/4FKC/v5/33nuPsrIy1Go1BQUFNxVZIQRmsxmdTpcAK9cOKpUKjUZDc3MzTz75JMXFxVgsFnJzc1csgY1EIoRCIebn57lw4QIDAwOMjY3xyiuvMD09fcsRx4ex5pTIYDBQX1/PxMSEMsS6D8LhMEtLS/LNS6/X3/SLbjAYqKqqory8HIvFglqtJhwOo9fr0ev18TY7Zbh69SoOh4Ouri5qamr4+7//e7KysjCbzSuOi6Yovv766/zoRz9KkLWpixACq9VKY2Mjn/zkJ3n44YcpKysjEokQiUSYmZmhv7+fY8eOyeEwv9/P/Pw83d3dBAIBgsEgTqfznhfTpLzIRu8+kiQhhECr1ZKdnY3JZEq0aSmJVqvFZrPJsUJJkgiFQvh8vg+kYAkhMJlMNDQ0UFxcLAtstFEuLCwk6CqSn+gkrcfjwePx3HKxgUqlwmq1YjAY4mzh2iC6WKmhoYFt27aRnp5OJBJhenoaj8fD7Owsra2tnD17lv7+fhYWFuQ0rbm5uVWxIaVFNioAgUCApaUldDodOp2O0tJS0tPTE21eSmKz2fjGN77Bvn37yMrKYnFxkampKTo6Oj6Qp2kwGCgrK+PP/uzPqKmpQaPRMDs7y+joKK+//joTExMJuorkp6qqir1791JRUUFZWRk2m+2mI4WlpSVaW1sZHx9PgJWpj9ls5ktf+hItLS1s3ryZ/v5+Tpw4wT/90z/R39+Px+MhHA4TDodX3OhWc4VdSovs0tISfr+fqakp3G43mZnLxfWjvQSFuyMtLY3CwkK2bdtGXl4esBw7bGtrY2ZmZkWeq0ql4sCBA6xfv57q6mqsVivBYJBTp05x4sQJFhYWVqWwyVoiIyODvLw8HnvsMaqqqtiwYQMZGRlymOVmM9qhUIju7m6Gh4cTYHHqodFoKCoqoq6uDrvdjtVqxeVycfHiRdRqNSdPnqSjo4Ph4WFcLldclnqntMiGw2E8Hg+Dg4NUVFTI+WwKd48QgoyMDCoqKqivr0ev1xOJRHjnnXf43e9+h8PhWBH012g0PProo2zbto2SkhJCoRB+v5/Dhw/z7//+7wm8kuRECEFubi7btm3jueeew2q1YjKZ5PzuWxEMBmltbVXmF+4AlUqFXq+nubmZQ4cOsX//fkwmE//4j/9IT08Px44do7W1ld7eXgKBQNzqQaS0yCqsDjk5OeTl5fGf//N/prGxEaPRyMLCAlevXuX06dOcO3duxR1/+/btNDc389hjj5Gfn48QAofDwfHjx+nv70/glSQvQgjsdrscyormYt5u0YzJZOLTn/40J06c4LXXXouTtalJTk4On/3sZ9m3bx87d+7E6XRy9epVTp06hcPhIBQKsbCwQDAYjGvBnTUpssqKrztDrVbLcdXa2lp27tyJzWZDo9HIS2KdTqc84RUVhcrKSjZu3Eh+fj4mk4lQKMTU1BSnT59mdnY2wVeVvGg0GlQq1YqJ2jt5T319vVxnVuHWpKWlsW3bNurr68nJyWF6eprR0VFGRkaYnp5OWPhqzYmsWq0mOztbSR+6DWq1mtLSUh577DGeffZZmpqa0Ol08jJPq9VKQ0MDn/vc59i0aRM/+tGPCAaDWCwWNm3axIEDB9DpdASDQdrb23n11Vf58Y9//MDXJ7gVkUiEI0eO0NPTQ0ZGBvv27aOxsfG2HQKVSiUvVlD4cIqLi3nyySflCcRzEM72sgAAIABJREFU587xzjvvJHx+YM2JrEajIT8//wP1ORX+gFqtpqSkhC1btnDgwAFKS0sxGo3yWvmlpSU5RtvY2Ehubi4Oh4NwOEx6ejqbNm2SJxm9Xi9Hjx6lu7sbj8eT4CtLboLBIHNzc7S1tWEwGLBYLCtWc0XFNrp+Pvp89Man8OFEc1+joQC73U5tbS0FBQXMzs7idDoTYteaE1mDwUBFRQUFBQWJNiVp0ev1PPzwwxw8eJCPfexj8vOSJBEOh1lcXESv15OdnU12dra8VYparcZsNqPX69Fqtfj9fiYnJ/n7v/97pQDMHTI/P8/hw4c5c+YMp06dwmw2o9VqgWWRVavVfO5zn6OhoUFZ5XWXeDweJiYmyMvLw2q1smXLFqqqqrh69SoXL17k5MmTCbFrTYjsyMgIQ0NDcok9hVtTW1tLfX09X/ziFykuLpaFNRwOc/bsWUZHR3E4HDQ3N9PQ0IDVakWj0ZCVlSWLQNTHWq2WwsJCvve973Hq1Cl+9atfMT8/f09LDx80pqamOHXqFHq9fkWbVavVFBYWolKpqK2tRaVSkZeXR2FhIdnZ2XFLO0p2TCYTBQUFzM3NyVsbjY+P86//+q9s2LCBzZs3Y7fbycjI4Etf+hInTpxgaGgIp9MZ9xHXmhDZ6elpBgcHiUQicowr+lC27VhJbW0tjY2Ncgw2FAqxtLSEz+fj1Vdfpbe3Vx5aZWdnYzQa0Wg0cjghiiRJqNVqLBYLn/jEJzAajbz//vvyrqsKH87CwsJNV8QJIf7/9s48uM3rutvPxUaABBeAJEgQ3ESJi6iFkijJli1bkt2MXU8d2x3bceS0Sd2JM2raNJ5kpm46bt1/MvX3TZw2k05rpW6dxWmcz20dt7bkNFpsLdZGSVxESRRFgTtBEgQ3kMR6vz9IvCFlSqIkYiPfZwYjEngBHPx0eXDvPeeeQ01NDXa7nfLycnQ6HRkZGeTk5JCZmcnk5OSydrJCCGWVtXbtWpqbm5UTWm63m5/85Cfs3LkTr9eL3W4nPT2dmpoadDodP/vZz5iamlKd7J3g8XiUQiSRgwhZWVkUFBQoqRsq02zfvp0dO3bg9/vp7e3lo48+4syZMzQ2NuJ0OpmamiIcDjM6OsqVK1fYs2cP5eXl5OTkzLtKCAaDXLlyhYaGBpxOp6r1XRCpyL9z506efPJJTCaTss8YDAbn7DcuR7KysrDb7bz22msUFBSQk5PDvn37OHLkCB9++KFylPvo0aO0t7fzuc99jtzcXGWSYLVa43IKcUk4WZ/P95lz9QaDAaPRqJ78ug6NRoOUkvb2dtrb2zl9+jTHjx+npaVlznVut5v29nampqaUP2y/34/X6yUYDCrFMsbHx2lsbKSrq0udwd4lqampOBwOSkpKlMBtJLfT4/Hg8/mWdcffqqoq1q5dy9q1azGZTMpqKhwOK9eEQiHC4fCctj6Rk6E+n2/OtbFiSTjZiLCzv+UjPXpUJzuXxsZGPB4P+/btw+Vy0dPTM+/AS0lJITc3l/z8fCwWCxqNhu7ubr7//e/T09NDX18fAJOTk3R2dqqtfhaBqqoq9uzZw6pVq5T7fD4fH374IadOnaKvr2/ZzmSFELz66qts27aNlJQUXC4Xb7/9Nvv37+fYsWNzvnxqa2t58sknyc7OVjpL1NXV0dTUFJdxeksnK4QoAn4C5AES2Cul/AchhBV4BygFnMCzUsq49Mhoa2sjPT2diYkJpW5kpEdPX19fwuZuxkPbxsZGpQxkpDjGdTah1+spLCykqqqKjIwMpV+X2+3m7NmzDAwMKIW4Q6FQwgZj4qGvXq/HbDbz6KOPEggEuHLlCp2dnTet6BQpNF9aWsqWLVvIyMhQHgsEAjQ3N9Pe3p5QDjbW2kopuXr1KiUlJUpNgoceegiHw8HnPvc5AKXl0dq1a9myZQsGg4HJyUk+/fRTzp8/z+TkZHxWArML1s53A+zAppmf04EWoBr4P8DLM/e/DLy2gNeS0brZ7XbZ0tIiR0ZGZDgcli0tLfIXv/iFLC0tXbT3uNXnu91bImqr1WqlxWKRX/rSl+R7770nPR6PDAaDcmhoSL733ntR+/9bbG3jpW9mZqasrq6Whw8flu+//778xje+IWtqam6peUFBgXzppZfk1NSUDAaDMkJPT4+sra2VNpstofSNh7Yvvvii/Jd/+Rc5Pj4uA4GADIfDc26BQEC63W7p9XplOByW4+PjsqOjQz799NOyqqoqbmP3ljNZKWUv0Dvz85gQ4iLgAJ4Ads5c9mPgMPAXt3q9aOH3+2lubiYcDmMymThy5AhvvfUW/f398TLpliSithaLRSlu/OCDD6LX6xkbG+O1116joaEhFiYsGvHQ9wtf+AJbtmxh/fr1BINBHA4Hqamp6PV6BgcHmZycxOVyYbFYyM/Px+FwYLPZ+PznP8/q1avR6/XKoZCWlhbq6+tv2lstXsRD23379nHx4kXMZrOiW6TDbITI7HVgYIAPPviApqYmPv3007jWNr6tPVkhRCmwETgJ5M0IDdDH9LJhvue8CLx45yYujHA4jMvloqCgQNmHOX369GcCYolKImir0WhIT09n7dq1VFRUKA0Rh4eHOXToUFIXf4mVvna7nYqKCqW1d2lpKTU1NQwODtLf36/UdigsLKSiooKCggLy8/O59957lb1vKadzl1taWmhoaGB8fDwht2MixErbzs5OPB4PJ06coKCggJUrV7Jq1ao53WOFEAwPD9Pb28vx48c5ceIE/f39yXGsVghhBv4D+KaUcvS6nEkphJDzPU9KuRfYO/Ma816zmESm6JEK/TKB9rFuRCJoK4QgLy+PTZs28fjjjytlIy9cuMDp06dpbW1N2rbUsdT3f//3f+nu7sZms2Gz2cjKyuL3f//3+fznP8/o6Cher1d53G63K8dqIzNYmA4mjoyM8Oabb3LixImbdk6IN7Eeu16vlx/96EeKZnq9/jOphaFQiGAwyMTEBH6/P+4ZGQtyskIIPdNCvi2l/M+Zu11CCLuUslcIYQcSYl2u0WjQaDRYLBYKCwvp7e1N9FlAQmir0WjYtWsXGzduJDMzEyklo6OjnDt3jrNnzzI5OZmwf+g3I9b6dnd3k5qaysDAAABTU1NkZWUptQjS0tIwGAykp6eTlpY2p0BMZHLQ0dHBuXPnaGlpYXh4OGF1j8fYlVImXY2MW+Y3iekR8CZwUUr5+qyH3ge+PPPzl4FfLb55t49Wq0Wv11NdXc1DDz2U0NWLEklbg8HASy+9xFe/+lXMZjMTExOcPXuWn//85/zbv/1bUtYmiIe+7e3t1NXV0djYyCeffMIbb7xBV1cXgFIUpqioiKysLHQ63ZxTdOFwGL/fz49//GN2797NpUuXEjb3OJHGbqKzkJns/cAfAI1CiPMz930H+Dvgl0KIPwbagWejY+LCMBgMbNiwAbvdTjgcZmhoiO7u7oSexZJA2oZCIc6fP49Op2PdunWMjo5y+vRpBgcHo/3W0SQu+nq9Xv7rv/4LrVbLxMQEFotFSXnT6/VYLBaysrLmpGoFg0EGBgb47//+b+rq6hbTnGiRMGM30VlIdsFR4EZ11h5eXHPuHJ1Oh91uJyMjg3A4zPDwMH19fQl9zDORtA2HwzQ3N2M2m6mqqsLtdnPp0qWk7jgbL339fj+/+c1vlJzt/Px8ZSVgMplYs2YNK1euVIJjUk6fSuro6OBnP/vZZ07fJSKJNHYTHRHL/Z5oBr5sNhvvvPMOBQUFWK1W/vmf/5m33nqLzs7ORVtySSkTtqjnYmhrs9lITU2loKCAoaEhXC4XY2NjMfmiSmRt4c71jRRAj6QZaTQajEYjZrNZmcmGw2FCoRDDw8N0dHTg9/sXXfNE1jcWAfFocittl8SxWpiePZw7d46BgQHsdjvd3d1MTk7G5axyshLJKXY6nfE1ZAkhpbxhxS2V5cGSmckCShqMRqMhEAgsetUidTYQPRJZW1D1jSZLXdslM5MFEj3IpaKisgxRS1SpqKioRBHVyaqoqKhEEdXJqqioqESRWO/JDgLemX8TnRzm2lkSL0MWiKptdBkHLsfbiAWSbPou6bEb0+wCACHEGSnl5pi+6R2QLHbOJllsThY7Z5NMNieTrRGSxeY7sVPdLlBRUVGJIqqTVVFRUYki8XCye+PwnndCstg5m2SxOVnsnE0y2ZxMtkZIFptv286Y78mqqKioLCfU7QIVFRWVKHJXTlYI8agQ4rIQolUI8fJiGaUyjapv9FC1jR6qttdxFy2BtcBVoAwwAPVA9U2uf5TpPMNWZloGJ8INKAIOAc3ABeDPZ+5/FegGzs/cHouxXaq+qraqtktA27sxYhvw0azf/xL4y8UQPsZi3qh//KvAt+Nol6qvqq2q7RLQ9m5OfDmAzlm/dwH3XH/RTOvfl5g+KTG7p/SF2f2N4s11tlyYdf//jfwsY1su7pb6zmqrnMb0QE1afRNNW1DH7h2iansdUQ98yenWv68A/y/a77XckFLuldOnT16Jty13ixDCEm8brkcdu9FjOWl7NzPZbqb3LSIUztynsjgkrL7p6elYLBb+6I/+iLy8PIQQHDlyhJ///Od387LfA15YJBNvRcJquwRQtb2Ou3Gyp4FyIcQKpkV8Dth9g2uvF17l1tyuvlFHCIFOpyM7O5uysjKee+457HY7UkqCweDdOtmti2XnAlDHbvRQtb2OO3ayUsqgEOJPgY+Y3sD+VynlhRtcfhoov9P3Wo7cgb5RZ82aNXz5y19my5YtFBcXk5+fj5SS9vZ2XC7X3b5802LYuBDUsRs9VG0/S8xOfAkhHgM+iMmbRYkYB2dui2j1SdJqtRgMBlasWMG9997LCy+8wIoVK7BarWg0GiYmJjh8+DD/8z//w5tvvnk3b1UgpexdLLsXE3XsRo/loO2SaqQYbRJ1oEL0tLVYLJSUlPDGG29QWFio7MFKKRkeHqa1tZXnn3+ewcHBu+rImsjagjp2o8lS1zbujRR1Oh1paWmsWrUKgFAoRG5uLmlpaej1etLT0ykuLsbj8TA6OnrDNt9SSsbHx/F4PHR0dDA8PMzExESsP86SwWg0kpaWxsMPP0xlZSWFhYVkZmai0WjweDwMDAzQ1tbGhQsXGBoaYnJyMt4mJxXl5eXcc8895OXlAfD2228zMjKi6rgEibuTTUlJITs7m127dikBlIqKCnJzczGbzRQWFlJVVcXg4CD9/f0MDQ3h9/s/8zqhUIi+vj46Ojo4cuQIly5dYnJykljO1JcSqampFBYW8thjj7F582Zyc3PR6aaHy9WrVzl48CBNTU10dXUxMjIy7xefyvwIIbjnnnvYs2cPVVVVAJw6dYrLly+rTnYJElcnq9Pp+MIXvsDatWt5+umnlWWo0WhEp9Oh0WjQ6XRotVqsVisZGRmEQqEbOs5gMIjP52P37t24XC76+vo4deoUPT09HDlyhOHhYcbGxmL8KZOTlJQUrFYra9asobS0FI3mtynVk5OTuFwuzpw5g8vlUh3sbSCEICUlhby8PFasWEFqaiqhUIgtW7ag1+s5ePBgvE1UWWTi6mSFEKxevZp77rkHh8Nx/ekKYHobIBwOo9Vq0Wq1n3lMSokQQrkB2O12CgoKGB8fx2g0cvHiRS5evEggEFCd7ALQarVkZWVRWFhIdnY2qampwPRqwe/3MzAwgNvtxuVy4fF44mxt8iGEwGg0kpqaqozp/Px8bDZbnC1bumi1WmUcazQatFotOp0Oo9GoXBMMBgmFQgwMDCzqxCHuM9nt27dTWVk5r4P1+Xz4fD4GB+fvrzY2NobH4yE3N5fU1FRlHzEjI4PU1FRMJhOPPPIImzZtorm5mXA4TF9fX7Q/VlJjMBgoLS3lueee44UXXiAnJ0d5rLe3lx/84AccPXqU+vp6pqam4mhpciKlZHJyErfbTX9/P4WFhQghsFqtmM3meJu3ZCkrK+PZZ59Fp9Oh0+lYsWIFhYWFbNy4UfE9Ho+H3t5efu/3fu+GPudOiKuTDYVCNDY2Eg6HycvLU2ZKMD0YvV4vY2NjXLt2bd7nj46OMjw8THZ2NmazmYyMDFauXElNTQ0ajUa5paWlUVxcTF9fH/X19bH8iEmHyWRi3bp1rFy5EovFomzBdHd309DQwPnz5+no6FD3Du+SoaEhnE4nNpsNo9FIVlYWaWlp8TZrSWAymUhPTyctLQ2j0UheXh6rV69mx44dyoo4OzubjIwMzGaz4mQjs9unnnqKxsZGTpw4sSj2xNXJBgIBvvvd75KTk0NlZSVut5v29nZg2slOTEzg9Xpxu923fK2IM33mmWd4/fXXMRqNGAwGYHp2dv/99y/qt9NSJTMzk127drF+/XpMJhOdnZ04nU6+9a1v0dnZSX9/f7xNXBI0NTXx7rvvUllZSWpqKg6HY86qQeXOsdlsbNy4kdLSUmw2G/fddx8Oh4OysrJ5r4/EeIxGI7m5uXz3u9/l448/5umnn14Ue+LqZKWUuN1uJfVqamoKr9erPB4MBgkGgwt6LZ1OR21tLRUVFUrgLPIeU1NTHD58mMuXL0flcywFdDodX//616murmbXrl1kZGQwOTnJ0aNH+fTTT+nq6mJ0dDTeZi4ZPB4Pra2t+Hw+NBoNK1euxOVyYbFY8Hq982bQqPwWnU6H2WzmkUceYe3atVgsFvR6PQBWqxWbzUZWVhYmk4msrCy0Wi2jo6NzJl/zIYRAr9ff9JrbtnXRXukOGRsbY2xs7K5mmZFc2+rqalatWjVHoEiw6/z587S1tS2GyUuOSCDmySefpLKykpycHAKBAF6vl1OnTvHrX/+awcFBQqFQvE1dMoyPj+N0OvH7/Wg0GqxWK8XFxWRlZREMBlUnexM0Gg0mkwmbzcYDDzzAo48+itVqVYKIkYwknU6HEIJQKMTU1BQul4ucnBw0Gs2cLYLZRALo199/N8Tdyd4NkSjhV7/6VTZt2sSjjz5KRkaG8ngoFOKHP/whdXV1HDt2TA3U3IDVq1dTU1NDRUUFOTk5aLVaBgYGqKuro6enB5/Pp+YbR4nIH3XE0T722GN8/PHHNDXFrJRDUqHX67n33nvZunUrX/nKV8jPzyc9PX2O45z979TUFH/zN39DT08P/f39SvzGYrGwYsUKnn/+eVJSUpSJWTgcZnBwkO7uxau5lJRONrL/mp6ejtVqZcOGDdTW1pKTk6NsE0xNTTE+Pk5DQwNnzpxhfHxcdRTXEfnjzs/Pp7CwkNTUVEU/n89HX18fIyMj+Hy+eZ+fk5ODwWCYs3JwuVxqUOwOiFQ4s1qtSqqRymfR6/VUVVWxdu1aSkpKSElJUcZsBL/fz9TUFIFAgOHhYU6ePEl7eztjY2OkpqaSlpbG5s2bycrKmvO8cDhMMBjE6XQqsaHFICmdbFpaGrt27WLdunXcd999bNu2jczMTOUbLBwOc+7cOfbt28cnn3xCX1+f6mDnQavVkp6eTkVFBbW1tXMGazAYZGRkhO7u7nkrbOl0Op555hmKiorIy8tDo9EQDof54Q9/yLlz52L5MZKayLiM5HtHDuGozI/ZbOZLX/oSK1asmDcbIxQKce3aNd577z0uXbpET08Pp0+fVlZjw8PDpKSk8OKLL3LvvfeSlpam6D01NYXb7eY73/kOra2ti2Zz0jlZm81GeXk5u3fvpqSkBLvdjslk+kye7cTEBENDQ/h8PnUv8SZIKcnIyKCgoGDOPtTExAQdHR1zApEWi4V169YhpSQlJYX777+f4uJicnNzlWs0Gg2XL1/mrbfeUs/iLxB1ArAw8vPzKS8vx263z9kWhOnYy9TUFA0NDZw8eZLjx4/T2dmJx+PB7/fP+TILh8NkZGSQlpY2x28MDw/T2dmJy+WaM+7vlqRyshqNhtLSUrZu3cpTTz2lRBPnw+v1Mjw8fNNjuCrTg85qtZKfn49Wq1WONo+NjXHp0iXGx8eVa+12O7/7u78L/PYgidVqnTNYq6qqGB8f5/jx40r9CBWVxaC8vJyamhry8vIwmUxzHvP5fPT29vL3f//3NDU10dLScsPX0Wg08+Yl9/T0UF9fr2Q6LRZJ52S3bdvG1q1bbxr902g07Nixg3Xr1vH8888rifSdnZ00NTXR29urVuhiOid28+bN1NbW4nA40Gq1SiTW7XYzNDREMBhEq9VSUlLChg0b2LBhA8XFxVitVjIzM5XnBINBAoEAJpMJg8HAK6+8wtGjR9m7dy8jIyOq3ip3zc6dO9m1axcGg0FZ4kcOLO3bt48LFy5w/PjxeY/Oa7VaTCYT5eXlVFRUsH79evLz85Xg2OjoKKdPn+bs2bOLntmRVE5WCEFmZiZZWVnKFsDsmgWzyczMJDMzk4yMDDweD2azmcuXL6PX6+fk54bD4WU50xVCkJaWRkFBAdnZ2coZ7mAwyMTEBG63m7GxMcLhMEajkZUrV1JYWKjUhUhPTyccDitOeWpqisnJSfLz85XgxOjoKKmpqaqDvQWJ1J01kbFYLFitVuC3y/6RkRG6uro4ceIEzc3N9Pf3z1t3wGAwkJOTQ3V1NWvWrMFqtZKSkkI4HMbr9dLV1UVbWxtOp3PBufkLJamcbCgU4ujRowwPD+NwOMjOzsZiscz5Zrsei8VCVlYWxcXFilM4ceIEdXV17N27F7fbfVfFppMRIQSpqalkZ2dTWFioLL0i3+iHDx/m2LFjXL16lYKCAoqKinjhhReorq5m9erVimO9du0aTqeTf//3fwemK3f99V//NUVFRUodCZWbozrYhXPs2DE8Hg9f+cpXMBgM+Hw+fvrTn/Lee+/hdDqZmJi4YWGXkpISdu/ezRNPPEF5eTl6vZ5wOIzH42H//v28/vrrdHR0MDY2RiAQWFS7k8rJSinp7OzEYDDwm9/8hsLCQgoLCzGbzRgMBuWkhtlsVpatEec7e3th5cqVGAwGnE4nly5d4vDhw3H6RPFBo9GQm5tLZWWlkpkhpcTj8dDT08PJkydxOp3o9XrWr19PRUUFq1evJjc3V2k5MzIywokTJ7h69SpdXV2UlJRQVlaGXq8nFAopB0y8Xu+iD1qV5UlraytCCA4ePIjBYGBqaoqmpiY6OjoYHx+fN8Ct0Wiw2WysW7eO+++/n7y8PFJSUoDpYFlXVxetra1Rc7CQhE72ypUrXLlyhf3795OZmUlOTo6SYJyXl0dRURG7du1iw4YNStX56yktLaWoqIjCwkIOHDjAJ598sqxqour1empra9mxYwePPPIIMJ32dvbsWY4fP84//uM/YjAYsFqtPPHEE9x3331UVFQghCAYDNLT00NDQwPf+973cLlclJWVUVNTwx/+4R+SmpqK3++ntbWV5uZmterZLYiU61S5NfX19dTX1/Puu+8u+DkpKSns2LGDhx56iAceeECZdEWqoX3wwQfU1dUxNDQULbOTy8lez8TEBP39/YyMjKDX63E6nbS0tNDW1sa2bduora2lsrISs9n8maVr5Btu06ZNPPPMM5w6deqG1b6WGlqtFrvdPqd+aTgc5sCBAxw7doxgMMiOHTt4/PHHefDBB8nPz1faznz00UecPXuWq1evYjKZeOCBB/jGN75BaWkpZrOZvr4++vr6+OlPf8rFixfj+CmTD7/fr3SaUFkc0tLS+NrXvkZZWZlyKkxKSWNjI83Nzfz617/G6XRG1YakdrKBQGDeQtx1dXX09/fT3t7O7t27KSgowGQyzdm3jQR+HA4HW7duxeVyLRsnq9FoyMnJwWKxKPeFw2HOnDnDmTNnEEJQWVnJ448/Tn5+vhIgGBgY4Je//CXnz5+np6eH7du3s3btWh588EHlMEJku+H9999XncVtEggE6O/vX9QczeWMVqtVMmhm1+qNxGVOnTrF8ePHo55Hn9RO9mbU1dXR3NzMwYMH2bBhA3/7t39LTk4O6enpc65LSUmhrKyMzMzMOFkae7RaLWazGaPRiBACt9tNd3c3Ho8Ho9HIqlWrKCsrw2KxoNVqCQQCfPTRRxw7doxDhw5hNpupqqrilVdeYdWqVWi1Wrq6uvj444959913qaurUztQ3AE+n4+Wlha12tkisXv3brZs2YLBYFBmsBGam5s5d+5cTLYJl6yT9Xq9eL1ehoaG0Gq1uN1uzGbzZ5wssKz2YyNotVplZu/3+xkbGyMYDKLRaBQHHDmcECmaEQ6HWbNmjZL25XA4MJlM9PX1cfnyZc6cOUN9ff2iFtdYToTDYSYnJxc9hWg5IoTA4XAosYQIkZomLpdrQXWqF4Ml62QjhMNhRkdHuXTpEpmZmXOOgML07KGtrW3Z9qqKBF5uFnwRQlBeXs769et5+eWXFQc9NjZGS0sLf/Znf6bk1aozWJV4E2nAWlxcTElJyZxZbENDA/v37+fYsWP09vbGJOh4SycrhCgCfgLkARLYK6X8ByGEFXgHKAWcwLNSykX3VFqtFqPRSElJCQUFBUpe67Vr1xYkUKQIr81mU1I3ZuPz+eIWbIiXtoFAgGvXrlFaWgpAeno6xcXFVFRUoNFoWLFiBdnZ2UqgQKvVUlpail6vJy0tDSkloVCIK1euUFdXR3d3NxMTE/j9/oRaFcR77N4ukbG6mLVMo0WiaqvT6bDZbKxcuZLq6mol7dDv9zM8PMy5c+doaGhQDiLFgoWU+wkC35JSVgP3Al8XQlQDLwMHpJTlwIGZ3xcdg8FAdnY227dv54knnmDHjh1s3rx5wZWKImUR8/PzP9OoLtI14dq1a1FN4bgJcdE2EAjQ0NCgVBoym80UFBRQUVHBunXrKCsrw263K0WPNRoNDodDyTGMBLkOHDjAgQMHGBwcZHx8POGcLHEeu7eLRqNBr9cnSxWuhNNWo9GQkpJCRUUFO3bsYNWqVVitVjQaDVNTU1y6dImTJ09y6NChOTU5os0tZ7JSyl6gd+bnMSHERcABPAHsnLnsx8Bh4C8WyzAhBFlZWdx33338yZ80/RxSAAAIrUlEQVT8CWVlZWRkZPDtb3+b06dP3/KPOTU1FbPZzOOPP05NTQ1lZWVzZrKRIiidnZ2cOHEiLsGGeGkrpcTv9xMIBAiFQkrDyW9+85sEAgFSUlJIT09XnGyEyIm5/v5+Ojo6OHr0qNIIMxGJl753itlsZufOnXzyyScJv32VaNoajUYcDgcvvvgimzZtYvXq1WRnZ+P3+zlx4gRnz57l/fff58qVK4yNjcW0Mt9t7ckKIUqBjcBJIG9GaIA+ppcN8z3nReDF2zVMq9WSn5+v1DrNyMhAp9Ph8/kYHh6+4fMirWhsNht2u52tW7eyfv36OXUjI+eePR4P7e3tCdEcMJbaRhKxx8fHmZiYwGAwoNPpcDgcilONbMWEQiGlvkMoFFJOydTX13P16tWYBQ/ulljquxAiegaDQaWWrF6vJy8vL+lagyeCtnq9nvz8fB588EFKSkrIy8tTVqrHjx/n/PnzcTt0tGAnK4QwA/8BfFNKOTp7hiOllEKIeTdIpZR7gb0zr7HgXeaMjAz+6q/+ivXr1yt9eYLBIDabDYfDgc/nw+/3z4nEGo1GVq9ezbPPPsvDDz/MqlWrMJlMSpQ8QigUYmJigh/84AcJ0SI81toGg0EaGxsZHx9nfHyc3/md32HNmjU4HI7PNJBzOp10d3fT0tJCf38/165dUxxssqQaxVrfhTA5OcnAwADd3d0UFxcnbQphImgrhMBms1FZWUlNTY1SAjWi8Ztvvkl3d3fcVlwLcrJCCD3TQr4tpfzPmbtdQgi7lLJXCGEHFnU6GPkWiixnYXrP5YEHHiAnJ4e2tjb8fr9SrzRSymzFihVs376doqIizGbznGr/kSi6y+Wiq6uLixcvxv0AQjy0hWlH63a7aWhoQK/X09fXR2VlJUajcU5Asa6ujt7eXnp6ehgdHVVac9xsNZFIxEvfWxEOh5VC04FAACkler0eu90+b5phIpII2hqNRkwmk7IlGPl79/v9tLW10dzczNDQUFz7+y0ku0AAbwIXpZSvz3rofeDLwN/N/PurxTTM7/fT0NBATk4ONTU1wLST/eIXv6icjPH7/UrtRyEEJpNJaZI2H5EePu+88w6HDh3i4MGDce0KGi9tIwwNDXH48OElWyAn3vreDCml0pU2shozmUzU1tZy9OjRWJtz2ySKthaLhaKiIh577DHWrVunZBKMjo7yxhtv8OGHH8Z9xbWQmez9wB8AjUKI8zP3fYdpEX8phPhjoB14djENCwQCXL58maysLDZu3Eh2drbSYE6r1WK1WgmHw8oSIBIFny/9xefz4XQ66e/vp7W1lUOHDlFfX58ISd9x0XYZkdD6Sinp6+vD7XZjs9kQQmAwGJIihYsE0Xbbtm3cc889ygpXCIHH46Guro4rV64oh2jiyUKyC44CNyp6+fDimvNbQqEQly9fJjs7mytXrqDX60lJSVH2ViOZAtfX44wEtWb/PDExwcmTJ2lqaqKuro76+vp4pWzNIV7aLheSQV+3283g4KAS7U6S9K2E0FYIMWd7MFJjY2hoiJMnT9LW1hbTVK0bkbAnvkKhEN3d3ezfv5+Ghga2b99OZWUlGzduRKvV0tvbS2FhIUVFRcr1g4ODyuzU6XTS1dVFd3c3g4ODHDlyhImJCbxeb1z3Z1RUZvOrX/2KtrY2ioqK8Hq9HDx4MO5xgmQgLS2NoqIibDYbqampCCHw+Xw0NDRw5MgRjh07lhATKUhgJwvTWwYej4exsTFSUlIYHR1lcnISnU6nONnISa1gMEh7e7syI7h27Rq9vb04nU4GBwdpb29X63aqJBw9PT1oNBrq6+sZGxvj4sWL87ZgV5lLWloalZWVSqBQSonP56OxsZGrV6/S2dmJz+eLt5kAiFg6nrtJ1Yjsuc7Odb2+v9fszzL7TP5iFUaWUiZsr5DFTjGKNYmsLURX30iOLMzNS15MElnfO9G2traWPXv28PDDD1NUVKS0sN+zZw9tbW309PTEbFJ1K20TeiY7m0jydixPaqioxILICTyVhTM2NkZHR4ey9dfS0sLZs2dpb29ndHQ0oVatSeNkVVRUVCKMjo7S3NzMyMgIoVCI/fv3c/r0abq6uuKeTXA9SbNdkAgstSVXIpHI2oKqbzS5E20NBgMWi4Xq6mocDgdnzpxhaGgoLkfkb6Wt6mRvg6U2UBOJRNYWVH2jyVLXNtbbBYOAd+bfRCeHuXaWxMuQBaJqG13GgcvxNmKBJJu+S3rsxnQmCyCEOCOl3BzTN70DksXO2SSLzcli52ySyeZksjVCsth8J3Ymx/ESFRUVlSRFdbIqKioqUSQeTnZvHN7zTkgWO2eTLDYni52zSSabk8nWCMli823bGfM9WRUVFZXlhLpdoKKiohJFVCeroqKiEkVi5mSFEI8KIS4LIVqFEAnRghmm+8cLIQ4JIZqFEBeEEH8+c/+rQohuIcT5mdtj8bb1Zqj6Rg9V2+ixLLSdr1rVYt8ALXAVKAMMQD1QHYv3XoBtdmDTzM/pQAtQDbwKfDve9qn6xt1+VVtV27vSNlYz2a1Aq5SyTUrpB37BdH/2uCOl7JVSnp35eQyI9I9PJlR9o4eqbfRYFtrGysk6gM5Zv3eRgINBzO0fD/CnQogGIcS/CiHm786YGKj6Rg9V2+ixLLRVA18ziOv6xwP/BKwENgC9wPfiaF7So+obPVRto8diaBsrJ9sNFM36vXDmvoRgvv7xUkqXlDIkpQwDP2J6aZOoqPpGD1Xb6LEstI2Vkz0NlAshVgghDMBzTPdnjzs36h8vhLDPuuwpoCnWtt0Gqr7RQ9U2eiwLbWNS6lBKGRRC/CnwEdMRxX+VUl6IxXsvgBv1j/+iEGIDIAEn8LX4mHdrVH2jh6pt9Fgu2qrHalVUVFSiiBr4UlFRUYkiqpNVUVFRiSKqk1VRUVGJIqqTVVFRUYkiqpNVUVFRiSKqk1VRUVGJIqqTVVFRUYki/x/WfkKLcWm0UgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 16 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(60000, 784) (60000,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CBAqt6eddASl"
      },
      "source": [
        "### Task 1: Activation functions: implement the softmax function. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tDE8VdYndASn",
        "outputId": "c3061e7a-542e-4a22-b726-e00fd573db21"
      },
      "source": [
        "# activation functions\n",
        "def sigm(z):\n",
        "    return 1/(1 + np.exp(-z))\n",
        "\n",
        "def dsigm(z):\n",
        "    return sigm(z)*(1 - sigm(z))\n",
        "\n",
        "def softmax(z):\n",
        "    ''' The softmax function should be able to work if batch size is greater than 1\n",
        "    \n",
        "        parameter:\n",
        "            z: input matrix. m_batch * 10\n",
        "        return: m_batch * 10\n",
        "            \n",
        "    '''\n",
        "    ##\n",
        "    #--------------------\n",
        "    b = np.exp(z - np.max(z, axis = 1, keepdims = True))# np.max(z)\n",
        "    return b / np.sum(b, axis = 1, keepdims = True)\n",
        "    #---------------------\n",
        "\n",
        "# test the softmax function\n",
        "z = np.array([[1, 2, 3], [4, 2, 4]])\n",
        "s = softmax(z)\n",
        "print(s)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[0.09003057 0.24472847 0.66524096]\n",
            " [0.46831053 0.06337894 0.46831053]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w0J843sHdASp"
      },
      "source": [
        "### Task 2: Create the layer class. \n",
        "    -- Create the Layer class with the constructor function (__init__) and the excution function (run)\n",
        "    -- Know how to create instances using class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c1DdTB-ldASp",
        "outputId": "8e5b4341-b93c-43cc-9434-d480954ba3ae"
      },
      "source": [
        "class Layer:\n",
        "    \n",
        "    def __init__(self, units, input_dim, activation = None): \n",
        "        '''initialize weights and bias. This is the constructor that \n",
        "            will be executed automatically when we create a layer instance. \n",
        "            \n",
        "            parameters:\n",
        "                units: the number of hidden nodes\n",
        "                input_dim: dimensionality of the layer input\n",
        "                activation: activation function\n",
        "        '''\n",
        "        \n",
        "        self.units = units\n",
        "        self.activation = activation\n",
        "        self.input_dim = input_dim\n",
        "        \n",
        "        #initialize weights and bias, and their gradients. 10 points.\n",
        "        #check here: https://www.deeplearning.ai/ai-notes/initialization/\n",
        "        ## \n",
        "        #------------------------\n",
        "        np.random.seed(0)        \n",
        "        sigma= np.sqrt(1/self.input_dim)\n",
        "        self.W = sigma*np.random.randn(self.units, self.input_dim)\n",
        "        self.bias = np.random.randn(self.units, 1)\n",
        "\n",
        "        \n",
        "        self.gW = np.zeros((units, input_dim))\n",
        "        self.gb = np.zeros((units,1))\n",
        "        #-----------------------\n",
        "        \n",
        "    def run(self, inputs):\n",
        "        ''' calculate the net input and activation output of the current layer\n",
        "        \n",
        "            inputs: layer input. (n_sample * n_features)\n",
        "          \n",
        "            return:\n",
        "                self.output. the activation output\n",
        "        '''\n",
        "        \n",
        "        ## \n",
        "        #-------------------------------------------\n",
        "        #calculate the net input. 5 points\n",
        "        self.net = np.dot(inputs, self.W.T) + self.bias.T\n",
        "       \n",
        "        #activation output . 10 points\n",
        "        if self.activation == 'sigm':\n",
        "            self.output = sigm(self.net)\n",
        "            \n",
        "        if self.activation == 'softmax':\n",
        "            self.output = softmax(self.net)\n",
        "        #-----------------------------------------\n",
        "        \n",
        "        return self.output\n",
        "\n",
        "## Test you code to create an NN with two layers \n",
        "# create a layer with 20 hidden nodes\n",
        "L1 = Layer(units = 20, input_dim = 784, activation = 'sigm')\n",
        "print('L1:', L1.input_dim, L1.units, L1.activation)\n",
        "\n",
        "# print out the setup of the second layer\n",
        "h1 = L1.run(x_train[0])\n",
        "print('h1:', h1.shape)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "L1: 784 20 sigm\n",
            "h1: (1, 20)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "61JKVeM9dASv"
      },
      "source": [
        "### Task 3-1: complete the following NN class. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qpgrXzngdASx"
      },
      "source": [
        "### Task 3-2: complete the following NN class.  70 points"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Q6XevqadASx"
      },
      "source": [
        "class NeuralNetwork:\n",
        "    \n",
        "    def __init__(self):\n",
        "        self.layers=[] # an empty list of layers\n",
        "    \n",
        "    ## add you code here. 10 points\n",
        "    # add the add, loss, forward_prop, predict_class functions you implemented in HW3 to this class\n",
        "    #---------------------------------------------------------------------------------\n",
        "    # Task 3.1: implement the 'add' function.   \n",
        "    def add(self, units, input_dim, activation):\n",
        "      ## add you code here\n",
        "      #---------------------\n",
        "      layer = Layer(units, input_dim, activation)\n",
        "      self.layers.append(layer)\n",
        "        \n",
        "    # Task 3.2: implement the cross-entropy loss.\n",
        "    def loss(self, y_pred, y):\n",
        "      los_fun = (1/len(y_pred)) * np.sum(np.sum(-y * np.log(y_pred)))\n",
        "      return (los_fun)\n",
        "    \n",
        "    # Task 3.3: implement the forward propagation process.\n",
        "    def forward_prop(self, inputs):\n",
        "      nLayers = len(self.layers)\n",
        "      h1 = inputs\n",
        "\n",
        "      for i in range(nLayers):\n",
        "        h1 = self.layers[i].run(h1)\n",
        "      output_layer = h1\n",
        "      return output_layer\n",
        "    \n",
        "    # Task 3.4: implement the prediction function. \n",
        "    def predict_class(self, x):\n",
        "      y_predict = self.forward_prop(x)\n",
        "      y_train_pred = np.argmax(y_predict, axis =1)\n",
        "      return(y_train_pred)\n",
        "\n",
        "   #--------------------------------------------------------------------------------------      \n",
        "\n",
        "    # Task 3.5: complete the following 'train' function. 20 points\n",
        "    def train_SGD(self, inputs, targets, lr = 0.001, batch_size = 32, epochs = 50):\n",
        "        '''implement the SGD process and use Back-Propagation algorithm to calculate gradients \n",
        "        \n",
        "        Parameters:\n",
        "            inputs: training samples. n_sample * n_features(784)\n",
        "            targets: training targets. onehot vectors\n",
        "            lr: learning rate\n",
        "            batch_size: batch size\n",
        "            epochs: max number of epochs\n",
        "            \n",
        "        '''\n",
        "        \n",
        "        m = len(targets)  \n",
        "        loss_hist = np.zeros(epochs)\n",
        "        nLayers = len(self.layers)\n",
        "        \n",
        "        for i in range(epochs):\n",
        "            #shuffle the data\n",
        "            idx = np.arange(m)\n",
        "            np.random.shuffle(idx)\n",
        "            inputs = inputs[idx]\n",
        "            targets = targets[idx]\n",
        "            \n",
        "            for b in range(int(m/batch_size)):\n",
        "                b_start= b*batch_size\n",
        "                b_end = min((b+1)*batch_size, m)\n",
        "                \n",
        "                x_batch = inputs[b_start:b_end, :]\n",
        "                y_batch = targets[b_start:b_end, :]\n",
        "                #print(x_batch.shape)\n",
        "                \n",
        "                ## add your code here. 20 points\n",
        "                #---------------------------------\n",
        "                # 1. run forward propagation to produce output for each layer. 5 points\n",
        "                y_predict = self.forward_prop(x_batch)\n",
        "                \n",
        "                # 2. call BP to calculate all gradients, e.g., gW and gBias. 5 points\n",
        "                self.BP(x_batch, y_batch)\n",
        "                \n",
        "                # 3. update all weights and bias. 10 points\n",
        "                for p in range(nLayers):\n",
        "                  self.layers[p].W -= self.layers[p].gW * lr\n",
        "                  self.layers[p].bias -= self.layers[p].gb *lr\n",
        "\n",
        "                #-----------------------------------\n",
        "            lr = lr * 0.9    \n",
        "            \n",
        "            # 4. calculate and record the loss of current epoch\n",
        "            y_pred = self.forward_prop(inputs)\n",
        "            loss_hist[i] = self.loss(y_pred,targets)\n",
        "            \n",
        "            # 5. print out the loss of current epoch\n",
        "            print('epoch:', i, 'loss:', round(loss_hist[i], 4))\n",
        "            \n",
        "        return loss_hist\n",
        "   \n",
        "    # Task 3.6: implement the BP algorithm. 30 points\n",
        "    def BP(self, x, y):\n",
        "        ''' Back-propagation algorithm calculate the gradients for all weights and bias\n",
        "        \n",
        "            x: input samples (n_samples * n_features)\n",
        "            y: ont-hot targets (n_samples * 10)\n",
        "        '''\n",
        "        \n",
        "        nLayers = len(self.layers) # the number of layers\n",
        "        m_batch = x.shape[0] # batch size\n",
        "        \n",
        "        ## add your code here\n",
        "        #-------------------------------------------------------------------------\n",
        "        # 1. calculate gradients between the hidden and output layers. Save them to layers[idx].gW and layers[idx].gBias\n",
        "        idx = nLayers - 1      \n",
        "        net = self.layers[idx].net\n",
        "        o = self.layers[idx].output\n",
        "        h = self.layers[idx -1].output\n",
        "        \n",
        "        errors = o - y\n",
        "\n",
        "        self.layers[idx].gW = np.dot(errors.T, h)\n",
        "        self.layers[idx].gBias = np.sum(errors, axis = 0, keepdims= True).T\n",
        "\n",
        "        ## Calculate gradients\n",
        "        #self.layers[idx].gW = np.dot(delta_k.T,h_j)\n",
        "        #self.layers[idx].gBias = np.sum(delta_k, axis = 0, keepdims=True).T\n",
        "        \n",
        "        # 2. calculate gradients between the input and hidden layers. Save them to layers[idx-1].gW and layers[idx-1].gBias\n",
        "\n",
        "        gh_prime = dsigm(self.layers[idx-1].net)\n",
        "        Uk = self.layers[idx].W\n",
        "        delta_j = np.multiply(np.dot(errors, Uk), gh_prime)\n",
        "\n",
        "        self.layers[idx - 1].gW = np.dot(delta_j.T, x)\n",
        "        self.layers[idx - 1].gBias = np.sum(delta_j, axis=0, keepdims = True)\n",
        "        #----------------------------------------------------------------------------\n",
        "        \n",
        "            \n",
        "    # Task 3.7: calculate the accuracy. 10 points         \n",
        "    def Acc(self, y, y_pred):\n",
        "        '''accuracy\n",
        "            \n",
        "            y: target: categorical values (0, 1, ...9). n_samples * 1\n",
        "            y_pred: prediction: 0,1,2, ..9. n_samples *1\n",
        "        '''\n",
        "        ##add your code here\n",
        "        #---------------------------------\n",
        "        y = y\n",
        "        y_pred = y_pred\n",
        "        value = np.sum(y_pred ==y)/len(y)\n",
        "        return round((value * 100),4)\n",
        "        \n",
        "        #---------------------------------              "
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fsNeWdNXdASy"
      },
      "source": [
        "## Task 4: Create and evaluate a neural network using the NeuralNetwork class. 30 points"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cICrDzhpdASy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4fc9b8f3-7617-4c4a-e13f-8b83504e87a2"
      },
      "source": [
        "# 1. create one-hidden layer NN.  10 points\n",
        "# it should have 50 hidden nodes and 10 output nodes\n",
        "# the hidden layer activation is 'sigmoid' and the output layer activation is the 'softmax'. \n",
        "\n",
        "##add you code here\n",
        "#------------------------------------\n",
        "nn = NeuralNetwork()\n",
        "\n",
        "nn.add(units = 50, input_dim = 784, activation='sigm')\n",
        "nn.add(units = 10, input_dim = 50, activation = \"softmax\")\n",
        "\n",
        "#------------------------------------\n",
        "\n",
        "# 2. train the NN.  10 points\n",
        "#------------------------------------\n",
        "# transform y_train to onehot vectors\n",
        "y_train_dat = keras.utils.to_categorical(y_train)\n",
        "y_test_dat = keras.utils.to_categorical(y_test)\n",
        "\n",
        "# train the network\n",
        "NN = nn.train_SGD(x_train, y_train_dat)\n",
        "\n",
        "y_train_pred = nn.predict_class(x_train)\n",
        "y_test_pred = nn.predict_class(x_test)\n",
        "#------------------------------------\n",
        "\n",
        "# 3. calculte and print out the test and training accuracy. 10 points\n",
        "##add you code here\n",
        "#---------------------------------------------\n",
        "Train = nn.Acc(y_train, y_train_pred)\n",
        "Test = nn.Acc(y_test, y_test_pred)\n",
        "\n",
        "print(f\"The train data accuracy is: {Train}\")\n",
        "print(f\"The test data accuracy is: {Test}\")\n",
        "\n",
        "#------------------------------------------------"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch: 0 loss: 0.5376\n",
            "epoch: 1 loss: 0.3912\n",
            "epoch: 2 loss: 0.3427\n",
            "epoch: 3 loss: 0.3164\n",
            "epoch: 4 loss: 0.3002\n",
            "epoch: 5 loss: 0.2874\n",
            "epoch: 6 loss: 0.2773\n",
            "epoch: 7 loss: 0.2695\n",
            "epoch: 8 loss: 0.2632\n",
            "epoch: 9 loss: 0.2579\n",
            "epoch: 10 loss: 0.2532\n",
            "epoch: 11 loss: 0.2492\n",
            "epoch: 12 loss: 0.2459\n",
            "epoch: 13 loss: 0.2429\n",
            "epoch: 14 loss: 0.2405\n",
            "epoch: 15 loss: 0.2382\n",
            "epoch: 16 loss: 0.2363\n",
            "epoch: 17 loss: 0.2346\n",
            "epoch: 18 loss: 0.233\n",
            "epoch: 19 loss: 0.2318\n",
            "epoch: 20 loss: 0.2305\n",
            "epoch: 21 loss: 0.2294\n",
            "epoch: 22 loss: 0.2284\n",
            "epoch: 23 loss: 0.2276\n",
            "epoch: 24 loss: 0.2268\n",
            "epoch: 25 loss: 0.2261\n",
            "epoch: 26 loss: 0.2255\n",
            "epoch: 27 loss: 0.225\n",
            "epoch: 28 loss: 0.2245\n",
            "epoch: 29 loss: 0.2241\n",
            "epoch: 30 loss: 0.2237\n",
            "epoch: 31 loss: 0.2234\n",
            "epoch: 32 loss: 0.2231\n",
            "epoch: 33 loss: 0.2228\n",
            "epoch: 34 loss: 0.2225\n",
            "epoch: 35 loss: 0.2223\n",
            "epoch: 36 loss: 0.2221\n",
            "epoch: 37 loss: 0.2219\n",
            "epoch: 38 loss: 0.2218\n",
            "epoch: 39 loss: 0.2216\n",
            "epoch: 40 loss: 0.2215\n",
            "epoch: 41 loss: 0.2214\n",
            "epoch: 42 loss: 0.2213\n",
            "epoch: 43 loss: 0.2212\n",
            "epoch: 44 loss: 0.2211\n",
            "epoch: 45 loss: 0.221\n",
            "epoch: 46 loss: 0.2209\n",
            "epoch: 47 loss: 0.2209\n",
            "epoch: 48 loss: 0.2208\n",
            "epoch: 49 loss: 0.2208\n",
            "The train data accuracy is: 93.7\n",
            "The test data accuracy is: 93.71\n"
          ]
        }
      ]
    }
  ]
}