IntroDL_Mnist_+_CIFAR.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/carranza96/DLSeminar/blob/master/IntroDL_Mnist_%2B_CIFAR.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r7v5_aIBuJc-"
      },
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow.keras as keras\n",
        "from tensorflow.keras.datasets import mnist\n",
        "from tensorflow.keras.layers import Input, Dense, Dropout, Flatten, Conv2D, MaxPool2D\n",
        "from tensorflow.keras import backend as K\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uwUCso1grsNb"
      },
      "source": [
        "# Fix random seed to allow reproducible experiments\n",
        "np.random.seed(1)\n",
        "tf.random.set_seed(1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1bW3b0IDwprh"
      },
      "source": [
        "# Load and inspect MNIST data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SUDgJmq1wWo5"
      },
      "source": [
        "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KO4lBXrYwz5v"
      },
      "source": [
        "print(\"Shape X_train: \", x_train.shape)\n",
        "print(\"Shape y_train: \", y_train.shape)\n",
        "print(\"Shape X_test: \", x_test.shape)\n",
        "print(\"Shape y_test: \", y_test.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AhXc08Sl2PDh"
      },
      "source": [
        "Values of one instance"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rT-OGTEG2Mxr",
        "collapsed": true
      },
      "source": [
        "print(\"y:\", y_train[0])\n",
        "print(\"x:\", x_train[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LKYXkjCf17mv"
      },
      "source": [
        "Visualize some examples"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_3BhFfAK1Vut",
        "collapsed": true
      },
      "source": [
        "plt.figure(figsize=(16,8))\n",
        "for i in range(10):\n",
        "    plt.subplot(1, 10, i+1)\n",
        "    plt.imshow(x_train[i])\n",
        "    plt.text(0, 0, y_train[i], color='black', bbox=dict(facecolor='white', alpha=1))\n",
        "    plt.axis('off')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QyHpaa2w2rz4"
      },
      "source": [
        "Class frequency"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vug2cKN22s4S",
        "collapsed": true
      },
      "source": [
        "from collections import Counter\n",
        "Counter(y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SCDEmRw96euB"
      },
      "source": [
        "NUM_CLASSES = 10"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x_gabl0Pz--h"
      },
      "source": [
        "# Multi-Layer Perceptron"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HRAixEkP6E4i"
      },
      "source": [
        "![texto alternativo](https://miro.medium.com/proxy/1*eloYEyFrblGHVZhU345PJw.jpeg)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HVOjdkmVKdvY"
      },
      "source": [
        "## Data preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vPe1va6o3UaV"
      },
      "source": [
        "# Convert matrices to vector (images are 28x28 pixels)\n",
        "X_train = x_train.reshape(60000, 784) # 28 * 28 = 784\n",
        "X_test = x_test.reshape(10000, 784)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4AaIdEBgyAil"
      },
      "source": [
        "# Normalize data\n",
        "# Transform from range [0., 255. ] to range [0., 1.]\n",
        "X_train = X_train.astype('float32')\n",
        "X_test = X_test.astype('float32')\n",
        "X_train /= 255\n",
        "X_test /= 255"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G-WlA48R4Kmr"
      },
      "source": [
        "# Transform y to one-hot enconding vectors\n",
        "Y_train = keras.utils.to_categorical(y_train, NUM_CLASSES)\n",
        "Y_test = keras.utils.to_categorical(y_test, NUM_CLASSES)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l9dNarKr4ep7",
        "collapsed": true
      },
      "source": [
        "# Show example of one-hot enconding vector\n",
        "print(\"Class of first example:\", y_train[0])\n",
        "print(\"One-hot enconding:\")\n",
        "print(list(range(NUM_CLASSES)))\n",
        "print(Y_train[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e6vgRAvR5aAg"
      },
      "source": [
        "NOTE: One-hot encoding not strictly necessary, it depends on the selected loss function.\n",
        "\n",
        "If categorical_cross_entropy -> one-hot enconding\n",
        "\n",
        "If sparse_categorical_cross_entropy -> integer class"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fk1e0G-eKgxg"
      },
      "source": [
        "## Create model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JbFvkq9Ax1nN"
      },
      "source": [
        "# Create model with three dense layers\n",
        "inp = Input(shape=(784,)) # Input vector shape (28x28=784)\n",
        "x = Dense(64, activation='relu', use_bias=True)(inp)\n",
        "x = Dense(128, activation='relu', use_bias=True)(x)\n",
        "x = Dense(NUM_CLASSES, activation='softmax')(x) # 10 output neurons, one associated to each digit. Softmax activation function\n",
        "\n",
        "model = keras.Model(inputs=inp, outputs=x)\n",
        "model.compile(loss='categorical_crossentropy', optimizer=keras.optimizers.SGD(learning_rate=0.01),\n",
        "              metrics=['accuracy'])\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sa_DrFWjCdnn"
      },
      "source": [
        "## Train model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uYWTthYW7TRk",
        "collapsed": true
      },
      "source": [
        "history = model.fit(X_train, Y_train,\n",
        "          batch_size=64,\n",
        "          epochs=20,\n",
        "          verbose=1,\n",
        "          validation_data=(X_test, Y_test))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cRu1VM9OCsAd",
        "collapsed": true
      },
      "source": [
        "## Visualize training stats\n",
        "\n",
        "plt.figure()\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.legend(['Training', 'Validation'])\n",
        "\n",
        "plt.figure()\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.plot(history.history['accuracy'])\n",
        "plt.plot(history.history['val_accuracy'])\n",
        "plt.legend(['Training', 'Validation'], loc='lower right')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RVu1o_7iDXqj"
      },
      "source": [
        "## Evaluate model\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eQss9b397Rns",
        "collapsed": true
      },
      "source": [
        "score = model.evaluate(X_test, Y_test, verbose=0)\n",
        "print('Test loss:', score[0])\n",
        "print('Test accuracy:', score[1])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PYqB1j85ECjH"
      },
      "source": [
        "### Visualize predictions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LCOIAGNzEEwO",
        "collapsed": true
      },
      "source": [
        "predictions = model.predict(X_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lfzTLEYGEYr4",
        "collapsed": true
      },
      "source": [
        "# Example of predictions for first image\n",
        "# Output is vector of size 10, one value for each digit. Maximum value is the predicted digit\n",
        "predictions[0]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rVKWmxmqERfZ",
        "collapsed": true
      },
      "source": [
        "from sklearn.metrics import confusion_matrix\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "\n",
        "\n",
        "df = pd.DataFrame(confusion_matrix(np.argmax(Y_test,axis=1), np.argmax(predictions,axis=1)), columns=list(range(10)))\n",
        "plt.figure(figsize=(16,8))\n",
        "sns.heatmap(df, annot=True, fmt='g', cmap=\"Blues\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rb_VEIwGGVIW",
        "collapsed": true
      },
      "source": [
        "## Visualize some wrong predictions\n",
        "integer_predictions = np.argmax(predictions,axis=1)\n",
        "integer_labels = np.argmax(Y_test,axis=1)\n",
        "wrong_predictions_indices = np.where( integer_predictions !=  integer_labels)[0]\n",
        "plt.figure(figsize=(16,8))\n",
        "for i in range(10):\n",
        "    index = wrong_predictions_indices[i]\n",
        "    plt.subplot(1, 10, i+1)\n",
        "    plt.imshow(x_test[index])\n",
        "    plt.text(0, 0, integer_predictions[index], color='black', bbox=dict(facecolor='white', alpha=1))\n",
        "    plt.axis('off')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I7X7_bKjDcuI"
      },
      "source": [
        "### MLP with Dropout\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i4cz0RJ_DiBf",
        "collapsed": true
      },
      "source": [
        "inp = Input(shape=(784,))\n",
        "x = Dense(64, activation='relu', use_bias=True)(inp)\n",
        "x = Dropout(0.2)(x)\n",
        "x = Dense(128, activation='relu', use_bias=True)(x)\n",
        "x = Dropout(0.2)(x)\n",
        "x = Dense(NUM_CLASSES, activation='softmax')(x) # 10 output neurons, one associated to each digit. Softmax activation function\n",
        "\n",
        "model = keras.Model(inputs=inp, outputs=x)\n",
        "\n",
        "model.compile(loss='categorical_crossentropy', optimizer=keras.optimizers.SGD(learning_rate=0.01),\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "history = model.fit(X_train, Y_train,\n",
        "          batch_size=64,\n",
        "          epochs=20,\n",
        "          verbose=1,\n",
        "          validation_data=(X_test, Y_test))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pO0Uk3w7FUK5"
      },
      "source": [
        "### How to improve results?\n",
        "\n",
        "*   Adjust parameters in dense layers (number of neurons, activation function)\n",
        "*   Add more dense layers\n",
        "*   Dropout\n",
        "*   Change optimizer: SGD, Adam, Adagrad, RMSProp\n",
        "*   Change architecture: CNN, LSTM\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fjqz9rm7IxTx"
      },
      "source": [
        "# Convolutional Neural Network"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MyETV28KKWl0"
      },
      "source": [
        "![texto alternativo](https://miro.medium.com/max/3744/1*SGPGG7oeSvVlV5sOSQ2iZw.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8X56XJ0mLMSq"
      },
      "source": [
        "## Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C_jw7yajI0-D"
      },
      "source": [
        "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
        "\n",
        "# Input shape now is 3-dimensional (num_examples, 28, 28, 1). We need to reshape to indicate number of channels\n",
        "X_train = x_train.reshape(x_train.shape[0], 28, 28, 1)\n",
        "X_test = x_test.reshape(x_test.shape[0], 28, 28, 1)\n",
        "input_shape = (28, 28, 1)\n",
        "\n",
        "# Normalization\n",
        "X_train = X_train.astype('float32')\n",
        "X_test = X_test.astype('float32')\n",
        "X_train /= 255\n",
        "X_test /= 255\n",
        "\n",
        "# One-hot encoding\n",
        "Y_train = keras.utils.to_categorical(y_train, 10)\n",
        "Y_test = keras.utils.to_categorical(y_test, 10)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PomOaB7ZLOCn"
      },
      "source": [
        "## Create CNN model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ME6E98q3LSp9",
        "collapsed": true
      },
      "source": [
        "# Conv2D with con 32 convolutional filters with kernel size 3x3\n",
        "inp = Input(shape=input_shape)\n",
        "x = Conv2D(32, (3, 3), padding='same', activation='relu')(inp)\n",
        "x = Flatten()(x)    # Convert from matrices to vector before Fully Connected Layers\n",
        "x = Dense(NUM_CLASSES, activation='softmax')(x)\n",
        "\n",
        "model = keras.Model(inputs=inp, outputs=x)\n",
        "\n",
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer=keras.optimizers.SGD(learning_rate=0.01),\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "# Print model architecture\n",
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UdZrD4fdqrO1"
      },
      "source": [
        "## Train CNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GGHFKTpxL0tE",
        "collapsed": true
      },
      "source": [
        "hist = model.fit(X_train, Y_train, batch_size=64,\n",
        "                 epochs=20, verbose=1,\n",
        "                 validation_data=(X_test, Y_test))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qIdwTc6ZMNSJ",
        "collapsed": true
      },
      "source": [
        "## Visualize training stats\n",
        "plt.figure()\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.legend(['Training', 'Validation'])\n",
        "\n",
        "plt.figure()\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.plot(history.history['accuracy'])\n",
        "plt.plot(history.history['val_accuracy'])\n",
        "plt.legend(['Training', 'Validation'], loc='lower right')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DfvtkW6yL9F2"
      },
      "source": [
        "## Evaluate model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RwhxJRNiL8ZX",
        "collapsed": true
      },
      "source": [
        "score = model.evaluate(X_test, Y_test, verbose=0)\n",
        "print('Test loss:', score[0])\n",
        "print('Test accuracy:', score[1])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OMr3Tm4OL-6P"
      },
      "source": [
        "## Visualize predictions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d2A4tJKvMBZQ",
        "collapsed": true
      },
      "source": [
        "predictions = model.predict(X_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "St5c0bmZMFV7",
        "collapsed": true
      },
      "source": [
        "## Visualize some wrong predictions\n",
        "integer_predictions = np.argmax(predictions,axis=1)\n",
        "integer_labels = np.argmax(Y_test,axis=1)\n",
        "wrong_predictions_indices = np.where( integer_predictions !=  integer_labels)[0]\n",
        "plt.figure(figsize=(16,8))\n",
        "for i in range(10):\n",
        "    index = wrong_predictions_indices[i]\n",
        "    plt.subplot(1, 10, i+1)\n",
        "    plt.imshow(x_test[index])\n",
        "    plt.text(0, 0, integer_predictions[index], color='black', bbox=dict(facecolor='white', alpha=1))\n",
        "    plt.axis('off')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cy2OhUUCo2PT"
      },
      "source": [
        "## Exercise: Create CNN model with the following characteristics\n",
        "\n",
        "2 convolution+pooling blocks with increasing number of filters and relu activation function\n",
        "\n",
        "Dropout after convolution\n",
        "\n",
        "Fully connected block at the end\n",
        "\n",
        "Change optimizer (search in Keras docs for available optimizers)\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B3puKrzEo2pS"
      },
      "source": [
        "# inp =\n",
        "# x =\n",
        "\n",
        "# model = keras.Model(...)\n",
        "\n",
        "\n",
        "# model.compile(...)\n",
        "\n",
        "\n",
        "\n",
        "#  hist = model.fit(X_train, Y_train, batch_size=64,\n",
        "#                  epochs=5, verbose=1,\n",
        "#                  validation_data=(X_test, Y_test))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Transfer learning with CIFAR-10 dataset"
      ],
      "metadata": {
        "id": "YA84yQE41AKk"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "(x_train, y_train), (x_test, y_test) = keras.datasets.cifar10.load_data()\n",
        "classes = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']\n"
      ],
      "metadata": {
        "id": "0KSmfKXW1EBX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Shape X_train: \", x_train.shape)\n",
        "print(\"Shape y_train: \", y_train.shape)\n",
        "print(\"Shape X_test: \", x_test.shape)\n",
        "print(\"Shape y_test: \", y_test.shape)"
      ],
      "metadata": {
        "id": "XjhxG9tx2NUT",
        "collapsed": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize=(16,16))\n",
        "for i in range(10):\n",
        "    plt.subplot(1, 10, i+1)\n",
        "    plt.imshow(x_train[i])\n",
        "    plt.text(0, 0, classes[y_train[i][0]], color='black', bbox=dict(facecolor='white', alpha=1))\n",
        "    plt.axis('off')"
      ],
      "metadata": {
        "id": "ffJv0U9j2QTY",
        "collapsed": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "input_shape=(32,32,3)\n",
        "\n",
        "x_train = x_train.astype('float32')\n",
        "x_test = x_test.astype('float32')\n",
        "# x_train /= 255\n",
        "# x_test /= 255\n",
        "\n",
        "# Custom normalization depending on base model\n",
        "x_train = keras.applications.vgg16.preprocess_input(x_train)\n",
        "x_test = keras.applications.vgg16.preprocess_input(x_test)\n",
        "\n",
        "# One-hot encoding\n",
        "y_train = keras.utils.to_categorical(y_train)\n",
        "y_test = keras.utils.to_categorical(y_test)"
      ],
      "metadata": {
        "id": "lTIF0YQa3hNV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Shape X_train: \", x_train.shape)\n",
        "print(\"Shape y_train: \", y_train.shape)\n",
        "print(\"Shape X_test: \", x_test.shape)\n",
        "print(\"Shape y_test: \", y_test.shape)"
      ],
      "metadata": {
        "id": "P0wiNko534Rt",
        "collapsed": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "[Keras applications API](https://keras.io/api/applications/)"
      ],
      "metadata": {
        "id": "73LgFm2W4AUg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "base_model = tf.keras.applications.VGG19(include_top=False,weights='imagenet',input_shape=(32,32,3),classes=y_train.shape[1])\n",
        "# Freeze the base model\n",
        "base_model.trainable = False"
      ],
      "metadata": {
        "id": "tqzCN4S93r87"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "inp = Input(shape=input_shape)\n",
        "x = base_model(inp)\n",
        "x = Flatten()(x)\n",
        "x = Dense(256,activation=('relu'))(x)\n",
        "x = Dropout(0.2)(x)  # Regularize with dropout\n",
        "outputs = Dense(10, activation='softmax')(x)\n",
        "model = keras.Model(inp, outputs)\n",
        "\n",
        "\n",
        "model = keras.Model(inputs=inp, outputs=outputs)\n",
        "\n",
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer=keras.optimizers.SGD(learning_rate=0.01),\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "# Print model architecture\n",
        "model.summary(show_trainable=True)"
      ],
      "metadata": {
        "id": "73-NjBXM6jTa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.compile(loss='categorical_crossentropy',\n",
        "              optimizer=keras.optimizers.SGD(learning_rate=0.01),\n",
        "              metrics=['accuracy'])"
      ],
      "metadata": {
        "id": "YJt3YlAS5dAS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "hist = model.fit(x_train, y_train, batch_size=64,\n",
        "                 epochs=20, verbose=1,\n",
        "                 validation_data=(x_test, y_test))"
      ],
      "metadata": {
        "id": "uFT-wmJh_ACD",
        "collapsed": true
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Exercise:\n",
        "Try to unfreeze the base model and train all layers after fine-tuning (with a low learning rate 1e-5 for example)."
      ],
      "metadata": {
        "id": "OhuViu-v-ZYU"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "g34_SuIm_FcQ"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}