ex03_activation-function-selection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import library\n",
    "import numpy as np\n",
    "exec(open(\"00_readingInput.py\").read())\n",
    "\n",
    "# set the size of mini-batch and number of epochs\n",
    "batch_size = 16\n",
    "epochs = 30\n",
    "\n",
    "''' Import keras to build a DL model '''\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 1: 修改 model_sp 的 activation functions\n",
    "1. 前面兩層 Dense layers 指定使用 'relu' activation function\n",
    "2. 最後一層使用 'softmax' activation function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use other activation function\n",
    "model_sp = Sequential()\n",
    "model_sp.add(Dense(128, input_dim=200))\n",
    "# (Do!) 加入 activation function\n",
    "\n",
    "model_sp.add(Dense(256))\n",
    "# (Do!) 加入 activation function\n",
    "\n",
    "model_sp.add(Dense(5))\n",
    "# (Do!) 加入 activation function\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lr set to 0.01\n",
    "from keras.optimizers import SGD\n",
    "sgd = SGD(lr=0.01,momentum=0.0,decay=0.0,nesterov=False)\n",
    "\n",
    "model_sp.compile(loss= 'categorical_crossentropy',\n",
    "              optimizer=sgd,\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "history_sp = model_sp.fit(X_train, Y_train,\n",
    "                          batch_size=batch_size,\n",
    "                          epochs=epochs,\n",
    "                          verbose=0,\n",
    "                          shuffle=True,\n",
    "                          validation_split=0.1)\n",
    "\n",
    "loss_sp = history_sp.history.get('loss')\n",
    "acc_sp = history_sp.history.get('acc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### standard model for comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_bm = Sequential()\n",
    "model_bm.add(Dense(128, input_dim=200))\n",
    "model_bm.add(Activation('sigmoid'))\n",
    "model_bm.add(Dense(256))\n",
    "model_bm.add(Activation('sigmoid'))\n",
    "model_bm.add(Dense(5))\n",
    "model_bm.add(Activation('softmax'))\n",
    "\n",
    "from keras.optimizers import SGD\n",
    "sgd = SGD(lr=0.01,momentum=0.0,decay=0.0,nesterov=False)\n",
    "\n",
    "model_bm.compile(loss='categorical_crossentropy',\n",
    "                 optimizer=sgd,\n",
    "                 metrics=['accuracy'])\n",
    "history_bm = model_bm.fit(X_train, Y_train,\n",
    "                          batch_size=batch_size,\n",
    "                          epochs=epochs,\n",
    "                          verbose=0,\n",
    "                          shuffle=True,\n",
    "                          validation_split=0.1)\n",
    "loss_bm\t= history_bm.history.get('loss')\n",
    "acc_bm \t= history_bm.history.get('acc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.figure(0,figsize=(8,6))\n",
    "plt.subplot(121)\n",
    "plt.plot(range(len(loss_sp)),loss_sp,label='relu')\n",
    "plt.plot(range(len(loss_bm)),loss_bm,label='Sigmoid')\n",
    "plt.title('Loss')\n",
    "plt.xlabel(\"epoch\")\n",
    "plt.legend(loc='upper left')\n",
    "plt.subplot(122)\n",
    "plt.plot(range(len(acc_sp)),acc_sp,label='relu')\n",
    "plt.plot(range(len(acc_bm)),acc_bm,label='Sigmoid')\n",
    "plt.title('Accuracy')\n",
    "plt.xlabel(\"epoch\")\n",
    "plt.tight_layout()\n",
    "plt.savefig('03_activationFuncSelection.png',dpi=300,format='png')\n",
    "plt.show()\n",
    "plt.close()\n",
    "print('Result saved into 03_activationFuncSelection.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Questions\n",
    "1. Is lr = 0.01 good when using 'relu' as the activation function?\n",
    "2. How about set lr at 0.001?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}