Keras_CNTK_RNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# High-level RNN Keras (CNTK) Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using CNTK backend\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import numpy as np\n",
    "os.environ['KERAS_BACKEND'] = \"cntk\"\n",
    "import keras as K\n",
    "import cntk\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Embedding, GRU, CuDNNGRU\n",
    "from common.params_lstm import *\n",
    "from common.utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OS:  linux\n",
      "Python:  3.5.2 |Anaconda custom (64-bit)| (default, Jul  2 2016, 17:53:06) \n",
      "[GCC 4.4.7 20120313 (Red Hat 4.4.7-1)]\n",
      "Keras:  2.1.1\n",
      "Numpy:  1.13.3\n",
      "CNTK:  2.2\n",
      "cntk\n"
     ]
    }
   ],
   "source": [
    "print(\"OS: \", sys.platform)\n",
    "print(\"Python: \", sys.version)\n",
    "print(\"Keras: \", K.__version__)\n",
    "print(\"Numpy: \", np.__version__)\n",
    "print(\"CNTK: \", cntk.__version__)\n",
    "print(K.backend.backend())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_symbol(CUDNN=True):\n",
    "    model = Sequential()\n",
    "    model.add(Embedding(MAXFEATURES, EMBEDSIZE, input_length=MAXLEN))\n",
    "    # Only return last output\n",
    "    if not CUDNN:\n",
    "        model.add(GRU(NUMHIDDEN, return_sequences=False, return_state=False))\n",
    "    else:\n",
    "        model.add(CuDNNGRU(NUMHIDDEN, return_sequences=False, return_state=False))\n",
    "    model.add(Dense(2, activation='softmax'))\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_model(m):\n",
    "    m.compile(\n",
    "        loss = \"categorical_crossentropy\",\n",
    "        optimizer = K.optimizers.Adam(LR, BETA_1, BETA_2, EPS),\n",
    "        metrics = ['accuracy'])\n",
    "    return m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing train set...\n",
      "Preparing test set...\n",
      "Trimming to 30000 max-features\n",
      "Padding to length 150\n",
      "(25000, 150) (25000, 150) (25000, 2) (25000, 2)\n",
      "int32 int32 int32 int32\n",
      "CPU times: user 5.3 s, sys: 278 ms, total: 5.58 s\n",
      "Wall time: 5.58 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Data into format for library\n",
    "x_train, x_test, y_train, y_test = imdb_for_library(seq_len=MAXLEN, max_features=MAXFEATURES, one_hot=True)\n",
    "print(x_train.shape, x_test.shape, y_train.shape, y_test.shape)\n",
    "print(x_train.dtype, x_test.dtype, y_train.dtype, y_test.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 137 ms, sys: 107 ms, total: 244 ms\n",
      "Wall time: 111 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Load symbol\n",
    "# CuDNN RNNs are only available with the TensorFlow backend.\n",
    "sym = create_symbol(CUDNN=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 7.83 ms, sys: 0 ns, total: 7.83 ms\n",
      "Wall time: 7.8 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# Initialise model\n",
    "model = init_model(sym)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 150, 125)          3750000   \n",
      "_________________________________________________________________\n",
      "gru_1 (GRU)                  (None, 100)               67800     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 2)                 202       \n",
      "=================================================================\n",
      "Total params: 3,818,002\n",
      "Trainable params: 3,818,002\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "25000/25000 [==============================] - 29s 1ms/step - loss: 0.4392 - acc: 0.7803\n",
      "Epoch 2/3\n",
      "25000/25000 [==============================] - 29s 1ms/step - loss: 0.2057 - acc: 0.9239\n",
      "Epoch 3/3\n",
      "25000/25000 [==============================] - 29s 1ms/step - loss: 0.1104 - acc: 0.9631\n",
      "CPU times: user 1min 23s, sys: 3.78 s, total: 1min 27s\n",
      "Wall time: 1min 26s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f33482c9550>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "# 86s\n",
    "# Train model\n",
    "model.fit(x_train,\n",
    "          y_train,\n",
    "          batch_size=BATCHSIZE,\n",
    "          epochs=EPOCHS,\n",
    "          verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 10.2 s, sys: 236 ms, total: 10.4 s\n",
      "Wall time: 10.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "y_guess = model.predict(x_test, batch_size=BATCHSIZE)\n",
    "y_guess = np.argmax(y_guess, axis=-1)\n",
    "y_truth = np.argmax(y_test, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.86092\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy: \", sum(y_guess == y_truth)/len(y_guess))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}