Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tutorial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow Version: 1.3.0\n"
     ]
    }
   ],
   "source": [
    "# coding: utf-8\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import time\n",
    "import collections\n",
    "import os\n",
    "from bz2 import BZ2File\n",
    "from io import open\n",
    "from collections import Counter\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.layers.core import Dense\n",
    "from tensorflow.python.ops.rnn_cell_impl import _zero_state_tensors\n",
    "print('TensorFlow Version: {}'.format(tf.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice: here the tensorflow version should be 1.30\n",
    "older version may have difference in the attention wrappers, decoder wappers. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# sometimes we can use natural language toolkit to clean the text\n",
    "import nltk\n",
    "# if get an error not found some packages, use nltk.download() \n",
    "# nltk.download()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import word_tokenize, sent_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# configurations of data path\n",
    "# please put the corpus at this path\n",
    "data_path = 'data/output1.bz2'\n",
    "output_path = 'models/model1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# read the dataset\n",
    "def read_data(data_path):\n",
    "    file_reference = BZ2File(data_path, \"r\")\n",
    "    try:\n",
    "        raw_data = file_reference.read()\n",
    "    finally:\n",
    "        file_reference.close()\n",
    "    str_data = raw_data.encode(encoding='utf-8')\n",
    "    return str_data\n",
    "\n",
    "str_data = read_data(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split the dataset into conversations, the cut to X and y\n",
    "def get_Xy(str_data):\n",
    "    # Cut the conversation to X and y\n",
    "    X = []\n",
    "    y = []\n",
    "    conversations = str_data.split('\\n\\n')\n",
    "    for conversation in conversations:\n",
    "        lines = conversation.split('\\n')\n",
    "        for i in range(len(lines) - 1):\n",
    "            temp_list = sent_tokenize(lines[i])\n",
    "            if len(temp_list[-1]) <= 2:\n",
    "                if len(temp_list) > 2 and len(temp_list[-2]) > 2:\n",
    "                    X.append(sent_tokenize(lines[i])[-2])\n",
    "                else:\n",
    "                    continue\n",
    "            else:\n",
    "                X.append(temp_list[-1])\n",
    "            y.append(sent_tokenize(lines[i + 1])[0])\n",
    "    return X, y\n",
    "\n",
    "X, y = get_Xy(str_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "> Maybe new player isn't the the right term, but 30 mil for a single hack for an explorer is really good ISK right now.\n",
      "> It's also not \"new\" isk in terms of the system.\n",
      "1\n",
      "It gives exploration a method to take other people's isk, which it currently lacks.\n",
      "> exploration makes money soley off of selling items, how does it generate new isk at all?\n",
      "2\n",
      "> exploration makes money soley off of selling items, how does it generate new isk at all?\n",
      "> I thought you were wrong for a second before realizing only things like missions and rat bounties actually generate new ISK.\n",
      "3\n",
      "> what\n",
      "> relevant flair LOL!!\n",
      "4\n",
      "> How?\n",
      "> ahri u kidding meimsosorry hahaha!\n",
      "5\n",
      "Otherwise nice video.\n",
      "> Windowed.\n",
      "6\n",
      "I should record on fullscreen shouldn't I?\n",
      "> Yep, and record it at a higher resolution.\n",
      "7\n",
      "For your 2nd video, you're improving, probably change the text format, and try some new concepts, but I believe in you.\n",
      "> yes the text font is not that great\n",
      "8\n",
      "is trade freedom for protection.\n",
      "> You don't have to be some kind of radical to recognize that the police in America in general and in new York in particular are way out of hand.\n",
      "9\n",
      "> You don't have to be some kind of radical to recognize that the police in America in general and in new York in particular are way out of hand.\n",
      "> America- I disagree from what I've seen New York- I can't speak for.\n",
      "10\n",
      "> America- I disagree from what I've seen New York- I can't speak for.\n",
      "> ok buddy, you do that.\n",
      "11\n",
      "> GOD DAMMIT WHY WON'T THEY GET THESE SERIES FOR AUSTRALIA!\n",
      "> Anime lab might be streaming it, they usually pick up a few each season\n",
      "12\n",
      "> Anime lab might be streaming it, they usually pick up a few each season\n",
      "> if only.\n",
      "13\n",
      "There's still a lot that they don't get.\n",
      "> Well a few = 3, so that's not too inaccurate, haha\n",
      "14\n",
      "> Well a few = 3, so that's not too inaccurate, haha\n",
      "> well, so far, the only new show for this season that they've added is Kamisama Kiss\n",
      "15\n",
      "> Flashlight.\n",
      "> Fleshlight\n",
      "16\n",
      "> Fleshlight\n",
      "> That would at least be kinda weird.\n",
      "17\n",
      "* *Ehm.. Dude, this might be a bit weird.\n",
      "> BRO JOB BRO JOB CHOO CHOO\n",
      "18\n",
      "> It's like saying \"wolf\" instead of \"black lab\".\n",
      "> But it's technically correct and that's the best kind!\n",
      "19\n",
      "> But it's technically correct and that's the best kind!\n",
      "> Something something unidan\n",
      "20\n",
      "> Something something unidan\n",
      "> Not sure if missed the Futurama reference or chose to ignore it.\n",
      "21\n",
      "Thanks\n",
      "> Nordstrom, it's on clearance for $240.\n",
      "22\n",
      "I'm returning mine though\n",
      "> They look killer.\n",
      "23\n",
      "I'm trying to figure out my next pair of brown boots.\n",
      "> I just bought the regular 1000 mile for $100 cheaper.\n",
      "24\n",
      "> I just bought the regular 1000 mile for $100 cheaper.\n",
      "> Good call.\n",
      "25\n",
      "They look slightly different online.\n",
      "> Nordstrom.\n",
      "26\n",
      "Yes, I think so.\n",
      "> Thanks for answering all my questions\n",
      "27\n",
      "> Thanks for answering all my questions\n",
      "> No problem\n",
      "28\n",
      "> Do you have Home insurance?\n",
      "> yes i do.\n",
      "29\n",
      "> yes i do.\n",
      "> Great fucking call.\n"
     ]
    }
   ],
   "source": [
    "for i in range(30):\n",
    "    print i\n",
    "    print X[i]\n",
    "    print y[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # split the dataset into conversations, the cut to X and y\n",
    "# def get_Xy(str_data):\n",
    "#     # Cut the conversation to X and y\n",
    "#     X = []\n",
    "#     y = []\n",
    "#     ck = 0\n",
    "#     conversations = str_data.split('\\n\\n')\n",
    "#     for conversation in conversations:\n",
    "#         lines = conversation.split('\\n')\n",
    "#         for i in range(len(lines) - 1):\n",
    "#             if len(sent_tokenize(lines[i])[-1]) <= 2:\n",
    "#                 print lines[i]\n",
    "#                 print sent_tokenize(lines[i])\n",
    "#                 print \"--------------------\"\n",
    "#                 ck = ck +1\n",
    "#                 if ck > 101:\n",
    "#                     return X, y\n",
    "#             X.append(sent_tokenize(lines[i])[-1])\n",
    "#             y.append(sent_tokenize(lines[i + 1])[0])\n",
    "#     return X, y\n",
    "\n",
    "# X, y = get_Xy(str_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# use the contractions dict from link:\n",
    "# https://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&uact=8&ved=0ahUKEwiw-_SY2snWAhUjllQKHaKZAkoQFggoMAA&url=https%3A%2F%2Fstackoverflow.com%2Fquestions%2F19790188%2Fexpanding-english-language-contractions-in-python&usg=AFQjCNEsCLtI_2UFgUEImnUWn2GsPb_S2Q\n",
    "\n",
    "contractions = { \n",
    "    \"ain't\": \"am not\",\n",
    "    \"aren't\": \"are not\",\n",
    "    \"can't\": \"cannot\",\n",
    "    \"can't've\": \"cannot have\",\n",
    "    \"'cause\": \"because\",\n",
    "    \"could've\": \"could have\",\n",
    "    \"couldn't\": \"could not\",\n",
    "    \"couldn't've\": \"could not have\",\n",
    "    \"didn't\": \"did not\",\n",
    "    \"doesn't\": \"does not\",\n",
    "    \"don't\": \"do not\",\n",
    "    \"hadn't\": \"had not\",\n",
    "    \"hadn't've\": \"had not have\",\n",
    "    \"hasn't\": \"has not\",\n",
    "    \"haven't\": \"have not\",\n",
    "    \"he'd\": \"he would\",\n",
    "    \"he'd've\": \"he would have\",\n",
    "    \"he'll\": \"he will\",\n",
    "    \"he's\": \"he is\",\n",
    "    \"how'd\": \"how did\",\n",
    "    \"how'll\": \"how will\",\n",
    "    \"how's\": \"how is\",\n",
    "    \"i'd\": \"i would\",\n",
    "    \"i'll\": \"i will\",\n",
    "    \"i'm\": \"i am\",\n",
    "    \"i've\": \"i have\",\n",
    "    \"isn't\": \"is not\",\n",
    "    \"it'd\": \"it would\",\n",
    "    \"it'll\": \"it will\",\n",
    "    \"it's\": \"it is\",\n",
    "    \"let's\": \"let us\",\n",
    "    \"ma'am\": \"madam\",\n",
    "    \"mayn't\": \"may not\",\n",
    "    \"might've\": \"might have\",\n",
    "    \"mightn't\": \"might not\",\n",
    "    \"must've\": \"must have\",\n",
    "    \"mustn't\": \"must not\",\n",
    "    \"needn't\": \"need not\",\n",
    "    \"oughtn't\": \"ought not\",\n",
    "    \"shan't\": \"shall not\",\n",
    "    \"sha'n't\": \"shall not\",\n",
    "    \"she'd\": \"she would\",\n",
    "    \"she'll\": \"she will\",\n",
    "    \"she's\": \"she is\",\n",
    "    \"should've\": \"should have\",\n",
    "    \"shouldn't\": \"should not\",\n",
    "    \"that'd\": \"that would\",\n",
    "    \"that's\": \"that is\",\n",
    "    \"there'd\": \"there had\",\n",
    "    \"there's\": \"there is\",\n",
    "    \"they'd\": \"they would\",\n",
    "    \"they'll\": \"they will\",\n",
    "    \"they're\": \"they are\",\n",
    "    \"they've\": \"they have\",\n",
    "    \"wasn't\": \"was not\",\n",
    "    \"we'd\": \"we would\",\n",
    "    \"we'll\": \"we will\",\n",
    "    \"we're\": \"we are\",\n",
    "    \"we've\": \"we have\",\n",
    "    \"weren't\": \"were not\",\n",
    "    \"what'll\": \"what will\",\n",
    "    \"what're\": \"what are\",\n",
    "    \"what's\": \"what is\",\n",
    "    \"what've\": \"what have\",\n",
    "    \"where'd\": \"where did\",\n",
    "    \"where's\": \"where is\",\n",
    "    \"who'll\": \"who will\",\n",
    "    \"who's\": \"who is\",\n",
    "    \"won't\": \"will not\",\n",
    "    \"wouldn't\": \"would not\",\n",
    "    \"you'd\": \"you would\",\n",
    "    \"you'll\": \"you will\",\n",
    "    \"you're\": \"you are\",\n",
    "    \"aint\": \"am not\",   # here I add some more contractions\n",
    "    \"arent\": \"are not\",\n",
    "    \"cant\": \"cannot\",\n",
    "    \"couldve\": \"could have\",\n",
    "    \"couldnt\": \"could not\",\n",
    "    \"didnt\": \"did not\",\n",
    "    \"doesnt\": \"does not\",\n",
    "    \"dont\": \"do not\",\n",
    "    \"hadnt\": \"had not\",\n",
    "    \"hasnt\": \"has not\",\n",
    "    \"havent\": \"have not\",\n",
    "    \"isnt\": \"is not\",\n",
    "    \"wont\": \"will not\",\n",
    "    \"wouldnt\": \"would not\",\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def clean_text(text):\n",
    "    # clean the conractions, clean starting symbols '>' and other symbols\n",
    "    text = text.lower()\n",
    "    \n",
    "    # replace contractions\n",
    "    text = word_tokenize(text)\n",
    "    new_text = []\n",
    "    for word in text:\n",
    "        if word == '>':\n",
    "            continue\n",
    "#         if word in contractions:\n",
    "#             word = contractions[word]\n",
    "#             new_text += word.split(' ')\n",
    "#             continue\n",
    "        new_text.append(word)\n",
    "    \n",
    "    text = \" \".join(new_text)\n",
    "    \n",
    "    # everything in the brackets\n",
    "    text = re.sub(r'[_\"\\-%()|+&=*%#$@\\[\\]/]', ' ', text)\n",
    "    # <br/> \n",
    "    # text = re.sub(r'<br />', ' ', text)\n",
    "    # text = re.sub(r'\\'', ' ', text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_clean = map(lambda line: clean_text(line), y)\n",
    "X_clean = map(lambda line: clean_text(line), X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('vocab size: ', 344199)\n"
     ]
    }
   ],
   "source": [
    "word_counter = Counter()\n",
    "def word_count(word_counter, text):\n",
    "    for sentence in text:\n",
    "        for word in sentence.split():\n",
    "            word_counter[word] += 1\n",
    "\n",
    "word_count(word_counter, X_clean)\n",
    "word_count(word_counter, y_clean)\n",
    "word_counts = dict(word_counter)\n",
    "del word_counter\n",
    "\n",
    "print(\"vocab size: \", len(word_counts))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The 'numberbatch-en-17.06.txt' file is a pretrained word embeddings.\n",
    "You can find it here:  \n",
    "\n",
    "The word to index (one-hot encoding) make an extremely sparse word space, which is unacceptable if the number of word is large.\n",
    "Word to vector (word2vec) use the words' relation with its context to represents a word. Some methods are Glove, skipgram and cbow.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from gensim.models.word2vec import Word2Vec\n",
    "# from gensim.models import KeyedVectors\n",
    "# # model = KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin', binary=True)\n",
    "# # model.wv.save_word2vec_format('googlenews.txt')\n",
    "# model = Word2Vec(X_clean, size=128, window=5, min_count=30, workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Size of Word Embeddings: ', 417195)\n"
     ]
    }
   ],
   "source": [
    "embeddings_index = {}\n",
    "with open('numberbatch-en-17.06.txt') as f:\n",
    "    for line in f:\n",
    "        # print line\n",
    "        values = line.split(' ')\n",
    "        word = values[0]\n",
    "        embedding = np.asarray(values[1:], dtype='float32')\n",
    "        embeddings_index[word] = embedding\n",
    "        \n",
    "print(\"Size of Word Embeddings: \", len(embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('number of missing words: ', 251408)\n"
     ]
    }
   ],
   "source": [
    "missing_words = 0\n",
    "threshold = 30\n",
    "# add the words to this string and print\n",
    "missing_words_list = []\n",
    "\n",
    "for word, count in word_counts.items():\n",
    "    if count < threshold:\n",
    "        if word not in embeddings_index:\n",
    "            missing_words += 1\n",
    "            missing_words_list.append(word)\n",
    "\n",
    "missing_ratio = round(missing_words/len(word_counts), 4)*100\n",
    "print(\"number of missing words: \", missing_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "set(['eeefffcd', 'sowell', 'fjate', 'paizo', 'fixxxer', 'moodet', 'canex', 'fagathians', 'xpole', 'icici', '..maybe..maybe', 'fariko', 'thatsthejokejpg', 'iguau', 'reacharounds', 'brrrrrrrrrrrrrrrrrt', 'canem', 'spingono', 'nergumne', '2048x858', 'kaushal', 'jrpg', 'weell', 'vlaars', '18f', 'iluminati', 'ysolda', 'lonelylosercreep', 'berufsrisiko', \"ohheyyou'rehot\"])\n"
     ]
    }
   ],
   "source": [
    "# to see what words not covered by our word embeddings\n",
    "print set(missing_words_list[:30])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### word to int\n",
    "word_to_int = {}\n",
    "\n",
    "# 8 special texts at beginning\n",
    "special_codes = [\"<UNK>\", \"<PAD>\", \"<EOS>\", \"<GO>\", \".\", \",\", \"!\", \"?\"]\n",
    "for code in special_codes:\n",
    "    word_to_int[code] = len(word_to_int)\n",
    "    \n",
    "value = len(word_to_int)\n",
    "for word, count in word_counts.items():\n",
    "    if count >= threshold and word in embeddings_index:\n",
    "        word_to_int[word] = value\n",
    "        value += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Number of word we will use', 26259)\n",
      "Percentage of words we will use: 7.63%\n"
     ]
    }
   ],
   "source": [
    "# add special tokens to our word to int dictionary\n",
    "# <GO> the first token which is fed to the decodeer along with the though vector in ordder to start generating okens of the answer\n",
    "# <EOS> \"end of sentence\" as soon as decoder generates this token we consider the answer to be complete\n",
    "# <UNK> \"unknown token\" is used to replace rare word which has too low frequency or does not fit to the embedding\n",
    "# <PAD> GPU or CPU processes the training data in batches and all the sequences in the batch should have the same length\n",
    "#       If the max lenth\n",
    "\n",
    "# reverse keys and values\n",
    "int_to_word = {}\n",
    "for word, value in word_to_int.items():\n",
    "    int_to_word[value] = word\n",
    "    \n",
    "usage_ratio = round(1.0 * len(word_to_int) / len(word_counts), 4)\n",
    "print(\"Number of word we will use\", len(word_to_int))\n",
    "print(\"Percentage of words we will use: {}%\".format(usage_ratio * 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('the total number of words in word_embedding_matrix:', 26259)\n"
     ]
    }
   ],
   "source": [
    "# # ### Select word embeddings which only appears in our samples to build our own word embedding\n",
    "# embedding_dim = 300\n",
    "# n_words = len(word_to_int)\n",
    "\n",
    "# count = 0\n",
    "# word_embedding_matrix = np.zeros((n_words, embedding_dim), dtype=np.float32)\n",
    "# for word, i in word_to_int.items():\n",
    "#     if word in embeddings_index: \n",
    "#         word_embedding_matrix[i] = embeddings_index[word]\n",
    "#     elif word in special_codes:\n",
    "#         new_embedding = np.array(np.random.uniform(-1.0, 1.0, embedding_dim))\n",
    "#         word_embedding_matrix[i] = new_embedding\n",
    "#         embeddings_index[word] = new_embedding\n",
    "#     #     else:\n",
    "# #         # if not in the embedding matrix, initialize this vector by random number\n",
    "# #         new_embedding = np.array(np.random.uniform(-1.0, 1.0, embedding_dim))\n",
    "# #         embeddings_index[word] = new_embedding\n",
    "# #         word_embedding_matrix[i] = new_embedding\n",
    "\n",
    "# print(\"the total number of words in word_embedding_matrix:\", len(word_embedding_matrix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('the total number of words in word_embedding_matrix:', 26259)\n"
     ]
    }
   ],
   "source": [
    "# ### Select word embeddings which only appears in our samples to build our own word embedding\n",
    "embedding_dim = 300 + len(special_codes)\n",
    "n_words = len(word_to_int)\n",
    "\n",
    "count = 0\n",
    "word_embedding_matrix = np.zeros((n_words, embedding_dim), dtype=np.float32)\n",
    "for word, i in word_to_int.items():\n",
    "    if word in embeddings_index: \n",
    "        temp = np.zeros((len(special_codes)))\n",
    "        word_embedding_matrix[i] = np.concatenate([embeddings_index[word], temp], axis=0)\n",
    "    elif word in special_codes:\n",
    "        temp = np.zeros((embedding_dim))\n",
    "        temp[embedding_dim - 1 - special_codes.index(word)] = 1\n",
    "        word_embedding_matrix[i] = temp\n",
    "    #     else:\n",
    "#         # if not in the embedding matrix, initialize this vector by random number\n",
    "#         new_embedding = np.array(np.random.uniform(-1.0, 1.0, embedding_dim))\n",
    "#         embeddings_index[word] = new_embedding\n",
    "#         word_embedding_matrix[i] = new_embedding\n",
    "\n",
    "print(\"the total number of words in word_embedding_matrix:\", len(word_embedding_matrix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Total number of words:', 44210395)\n",
      "('Total number of UNKs:', 3605384)\n",
      "Percent of words that are UNK: 8.16%\n"
     ]
    }
   ],
   "source": [
    "def convert_to_ints(text, n_words, n_unk, eos=False):\n",
    "    ints = []\n",
    "    for sentence in text:\n",
    "        sentence_ints = []\n",
    "        for word in sentence.split():\n",
    "            n_words += 1\n",
    "            if word in word_to_int and word_counts[word] > threshold:\n",
    "                sentence_ints.append(word_to_int[word])\n",
    "            else:\n",
    "                sentence_ints.append(word_to_int[\"<UNK>\"])\n",
    "                n_unk += 1\n",
    "        if eos:\n",
    "            sentence_ints.append(word_to_int[\"<EOS>\"])\n",
    "        ints.append(sentence_ints)\n",
    "    return ints, n_words, n_unk\n",
    "\n",
    "n_words = 0\n",
    "n_unk = 0\n",
    "\n",
    "int_y, n_words, n_unk = convert_to_ints(y_clean, n_words, n_unk)\n",
    "int_X, n_words, n_unk = convert_to_ints(X_clean, n_words, n_unk, eos=True)\n",
    "\n",
    "unk_percent = round(1.0 * n_unk / n_words, 4) * 100\n",
    "\n",
    "print(\"Total number of words:\", n_words)\n",
    "print(\"Total number of UNKs:\", n_unk)\n",
    "print(\"Percent of words that are UNK: {}%\".format(unk_percent))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sentences_lengths(text):\n",
    "    lengths = []\n",
    "    for sentence in text:\n",
    "        lengths.append(len(sentence))\n",
    "    return pd.DataFrame(lengths, columns=['counts'])\n",
    "\n",
    "lengths_y = get_sentences_lengths(int_y)\n",
    "lengths_X = get_sentences_lengths(int_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAEKCAYAAADEovgeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHDZJREFUeJzt3X+cVfWd3/HXW9AVNQbU0VLAHd1MNcSuCBMldTebxAQH\nzQZ3VxttWmYtu7SWNKbZ7AbTdEl0fRQfTWNCY9gQJYJNVDQxUn8EJ2iSdovKoAZUtMwqkVlYGQUR\nY1YX8+kf53vNdbxz5844X47ceT8fj/u453zO95zv93h0Pn7P+d7vUURgZmaW00FlN8DMzJqfk42Z\nmWXnZGNmZtk52ZiZWXZONmZmlp2TjZmZZedkY2Zm2TnZmJlZdk42ZmaW3diyG/B2ccwxx0Rra2vZ\nzTAzO6Bs2LDhuYhoGayck03S2tpKd3d32c0wMzugSPp5I+V8G83MzLJzsjEzs+ycbMzMLDsnGzMz\ny87JxszMsnOyMTOz7JxszMwsOycbMzPLzsnGzMyy8wwCJWtdeOew9926+NwRbImZWT7u2ZiZWXZO\nNmZmll22ZCPpJEmPVH1elPRpSUdJ6pK0JX1PSOUlaYmkHkkbJU2vOlZnKr9FUmdVfIakTWmfJZKU\n4jXrMDOzcmRLNhHxZERMi4hpwAzgZeA2YCGwNiLagLVpHWA20JY+84GlUCQOYBFwBnA6sKgqeSxN\nZSv7daT4QHWYmVkJ9tdttLOAv42InwNzgBUpvgI4Ly3PAVZG4X5gvKSJwNlAV0TsiojdQBfQkbYd\nGRHrIiKAlf2OVasOMzMrwf5KNhcCN6bl4yJiB0D6PjbFJwHbqvbpTbF68d4a8Xp1vIGk+ZK6JXX3\n9fUN89TMzGww2ZONpEOAjwG3DFa0RiyGEW9YRCyLiPaIaG9pGfRFc2ZmNkz7o2czG3goIp5N68+m\nW2Ck750p3gtMqdpvMrB9kPjkGvF6dZiZWQn2R7K5iF/fQgNYDVRGlHUCt1fF56ZRaTOBPekW2Bpg\nlqQJaWDALGBN2rZX0sw0Cm1uv2PVqsPMzEqQdQYBSYcBHwH+XVV4MbBK0jzgGeCCFL8LOAfooRi5\ndjFAROySdAWwPpW7PCJ2peVLgOuBccDd6VOvDjMzK0HWZBMRLwNH94s9TzE6rX/ZABYMcJzlwPIa\n8W7glBrxmnWYmVk5PIOAmZll52RjZmbZOdmYmVl2TjZmZpadk42ZmWXnZGNmZtk52ZiZWXZONmZm\nlp2TjZmZZedkY2Zm2TnZmJlZdk42ZmaWnZONmZll52RjZmbZOdmYmVl2TjZmZpadk42ZmWXnZGNm\nZtk52ZiZWXZZk42k8ZJulfSEpM2S3ifpKEldkrak7wmprCQtkdQjaaOk6VXH6Uzlt0jqrIrPkLQp\n7bNEklK8Zh1mZlaO3D2brwE/jIiTgVOBzcBCYG1EtAFr0zrAbKAtfeYDS6FIHMAi4AzgdGBRVfJY\nmspW9utI8YHqMDOzEmRLNpKOBN4PXAcQEa9GxAvAHGBFKrYCOC8tzwFWRuF+YLykicDZQFdE7IqI\n3UAX0JG2HRkR6yIigJX9jlWrDjMzK0HOns2JQB/wbUkPS7pW0uHAcRGxAyB9H5vKTwK2Ve3fm2L1\n4r014tSpw8zMSpAz2YwFpgNLI+I04BfUv52lGrEYRrxhkuZL6pbU3dfXN5RdzcxsCHImm16gNyIe\nSOu3UiSfZ9MtMNL3zqryU6r2nwxsHyQ+uUacOnW8QUQsi4j2iGhvaWkZ1kmamdngsiWbiPh7YJuk\nk1LoLOBxYDVQGVHWCdyellcDc9OotJnAnnQLbA0wS9KENDBgFrAmbdsraWYahTa337Fq1WFmZiUY\nm/n4/xH4jqRDgKeAiykS3CpJ84BngAtS2buAc4Ae4OVUlojYJekKYH0qd3lE7ErLlwDXA+OAu9MH\nYPEAdZiZWQmyJpuIeARor7HprBplA1gwwHGWA8trxLuBU2rEn69Vh5mZlcMzCJiZWXZONmZmlp2T\njZmZZedkY2Zm2TnZmJlZdk42ZmaWnZONmZll52RjZmbZOdmYmVl2TjZmZpadk42ZmWXnZGNmZtk5\n2ZiZWXZONmZmlp2TjZmZZedkY2Zm2TnZmJlZdk42ZmaWnZONmZll52RjZmbZZU02krZK2iTpEUnd\nKXaUpC5JW9L3hBSXpCWSeiRtlDS96jidqfwWSZ1V8Rnp+D1pX9Wrw8zMyrE/ejYfjIhpEdGe1hcC\nayOiDVib1gFmA23pMx9YCkXiABYBZwCnA4uqksfSVLayX8cgdZiZWQnKuI02B1iRllcA51XFV0bh\nfmC8pInA2UBXROyKiN1AF9CRth0ZEesiIoCV/Y5Vqw4zMytB7mQTwD2SNkian2LHRcQOgPR9bIpP\nArZV7dubYvXivTXi9eowM7MSjM18/DMjYrukY4EuSU/UKasasRhGvGEpAc4HOP7444eyq5mZDUHW\nnk1EbE/fO4HbKJ65PJtugZG+d6bivcCUqt0nA9sHiU+uEadOHf3btywi2iOivaWlZbinaWZmg8iW\nbCQdLukdlWVgFvAosBqojCjrBG5Py6uBuWlU2kxgT7oFtgaYJWlCGhgwC1iTtu2VNDONQpvb71i1\n6jAzsxLkvI12HHBbGo08FvhuRPxQ0npglaR5wDPABan8XcA5QA/wMnAxQETsknQFsD6VuzwidqXl\nS4DrgXHA3ekDsHiAOppK68I739L+WxefO0ItMTOrL1uyiYingFNrxJ8HzqoRD2DBAMdaDiyvEe8G\nTmm0DjMzK4dnEDAzs+ycbMzMLDsnGzMzy87JxszMsnOyMTOz7BpKNpLeNOLLzMysUY32bP5a0oOS\n/oOk8VlbZGZmTaehZBMRvwN8gmLamG5J35X0kawtMzOzptHwM5uI2AJ8Afgc8HvAEklPSPrDXI0z\nM7Pm0Ogzm9+WdDWwGfgQ8PsR8e60fHXG9pmZWRNodLqarwPfAj4fEb+sBNPrA76QpWVmZtY0Gk02\n5wC/jIjXACQdBBwaES9HxA3ZWmdmZk2h0Wc2P6KYWbnisBQzMzMbVKPJ5tCIeKmykpYPy9MkMzNr\nNo0mm19Iml5ZkTQD+GWd8mZmZq9r9JnNp4FbJFVeuzwR+HieJpmZWbNpKNlExHpJJwMnAQKeiIh/\nzNoyMzNrGkN5U+d7gda0z2mSiIiVWVplZmZNpaFkI+kG4LeAR4DXUjgAJxszMxtUoz2bdmBqRETO\nxpiZWXNqdDTao8A/GU4FksZIeljSHWn9BEkPSNoi6WZJh6T4b6T1nrS9teoYl6X4k5LOrop3pFiP\npIVV8Zp1mJlZORpNNscAj0taI2l15dPgvpdSzKlWcRVwdUS0AbuBeSk+D9gdEe+imG/tKgBJU4EL\ngfcAHcA3UgIbA1wDzAamAhelsvXqMDOzEjR6G+2Lwzm4pMnAucCVwGckiWLyzn+ViqxIx14KzKmq\n51bg66n8HOCmiHgFeFpSD3B6KtcTEU+lum4C5kjaXKcOMzMrQaPvs/kJsBU4OC2vBx5qYNevAn8B\n/CqtHw28EBH70novMCktTwK2pfr2AXtS+dfj/fYZKF6vjjeQNF9St6Tuvr6+Bk7HzMyGo9FXDPwp\nRW/jmyk0CfjBIPt8FNgZERuqwzWKxiDbRir+5mDEsohoj4j2lpaWWkXMzGwENHobbQHFrasHoHiR\nmqRjB9nnTOBjks4BDgWOpOjpjJc0NvU8JgOVWQl6Kd4E2itpLPBOYFdVvKJ6n1rx5+rUYWZmJWh0\ngMArEfFqZSUlg7rDoCPisoiYHBGtFA/4742ITwD3AeenYp3A7Wl5dVonbb83DbVeDVyYRqudALQB\nD1LcymtLI88OSXWsTvsMVIeZmZWg0WTzE0mfB8ZJ+ghwC/C/hlnn5ygGC/RQPF+5LsWvA45O8c8A\nCwEi4jFgFfA48ENgQUS8lnotnwTWUIx2W5XK1qvDzMxKoEZ+p5leljYPmEXxTGQNcG0z/cizvb09\nuru793u9rQvv3O91VmxdfG5pdZtZc5C0ISLaByvX6EScv6J4LfS33mrDzMxs9Gl0brSnqfGMJiJO\nHPEWmZlZ0xnK3GgVhwIXAEeNfHPMzKwZNfqjzuerPn8XEV+l+JW+mZnZoBq9jTa9avUgip7OO7K0\nyMzMmk6jt9H+e9XyPoqpa/7liLfGzMyaUqOj0T6YuyEHsjKHL5uZHQgavY32mXrbI+IrI9McMzNr\nRkMZjfZeiqljAH4f+ClvnHXZzMyspkaTzTHA9IjYCyDpi8AtEfEnuRpmZmbNo9G50Y4HXq1afxVo\nHfHWmJlZU2q0Z3MD8KCk2yhmEvgDYGW2VpmZWVNpdDTalZLuBn43hS6OiIfzNcvMzJpJo7fRAA4D\nXoyIr1G84OyETG0yM7Mm0+hroRdRvCPmshQ6GPifuRplZmbNpdGezR8AHwN+ARAR2/F0NWZm1qBG\nk82r6UVpASDp8HxNMjOzZtNoslkl6ZvAeEl/CvwIv0jNzMwa1OhotC9L+gjwInAS8JcR0ZW1ZWZm\n1jQG7dlIGiPpRxHRFRF/HhGfbSTRSDpU0oOSfibpMUlfSvETJD0gaYukmyUdkuK/kdZ70vbWqmNd\nluJPSjq7Kt6RYj2SFlbFa9ZhZmblGDTZRMRrwMuS3jnEY78CfCgiTgWmAR2SZgJXAVdHRBuwG5iX\nys8DdkfEu4CrUzkkTQUuBN4DdADfSAlwDHANMBuYClyUylKnDjMzK0Gjz2z+Adgk6TpJSyqfejtE\n4aW0enD6BMUbPm9N8RXAeWl5TlonbT9LklL8poh4JSKeBnqA09OnJyKeiohXgZuAOWmfgeowM7MS\nNDpdzZ3pMySp97EBeBdFL+RvgRciYl8q0gtMSsuTSLNIR8Q+SXuAo1P8/qrDVu+zrV/8jLTPQHWY\nmVkJ6iYbScdHxDMRsaJeuYGkW3DTJI0HbgPeXatYpboBtg0Ur9Urq1f+TSTNB+YDHH/88bWKmJnZ\nCBjsNtoPKguSvjfcSiLiBeDHwEyK4dOVJDcZ2J6We4Epqa6xwDuBXdXxfvsMFH+uTh3927UsItoj\nor2lpWW4p2dmZoMYLNlU9xJOHMqBJbWkHg2SxgEfBjYD9wHnp2KdwO1peXVaJ22/N/2QdDVwYRqt\ndgLQBjwIrAfa0sizQygGEaxO+wxUh5mZlWCwZzYxwHIjJgIr0nObg4BVEXGHpMeBmyT9FfAwcF0q\nfx1wg6Qeih7NhQAR8ZikVcDjwD5gQbo9h6RPAmuAMcDyiHgsHetzA9RhZmYlGCzZnCrpRYoezri0\nTFqPiDhyoB0jYiNwWo34UxQjyfrH/wG4YIBjXQlcWSN+F3BXo3WYmVk56iabiBizvxpiZmbNayjv\nszEzMxuWRn9nY02odeGQfzr1uq2Lzx3BlphZs3PPxszMsnOyMTOz7JxszMwsOycbMzPLzsnGzMyy\nc7IxM7PsnGzMzCw7JxszM8vOycbMzLJzsjEzs+ycbMzMLDsnGzMzy87JxszMsnOyMTOz7JxszMws\nOycbMzPLzsnGzMyyy5ZsJE2RdJ+kzZIek3Rpih8lqUvSlvQ9IcUlaYmkHkkbJU2vOlZnKr9FUmdV\nfIakTWmfJZJUrw4zMytHzp7NPuDPIuLdwExggaSpwEJgbUS0AWvTOsBsoC195gNLoUgcwCLgDOB0\nYFFV8liaylb260jxgeowM7MSZEs2EbEjIh5Ky3uBzcAkYA6wIhVbAZyXlucAK6NwPzBe0kTgbKAr\nInZFxG6gC+hI246MiHUREcDKfseqVYeZmZVgvzyzkdQKnAY8ABwXETugSEjAsanYJGBb1W69KVYv\n3lsjTp06zMysBNmTjaQjgO8Bn46IF+sVrRGLYcSH0rb5kroldff19Q1lVzMzG4KsyUbSwRSJ5jsR\n8f0UfjbdAiN970zxXmBK1e6Tge2DxCfXiNer4w0iYllEtEdEe0tLy/BO0szMBpVzNJqA64DNEfGV\nqk2rgcqIsk7g9qr43DQqbSawJ90CWwPMkjQhDQyYBaxJ2/ZKmpnqmtvvWLXqMDOzEozNeOwzgX8D\nbJL0SIp9HlgMrJI0D3gGuCBtuws4B+gBXgYuBoiIXZKuANancpdHxK60fAlwPTAOuDt9qFOHmZmV\nIFuyiYj/Q+3nKgBn1SgfwIIBjrUcWF4j3g2cUiP+fK06zMysHJ5BwMzMsnOyMTOz7JxszMwsOycb\nMzPLzsnGzMyyc7IxM7PsnGzMzCw7JxszM8vOycbMzLJzsjEzs+ycbMzMLDsnGzMzyy7nrM/WxFoX\n3jnsfbcuPncEW2JmBwL3bMzMLDsnGzMzy87JxszMsnOyMTOz7JxszMwsOycbMzPLzsnGzMyyy5Zs\nJC2XtFPSo1WxoyR1SdqSviekuCQtkdQjaaOk6VX7dKbyWyR1VsVnSNqU9lkiSfXqMDOz8uTs2VwP\ndPSLLQTWRkQbsDatA8wG2tJnPrAUisQBLALOAE4HFlUlj6WpbGW/jkHqMDOzkmRLNhHxU2BXv/Ac\nYEVaXgGcVxVfGYX7gfGSJgJnA10RsSsidgNdQEfadmRErIuIAFb2O1atOszMrCT7+5nNcRGxAyB9\nH5vik4BtVeV6U6xevLdGvF4dbyJpvqRuSd19fX3DPikzM6vv7TJAQDViMYz4kETEsohoj4j2lpaW\noe5uZmYN2t/J5tl0C4z0vTPFe4EpVeUmA9sHiU+uEa9Xh5mZlWR/J5vVQGVEWSdwe1V8bhqVNhPY\nk26BrQFmSZqQBgbMAtakbXslzUyj0Ob2O1atOszMrCTZXjEg6UbgA8AxknopRpUtBlZJmgc8A1yQ\nit8FnAP0AC8DFwNExC5JVwDrU7nLI6Iy6OASihFv44C704c6dZiZWUmyJZuIuGiATWfVKBvAggGO\nsxxYXiPeDZxSI/58rTrMzKw8b5cBAmZm1sScbMzMLDsnGzMzyy7bMxuzgbQuvHPY+25dfO4ItsTM\n9hf3bMzMLDsnGzMzy87JxszMsnOyMTOz7JxszMwsOycbMzPLzsnGzMyyc7IxM7PsnGzMzCw7Jxsz\nM8vO09XYAeWtTHUDnu7GrCzu2ZiZWXZONmZmlp2TjZmZZedkY2Zm2TXtAAFJHcDXgDHAtRGxuOQm\n2duA36VjVo6m7NlIGgNcA8wGpgIXSZpabqvMzEavZu3ZnA70RMRTAJJuAuYAj5faKjuguVdkNnxN\n2bMBJgHbqtZ7U8zMzErQrD0b1YjFmwpJ84H5afUlSU8OoY5jgOeG0bYDmc95mHTVCLRk/xlt13m0\nnS+M7Dn/ZiOFmjXZ9AJTqtYnA9v7F4qIZcCy4VQgqTsi2ofXvAOTz3l0GG3nPNrOF8o552a9jbYe\naJN0gqRDgAuB1SW3ycxs1GrKnk1E7JP0SWANxdDn5RHxWMnNMjMbtZoy2QBExF3AXRmrGNbttwOc\nz3l0GG3nPNrOF0o4Z0W86bm5mZnZiGrWZzZmZvY24mQzRJI6JD0pqUfSwrLbk4OkKZLuk7RZ0mOS\nLk3xoyR1SdqSvieU3daRJmmMpIcl3ZHWT5D0QDrnm9OAk6YhabykWyU9ka73+5r9Okv6T+nf60cl\n3Sjp0Ga7zpKWS9op6dGqWM3rqsKS9Ddto6TpOdrkZDMEo2ganH3An0XEu4GZwIJ0nguBtRHRBqxN\n683mUmBz1fpVwNXpnHcD80ppVT5fA34YEScDp1Kce9NeZ0mTgE8B7RFxCsUAogtpvut8PdDRLzbQ\ndZ0NtKXPfGBpjgY52QzN69PgRMSrQGUanKYSETsi4qG0vJfiD9AkinNdkYqtAM4rp4V5SJoMnAtc\nm9YFfAi4NRVpqnOWdCTwfuA6gIh4NSJeoMmvM8XAqHGSxgKHATtosuscET8FdvULD3Rd5wAro3A/\nMF7SxJFuk5PN0Iy6aXAktQKnAQ8Ax0XEDigSEnBseS3L4qvAXwC/SutHAy9ExL603mzX+0SgD/h2\nunV4raTDaeLrHBF/B3wZeIYiyewBNtDc17lioOu6X/6uOdkMTUPT4DQLSUcA3wM+HREvlt2enCR9\nFNgZERuqwzWKNtP1HgtMB5ZGxGnAL2iiW2a1pOcUc4ATgH8KHE5xG6m/ZrrOg9kv/5472QxNQ9Pg\nNANJB1Mkmu9ExPdT+NlK9zp97yyrfRmcCXxM0laK26MfoujpjE+3W6D5rncv0BsRD6T1WymSTzNf\n5w8DT0dEX0T8I/B94F/Q3Ne5YqDrul/+rjnZDM2omAYnPau4DtgcEV+p2rQa6EzLncDt+7ttuUTE\nZRExOSJaKa7rvRHxCeA+4PxUrNnO+e+BbZJOSqGzKF7D0bTXmeL22UxJh6V/zyvn3LTXucpA13U1\nMDeNSpsJ7KncbhtJ/lHnEEk6h+L/eCvT4FxZcpNGnKTfAf43sIlfP7/4PMVzm1XA8RT/0V4QEf0f\nQh7wJH0A+GxEfFTSiRQ9naOAh4F/HRGvlNm+kSRpGsWAiEOAp4CLKf4ntGmvs6QvAR+nGHX5MPAn\nFM8omuY6S7oR+ADF7M7PAouAH1Djuqak+3WK0WsvAxdHRPeIt8nJxszMcvNtNDMzy87JxszMsnOy\nMTOz7JxszMwsOycbMzPLzsnGRi1J/znN/rtR0iOSzii7TW+FpOslnT94yWEff1oa+l9Z/6Kkz+aq\nz5pL076p06weSe8DPgpMj4hXJB1D8VsTG9g0oJ28b8C1JuWejY1WE4HnKj/ci4jnImI7gKQZkn4i\naYOkNVVTfMyQ9DNJ6yT9t8q7QiT9saSvVw4s6Y70w1AkzUrlH5J0S5pvDklbJX0pxTdJOjnFj5D0\n7RTbKOmP6h2nEZL+XNL6dLwvpVirivfXfCv17u6RNC5te28q+/p5phkzLgc+nnqBH0+Hnyrpx5Ke\nkvSpYV8Na3pONjZa3QNMkfT/JH1D0u/B63PC/Q/g/IiYASwHKrNEfBv4VES8r5EKUm/pC8CHI2I6\n0A18pqrIcym+FKjcjvovFNOF/POI+G3g3gaOU68NsyjeU3I6Rc9khqT3p81twDUR8R7gBeCPqs7z\n36fzfA2K1w8AfwncHBHTIuLmVPZk4Ox0/EXpn5/Zm/g2mo1KEfGSpBnA7wIfBG5W8ebVbuAUoKuY\nxYMxwA5J7wTGR8RP0iFuoPZswdVmUrxk72/SsQ4B1lVtr0xwugH4w7T8YYq52Srt3J1mpK53nHpm\npc/Daf0IiiTzDMWElI9UtaFV0njgHRHxf1P8uxS3GwdyZ+odviJpJ3AcxcSOZm/gZGOjVkS8BvwY\n+LGkTRSTE24AHuvfe0l/hAea22kfb7xLcGhlN6ArIi4aYL/K3Fuv8ev/FlWjnsGOU4+A/xoR33xD\nsHhPUfXcX68B46g93Xw9/Y/hvylWk2+j2agk6SRJbVWhacDPgSeBljSAAEkHS3pPeoPlnjRJKcAn\nqvbdCkyTdJCkKRS3lADuB86U9K50rMMk/bNBmnYP8Mmqdk4Y5nEq1gD/tupZ0SRJA74MLSJ2A3vT\n7L9Q1csC9gLvaLBeszdwsrHR6ghghaTHJW2kuE31xfRs4nzgKkk/Ax6heN8JFDMiXyNpHfDLqmP9\nDfA0xSzZXwYqr9TuA/4YuDHVcT/FM456/gqYkB7K/wz44BCP801JvemzLiLuobgVti713m5l8IQx\nD1iWzlMUb7OEYhr+qf0GCJg1xLM+mw1Dug11R0ScUnJTRpykIyLipbS8EJgYEZeW3Cw7wPn+qpn1\nd66kyyj+Pvycoldl9pa4Z2NmZtn5mY2ZmWXnZGNmZtk52ZiZWXZONmZmlp2TjZmZZedkY2Zm2f1/\n9nSIlffErG4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7feed9e95dd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#### plot to see the distribution of sentences length\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "get_ipython().magic(u'matplotlib inline')\n",
    "plt.hist(lengths_X['counts'], 20)\n",
    "plt.xlabel('Sequence Length')\n",
    "plt.ylabel('Frequency')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAEKCAYAAAC7c+rvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAH3NJREFUeJzt3X+8VXWd7/HXO8hfmYJyNAOaQ3X6gd4yPCn9HNNC0Eac\nGZ1wfFzPGHe409WppmkSshtl+Ri9043ijjpRkOAtkeiHTGpIqPm4c/HHwR8g/ohzleQIxTEQKQvC\nPveP9d21OO59zmbDPt+D5/18PPZjr/VZ3/X9fveSfT6utb77uxQRmJmZ5fCy3B0wM7Ohy0nIzMyy\ncRIyM7NsnITMzCwbJyEzM8vGScjMzLJxEjIzs2ychMzMLBsnITMzy2Z47g4MdqNGjYrW1tbc3TAz\nO6CsXr36mYho6a+ck1A/Wltb6ezszN0NM7MDiqSf1VPOl+PMzCwbJyEzM8vGScjMzLJpWhKStEDS\nFkkP94r/vaTHJa2T9D9K8VmSutK2M0rxySnWJWlmKT5O0j2S1ku6UdJBKX5wWu9K21v7a8PMzPJo\n5pnQdcDkckDS+4CpwFsi4njgSyk+HpgGHJ/2uUbSMEnDgKuBKcB44PxUFuAqYE5EtAHbgOkpPh3Y\nFhGvB+akcjXbaMLnNjOzOjUtCUXEXcDWXuGPAFdGxM5UZkuKTwUWR8TOiHgS6AJOTq+uiHgiInYB\ni4GpkgScBixN+y8EzinVtTAtLwVOT+VrtWFmZpkM9D2hNwDvSZfJfiLp7Sk+GthYKtedYrXiRwPP\nRsTuXvE96krbt6fytep6EUkzJHVK6uzp6Wnog5qZWf8GOgkNB0YCE4F/ApaksxRVKRsNxGlwnz2D\nEfMioj0i2lta+v2tlZmZNWigk1A38L0o3Av8HhiV4mNL5cYAm/qIPwOMkDS8V5zyPmn7kRSXBWvV\nZWZmmQz0jAk/oLiXc6ekNwAHUSSUZcC3JX0ZeDXQBtxLcfbSJmkc8DTFwIK/joiQdAdwLsV9og7g\nptTGsrS+Km2/PZWv1UbTtM68uZnV92nDlWdla9vMrF5NS0KSbgBOBUZJ6gZmAwuABWnY9i6gIyIC\nWCdpCfAIsBu4OCJeSPVcAiwHhgELImJdauJSYLGkLwIPAPNTfD5wvaQuijOgaQARUbMNMzPLQ0UO\nsFra29uj0bnjfCZkZkOVpNUR0d5fOc+YYGZm2TgJmZlZNk5CZmaWjZOQmZll4yRkZmbZOAmZmVk2\nTkJmZpaNk5CZmWXjJGRmZtk4CZmZWTZOQmZmlo2TkJmZZeMkZGZm2TgJmZlZNk5CZmaWjZOQmZll\n4yRkZmbZNC0JSVogaUt6lHfvbZ+UFJJGpXVJmiupS9IaSRNKZTskrU+vjlL8JElr0z5zJSnFj5K0\nIpVfIWlkf22YmVkezTwTug6Y3DsoaSzwAeCpUngK0JZeM4BrU9mjgNnAKcDJwOxKUkllZpT2q7Q1\nE1gZEW3AyrResw0zM8unaUkoIu4CtlbZNAf4FBCl2FRgURTuBkZIOg44A1gREVsjYhuwApicth0R\nEasiIoBFwDmluham5YW94tXaMDOzTAb0npCks4GnI+KhXptGAxtL690p1le8u0oc4NiI2AyQ3o/p\np41q/ZwhqVNSZ09PT52fzszM9taAJSFJhwGXAZ+ttrlKLBqI99mFeveJiHkR0R4R7S0tLf1Ua2Zm\njRrIM6HXAeOAhyRtAMYA90t6FcVZydhS2THApn7iY6rEAX5RucyW3rekeK26zMwskwFLQhGxNiKO\niYjWiGilSAoTIuLnwDLgwjSCbSKwPV1KWw5MkjQyDUiYBCxP23ZImphGxV0I3JSaWgZURtF19IpX\na8PMzDIZ3qyKJd0AnAqMktQNzI6I+TWK3wKcCXQBzwMXAUTEVklfAO5L5S6PiMpgh49QjMA7FLg1\nvQCuBJZImk4xAu+8vtowM7N8mpaEIuL8fra3lpYDuLhGuQXAgirxTuCEKvFfAqdXiddsw8zM8vCM\nCWZmlo2TkJmZZeMkZGZm2TgJmZlZNk5CZmaWjZOQmZll4yRkZmbZOAmZmVk2TkJmZpaNk5CZmWXj\nJGRmZtk4CZmZWTZOQmZmlo2TkJmZZeMkZGZm2TgJmZlZNk5CZmaWTdOSkKQFkrZIergU+xdJj0la\nI+n7kkaUts2S1CXpcUlnlOKTU6xL0sxSfJykeyStl3SjpINS/OC03pW2t/bXhpmZ5dHMM6HrgMm9\nYiuAEyLiLcBPgVkAksYD04Dj0z7XSBomaRhwNTAFGA+cn8oCXAXMiYg2YBswPcWnA9si4vXAnFSu\nZhv7+0ObmVn9mpaEIuIuYGuv2G0RsTut3g2MSctTgcURsTMingS6gJPTqysinoiIXcBiYKokAacB\nS9P+C4FzSnUtTMtLgdNT+VptmJlZJjnvCX0YuDUtjwY2lrZ1p1it+NHAs6WEVonvUVfavj2Vr1WX\nmZllkiUJSboM2A18qxKqUiwaiDdSV7X+zZDUKamzp6enWhEzM9sPBjwJSeoAPghcEBGVJNANjC0V\nGwNs6iP+DDBC0vBe8T3qStuPpLgsWKuuF4mIeRHRHhHtLS0tjXxMMzOrw4AmIUmTgUuBsyPi+dKm\nZcC0NLJtHNAG3AvcB7SlkXAHUQwsWJaS1x3AuWn/DuCmUl0daflc4PZUvlYbZmaWyfD+izRG0g3A\nqcAoSd3AbIrRcAcDK4qxAtwdEX8XEeskLQEeobhMd3FEvJDquQRYDgwDFkTEutTEpcBiSV8EHgDm\np/h84HpJXRRnQNMA+mrDzMzy0B+viFk17e3t0dnZ2dC+rTNv3s+9qd+GK8/K1raZmaTVEdHeXznP\nmGBmZtk4CZmZWTZOQmZmlo2TkJmZZeMkZGZm2TgJmZlZNk5CZmaWjZOQmZll4yRkZmbZOAmZmVk2\nTkJmZpaNk5CZmWXjJGRmZtk4CZmZWTZOQmZmlo2TkJmZZeMkZGZm2TQtCUlaIGmLpIdLsaMkrZC0\nPr2PTHFJmiupS9IaSRNK+3Sk8usldZTiJ0lam/aZq/S88EbaMDOzPJp5JnQdMLlXbCawMiLagJVp\nHWAK0JZeM4BroUgowGzgFOBkYHYlqaQyM0r7TW6kDTMzy6euJCTphL2tOCLuArb2Ck8FFqblhcA5\npfiiKNwNjJB0HHAGsCIitkbENmAFMDltOyIiVkVEAIt61bU3bZiZWSb1ngn9m6R7Jf03SSP2ob1j\nI2IzQHo/JsVHAxtL5bpTrK94d5V4I22YmVkmdSWhiHg3cAEwFuiU9G1JH9iP/VC1ZhuIN9LGiwtK\nMyR1Surs6enpp1ozM2tU3feEImI98BngUuBPgbmSHpP0F3vR3i8ql8DS+5YU76ZIcBVjgE39xMdU\niTfSxotExLyIaI+I9paWlr34eGZmtjfqvSf0FklzgEeB04A/i4g3p+U5e9HeMqAywq0DuKkUvzCN\nYJsIbE+X0pYDkySNTAMSJgHL07YdkiamUXEX9qprb9owM7NMhtdZ7l+BrwOfjojfVIIRsUnSZ6rt\nIOkG4FRglKRuilFuVwJLJE0HngLOS8VvAc4EuoDngYtS/VslfQG4L5W7PCIqgx0+QjEC71Dg1vRi\nb9swM7N8VAwu66eQdDjwm4h4Ia2/DDgkIp5vcv+ya29vj87Ozob2bZ15837uTf02XHlWtrbNzCSt\njoj2/srVe0/oxxRnHBWHpZiZmVnD6k1Ch0TEryorafmw5nTJzMyGinqT0K97TaVzEvCbPsqbmZn1\nq96BCR8HviOpMqT5OOBDzemSmZkNFXUloYi4T9KbgDdS/OjzsYj4XVN7ZmZmL3n1ngkBvB1oTfu8\nTRIRsagpvTIzsyGhriQk6XrgdcCDwAspXJk41MzMrCH1ngm1A+Ojnh8VmZmZ1ane0XEPA69qZkfM\nzGzoqfdMaBTwiKR7gZ2VYESc3ZRemZnZkFBvEvpcMzthZmZDU71DtH8i6U+Atoj4saTDgGHN7ZqZ\nmb3U1fsoh78FlgJfS6HRwA+a1SkzMxsa6h2YcDHwLuA5+MMD7o7pcw8zM7N+1JuEdkbErsqKpOH0\n/zhtMzOzPtWbhH4i6dPAoZI+AHwH+PfmdcvMzIaCepPQTKAHWAv8V4qnlFZ9oqqZmVm96kpCEfH7\niPh6RJwXEeem5YYvx0n6B0nrJD0s6QZJh0gaJ+keSesl3SjpoFT24LTelba3luqZleKPSzqjFJ+c\nYl2SZpbiVdswM7M86h0d96SkJ3q/GmlQ0mjgo0B7RJxAMdR7GnAVMCci2oBtwPS0y3RgW0S8HpiT\nyiFpfNrveGAycI2kYZKGAVcDU4DxwPmpLH20YWZmGdR7Oa6dYhbttwPvAeYC/3sf2h1OcX9pOMUT\nWjcDp1EMAwdYCJyTlqemddL20yUpxRdHxM6IeBLoAk5Or66IeCINplgMTE371GrDzMwyqPdy3C9L\nr6cj4isUf9D3WkQ8DXwJeIoi+WwHVgPPRsTuVKyb4rdIpPeNad/dqfzR5XivfWrFj+6jDTMzy6De\nRzlMKK2+jOLM6JWNNChpJMVZzDjgWYqRdlOqFK3cc1KNbbXi1RJrX+Wr9XEGMAPgNa95TbUiZma2\nH9Q7d9z/LC3vBjYAf9Vgm+8HnoyIHgBJ3wPeCYyQNDydqYwBKo8S7wbGAt3p8t2RwNZSvKK8T7X4\nM320sYeImAfMA2hvb/fvoczMmqTeuePetx/bfAqYmOaf+w1wOtAJ3AGcS3EPpwO4KZVfltZXpe23\nR0RIWgZ8W9KXgVcDbcC9FGc8bZLGAU9TDF7467RPrTbMzCyDei/HfaKv7RHx5XobjIh7JC0F7qc4\nq3qA4qzjZmCxpC+m2Py0y3zgekldFGdA01I96yQtAR5J9VwcES+k/l4CLKcYebcgItalui6t0YaZ\nmWWwN09WfTvFWQnAnwF3secAgLpFxGxgdq/wExQj23qX/S1wXo16rgCuqBK/heIHtb3jVdswM7M8\n9uahdhMiYgeApM8B34mI/9KsjpmZ2Utfvb8Teg2wq7S+C2jd770xM7Mhpd4zoeuBeyV9n2JY858D\ni5rWKzMzGxLqHR13haRbKWZLALgoIh5oXrfMzGwoqPdyHBTT6zwXEV+l+M3OuCb1yczMhoh6JzCd\nTTG8eVYKvZx9mzvOzMys7jOhPwfOBn4NEBGbaHDaHjMzs4p6k9Cu9PygAJD0iuZ1yczMhop6k9AS\nSV+jmHvtb4EfA19vXrfMzGwoqHd03JckfQB4Dngj8NmIWNHUnpmZ2Utev0koPal0eUS8H3DiMTOz\n/abfy3FpUtDnJR05AP0xM7MhpN4ZE34LrJW0gjRCDiAiPtqUXpmZ2ZBQbxK6Ob3MzMz2mz6TkKTX\nRMRTEbFwoDpkZmZDR3/3hH5QWZD03Sb3xczMhpj+kpBKy69tZkfMzGzo6S8JRY1lMzOzfdZfEnqr\npOck7QDekpafk7RD0nONNipphKSlkh6T9Kikd0g6StIKSevT+8hUVpLmSuqStEbShFI9Han8ekkd\npfhJktamfeZKUopXbcPMzPLoMwlFxLCIOCIiXhkRw9NyZf2IfWj3q8CPIuJNwFuBR4GZwMqIaANW\npnWAKUBbes0AroUioQCzgVOAk4HZpaRybSpb2W9yitdqw8zMMtib5wntF5KOAN4LzAeIiF0R8Sww\nFaiMwlsInJOWpwKLonA3xfx1xwFnACsiYmtEbKOYzWFy2nZERKxKk64u6lVXtTbMzCyDAU9CFAMc\neoBvSnpA0jfSrNzHRsRmgPR+TCo/GthY2r87xfqKd1eJ00cbe5A0Q1KnpM6enp7GP6mZmfUpRxIa\nDkwAro2It1HMwNDXZTFViUUD8bpFxLyIaI+I9paWlr3Z1czM9kKOJNQNdEfEPWl9KUVS+kW6lEZ6\n31IqP7a0/xhgUz/xMVXi9NGGmZllMOBJKCJ+DmyU9MYUOh14BFgGVEa4dQA3peVlwIVplNxEYHu6\nlLYcmCRpZBqQMIlitu/NwA5JE9OouAt71VWtDTMzy6DeueP2t78HviXpIOAJ4CKKhLhE0nTgKeC8\nVPYW4EygC3g+lSUitkr6AnBfKnd5RGxNyx8BrgMOBW5NL4Ara7RhZmYZZElCEfEg0F5l0+lVygZw\ncY16FgALqsQ7gROqxH9ZrQ0zM8sjxz0hMzMzwEnIzMwychIyM7NsnITMzCwbJyEzM8vGScjMzLJx\nEjIzs2ychMzMLBsnITMzy8ZJyMzMsnESMjOzbJyEzMwsm1yzaFuTtc68OUu7G648K0u7ZnZg8pmQ\nmZll4yRkZmbZOAmZmVk2TkJmZpZNtiQkaZikByT9MK2Pk3SPpPWSbkyP/kbSwWm9K21vLdUxK8Uf\nl3RGKT45xbokzSzFq7ZhZmZ55DwT+hjwaGn9KmBORLQB24DpKT4d2BYRrwfmpHJIGg9MA44HJgPX\npMQ2DLgamAKMB85PZftqw8zMMsiShCSNAc4CvpHWBZwGLE1FFgLnpOWpaZ20/fRUfiqwOCJ2RsST\nQBdwcnp1RcQTEbELWAxM7acNMzPLINeZ0FeATwG/T+tHA89GxO603g2MTsujgY0Aafv2VP4P8V77\n1Ir31cYeJM2Q1Cmps6enp9HPaGZm/RjwJCTpg8CWiFhdDlcpGv1s21/xFwcj5kVEe0S0t7S0VCti\nZmb7QY4ZE94FnC3pTOAQ4AiKM6MRkoanM5UxwKZUvhsYC3RLGg4cCWwtxSvK+1SLP9NHG2ZmlsGA\nnwlFxKyIGBMRrRQDC26PiAuAO4BzU7EO4Ka0vCytk7bfHhGR4tPS6LlxQBtwL3Af0JZGwh2U2liW\n9qnVhpmZZTCYfid0KfAJSV0U92/mp/h84OgU/wQwEyAi1gFLgEeAHwEXR8QL6SznEmA5xei7Jals\nX22YmVkGWScwjYg7gTvT8hMUI9t6l/ktcF6N/a8ArqgSvwW4pUq8ahtmZpbHYDoTMjOzIcZJyMzM\nsnESMjOzbJyEzMwsGychMzPLxknIzMyycRIyM7NsnITMzCwbJyEzM8vGScjMzLJxEjIzs2ychMzM\nLBsnITMzy8ZJyMzMsnESMjOzbJyEzMwsGychMzPLZsCTkKSxku6Q9KikdZI+luJHSVohaX16H5ni\nkjRXUpekNZImlOrqSOXXS+ooxU+StDbtM1eS+mrDzMzyyHEmtBv4x4h4MzARuFjSeGAmsDIi2oCV\naR1gCtCWXjOAa6FIKMBs4BSKR3bPLiWVa1PZyn6TU7xWG2ZmlsGAJ6GI2BwR96flHcCjwGhgKrAw\nFVsInJOWpwKLonA3MELSccAZwIqI2BoR24AVwOS07YiIWBURASzqVVe1NszMLIOs94QktQJvA+4B\njo2IzVAkKuCYVGw0sLG0W3eK9RXvrhKnjzbMzCyDbElI0uHAd4GPR8RzfRWtEosG4nvTtxmSOiV1\n9vT07M2uZma2F7IkIUkvp0hA34qI76XwL9KlNNL7lhTvBsaWdh8DbOonPqZKvK829hAR8yKiPSLa\nW1paGvuQZmbWrxyj4wTMBx6NiC+XNi0DKiPcOoCbSvEL0yi5icD2dCltOTBJ0sg0IGESsDxt2yFp\nYmrrwl51VWvDzMwyGJ6hzXcB/xlYK+nBFPs0cCWwRNJ04CngvLTtFuBMoAt4HrgIICK2SvoCcF8q\nd3lEbE3LHwGuAw4Fbk0v+mjDzMwyGPAkFBH/h+r3bQBOr1I+gItr1LUAWFAl3gmcUCX+y2ptmJlZ\nHp4xwczMsnESMjOzbJyEzMwsGychMzPLxknIzMyycRIyM7NsnITMzCwbJyEzM8vGScjMzLJxEjIz\ns2ychMzMLBsnITMzy8ZJyMzMssnxKAd7CWudeXO2tjdceVa2ts2sMT4TMjOzbJyEzMwsGychMzPL\nZkgmIUmTJT0uqUvSzNz9MTMbqoZcEpI0DLgamAKMB86XND5vr8zMhqYhl4SAk4GuiHgiInYBi4Gp\nmftkZjYkDcUh2qOBjaX1buCUTH2x/SjX8HAPDTdr3FBMQqoSiz0KSDOAGWn1V5Ieb7CtUcAzDe6b\ni/u8l3RVQ7sdaMf5QOsvuM8DpVaf/6SenYdiEuoGxpbWxwCbygUiYh4wb18bktQZEe37Ws9Acp8H\nxoHW5wOtv+A+D5R97fNQvCd0H9AmaZykg4BpwLLMfTIzG5KG3JlQROyWdAmwHBgGLIiIdZm7ZWY2\nJA25JAQQEbcAtwxAU/t8SS8D93lgHGh9PtD6C+7zQNmnPisi+i9lZmbWBEPxnpCZmQ0STkJNciBM\nDSRprKQ7JD0qaZ2kj6X4UZJWSFqf3kfm7muZpGGSHpD0w7Q+TtI9qb83pgEng4akEZKWSnosHet3\nHADH+B/Sv4mHJd0g6ZDBdpwlLZC0RdLDpVjV46rC3PR9XCNpwiDq87+kfxtrJH1f0ojStlmpz49L\nOmMw9Le07ZOSQtKotN7QMXYSaoIDaGqg3cA/RsSbgYnAxamfM4GVEdEGrEzrg8nHgEdL61cBc1J/\ntwHTs/Sqtq8CP4qINwFvpej7oD3GkkYDHwXaI+IEigE80xh8x/k6YHKvWK3jOgVoS68ZwLUD1Mfe\nruPFfV4BnBARbwF+CswCSN/FacDxaZ9r0t+WgXQdL+4vksYCHwCeKoUbOsZOQs1xQEwNFBGbI+L+\ntLyD4o/jaIq+LkzFFgLn5Onhi0kaA5wFfCOtCzgNWJqKDLb+HgG8F5gPEBG7IuJZBvExToYDh0oa\nDhwGbGaQHeeIuAvY2itc67hOBRZF4W5ghKTjBqanf1StzxFxW0TsTqt3U/x2EYo+L46InRHxJNBF\n8bdlwNQ4xgBzgE+x5w/9GzrGTkLNUW1qoNGZ+lIXSa3A24B7gGMjYjMUiQo4Jl/PXuQrFP/4f5/W\njwaeLX2JB9uxfi3QA3wzXUL8hqRXMIiPcUQ8DXyJ4v9yNwPbgdUM7uNcUeu4HijfyQ8Dt6blQdln\nSWcDT0fEQ702NdRfJ6Hm6HdqoMFE0uHAd4GPR8RzuftTi6QPAlsiYnU5XKXoYDrWw4EJwLUR8Tbg\n1wyiS2/VpPsoU4FxwKuBV1BcaultMB3n/gz2fydIuoziEvm3KqEqxbL2WdJhwGXAZ6ttrhLrt79O\nQs3R79RAg4Wkl1MkoG9FxPdS+BeV0+j0viVX/3p5F3C2pA0UlzhPozgzGpEuG8HgO9bdQHdE3JPW\nl1IkpcF6jAHeDzwZET0R8Tvge8A7GdzHuaLWcR3U30lJHcAHgQvij7+bGYx9fh3F/5w8lL6HY4D7\nJb2KBvvrJNQcB8TUQOl+ynzg0Yj4cmnTMqAjLXcANw1036qJiFkRMSYiWimO6e0RcQFwB3BuKjZo\n+gsQET8HNkp6YwqdDjzCID3GyVPAREmHpX8jlT4P2uNcUuu4LgMuTCO4JgLbK5ftcpM0GbgUODsi\nni9tWgZMk3SwpHEUN/zvzdHHiohYGxHHRERr+h52AxPSv/PGjnFE+NWEF3AmxUiX/wdclrs/Nfr4\nborT5TXAg+l1JsV9lpXA+vR+VO6+Vun7qcAP0/JrKb6cXcB3gINz969XX08EOtNx/gEwcrAfY+Dz\nwGPAw8D1wMGD7TgDN1Dcs/pd+mM4vdZxpbhUdHX6Pq6lGPk3WPrcRXEvpfId/LdS+ctSnx8HpgyG\n/vbavgEYtS/H2DMmmJlZNr4cZ2Zm2TgJmZlZNk5CZmaWjZOQmZll4yRkZmbZOAmZ9SLpsjSD9BpJ\nD0o6JXef9oWk6ySd23/Jhus/UdKZpfXPSfpks9qzl5Yh+WRVs1okvYPil+sTImJnmqZ+UD0aYhA6\nEWhnYJ5WbC8xPhMy29NxwDMRsRMgIp6JiE0Akk6S9BNJqyUtL00Pc5KkhyStSs+GeTjF/0bSv1Yq\nlvRDSaem5Ump/P2SvpPm70PSBkmfT/G1kt6U4odL+maKrZH0l33VUw9J/yTpvlTf51OsVcUzj76e\nzgZvk3Ro2vb2VPYPnzPNCHI58KF01vihVP14SXdKekLSRxv+r2EveU5CZnu6DRgr6aeSrpH0p/CH\nOfb+F3BuRJwELACuSPt8E/hoRLyjngbS2dVngPdHxASK2RQ+USryTIpfC1Qua/13imlQ/lMUz525\nvY56+urDJIppYE6mOJM5SdJ70+Y24OqIOB54FvjL0uf8u/Q5X4Di0RQUk1neGBEnRsSNqeybgDNS\n/bPT8TN7EV+OMyuJiF9JOgl4D/A+4EYVT8btBE4AVhTTqTEM2CzpSGBERPwkVXE91WecLptI8bDD\n/0h1HQSsKm2vTCS7GviLtPx+ivnyKv3clmYV76uevkxKrwfS+uEUyecpislLHyz1oVXF0z5fGRH/\nN8W/TXHZspab09nkTklbgGMppn0x24OTkFkvEfECcCdwp6S1FBNhrgbW9T7bSX+ca819tZs9rzYc\nUtkNWBER59fYb2d6f4E/fkdVpZ3+6umLgH+OiK/tESyeK7WzFHoBOJTq0/T3pXcd/ltjVflynFmJ\npDdKaiuFTgR+RjGBZEsauICkl0s6PoqnpG6X9O5U/oLSvhuAEyW9TMXjkCtPxbwbeJek16e6DpP0\nhn66dhtwSamfIxusp2I58OHSvajRkmo+WC8itgE70uzIUDorA3YAr6yzXbM9OAmZ7elwYKGkRySt\nobjc9bl07+Nc4CpJD1HMdvzOtM9FwNWSVgG/KdX1H8CTFDMKfwmoPEq9B/gb4IbUxt0U91D68kVg\nZBoM8BDwvr2s52uSutNrVUTcRnFJbVU621tK/4lkOjAvfU5RPHEVikc8jO81MMGsLp5F22w/Spez\nfhgRJ2Tuyn4n6fCI+FVangkcFxEfy9wtO8D5Oq2Z1essSbMo/m78jOIszGyf+EzIzMyy8T0hMzPL\nxknIzMyycRIyM7NsnITMzCwbJyEzM8vGScjMzLL5/7vNKAUXBA3DAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7feed94e33d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(lengths_y['counts'], 10)\n",
    "plt.xlabel('Sequence Length')\n",
    "plt.ylabel('Frequency')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unk_counter(sentence):\n",
    "    unk_count = 0\n",
    "    for word in sentence:\n",
    "        if word == word_to_int[\"<UNK>\"]:\n",
    "            unk_count += 1\n",
    "    return unk_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             counts\n",
      "count  2.108530e+06\n",
      "mean   1.003121e+01\n",
      "std    7.170949e+00\n",
      "min    0.000000e+00\n",
      "25%    5.000000e+00\n",
      "50%    8.000000e+00\n",
      "75%    1.400000e+01\n",
      "max    1.360000e+02\n",
      "             counts\n",
      "count  2.108530e+06\n",
      "mean   1.193619e+01\n",
      "std    7.234319e+00\n",
      "min    1.000000e+00\n",
      "25%    7.000000e+00\n",
      "50%    1.000000e+01\n",
      "75%    1.600000e+01\n",
      "max    1.010000e+02\n"
     ]
    }
   ],
   "source": [
    "print lengths_y.describe()\n",
    "print lengths_X.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "# parameters for batch parser\n",
    "# set the length according to the 90% and 95% quantile\n",
    "max_X_length = 50\n",
    "max_y_length = 50\n",
    "min_length = 2\n",
    "\n",
    "# drop the sample if we got too much unknown words\n",
    "unk_X_threshold = 1\n",
    "unk_y_threshold = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "def invalid_sample(code_list, invalid_ratio=0.75):\n",
    "    count = 0\n",
    "    for code in code_list:\n",
    "        if code < len(special_codes):\n",
    "            count += 1\n",
    "    if 1.0 * count / len(code_list) > invalid_ratio:\n",
    "        return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('number of sorted input', 879048)\n",
      "('number of sorted output', 879048)\n"
     ]
    }
   ],
   "source": [
    "sorted_y = []\n",
    "sorted_X = []\n",
    "\n",
    "for length in range(1, max_X_length):\n",
    "    for i, sentence in enumerate(int_y):\n",
    "        if (len(int_y[i]) >= min_length and\n",
    "            len(int_y[i]) <= max_y_length and\n",
    "            unk_counter(int_y[i]) <= unk_y_threshold and\n",
    "            unk_counter(int_X[i]) <= unk_X_threshold and\n",
    "            length == len(int_X[i])\n",
    "           ):\n",
    "            if invalid_sample(int_X[i]):\n",
    "                continue\n",
    "            sorted_y.append(int_y[i])\n",
    "            sorted_X.append(int_X[i])\n",
    "\n",
    "print('number of sorted input', len(sorted_X))\n",
    "print('number of sorted output', len(sorted_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[18868, 2],\n",
       " [5528, 2],\n",
       " [1752, 2],\n",
       " [18448, 2],\n",
       " [15938, 2],\n",
       " [14997, 2],\n",
       " [22241, 2],\n",
       " [22241, 2],\n",
       " [20821, 2],\n",
       " [15675, 2],\n",
       " [15675, 2],\n",
       " [14866, 2],\n",
       " [4885, 2],\n",
       " [25697, 2],\n",
       " [24223, 2],\n",
       " [14079, 2],\n",
       " [18448, 2],\n",
       " [23886, 2],\n",
       " [18868, 2],\n",
       " [10424, 2],\n",
       " [21044, 2],\n",
       " [13878, 2],\n",
       " [18192, 2],\n",
       " [17602, 2],\n",
       " [25697, 2],\n",
       " [22399, 2],\n",
       " [12937, 2],\n",
       " [16029, 2],\n",
       " [2504, 2],\n",
       " [8825, 2],\n",
       " [17249, 2],\n",
       " [6778, 2],\n",
       " [13157, 2],\n",
       " [1052, 2],\n",
       " [13265, 2],\n",
       " [25697, 2],\n",
       " [5504, 2],\n",
       " [20690, 2],\n",
       " [11915, 2],\n",
       " [15218, 2],\n",
       " [18448, 2],\n",
       " [4885, 2],\n",
       " [20093, 2],\n",
       " [13069, 2],\n",
       " [13069, 2],\n",
       " [8974, 2],\n",
       " [13188, 2],\n",
       " [8353, 2],\n",
       " [382, 2],\n",
       " [25697, 2],\n",
       " [15774, 2],\n",
       " [18592, 2],\n",
       " [3675, 2],\n",
       " [6845, 2],\n",
       " [1052, 2],\n",
       " [11235, 2],\n",
       " [22657, 2],\n",
       " [20821, 2],\n",
       " [22210, 2],\n",
       " [2202, 2],\n",
       " [10137, 2],\n",
       " [11080, 2],\n",
       " [25840, 2],\n",
       " [16857, 2],\n",
       " [11351, 2],\n",
       " [11209, 2],\n",
       " [18448, 2],\n",
       " [18448, 2],\n",
       " [5308, 2],\n",
       " [9668, 2],\n",
       " [19321, 2],\n",
       " [13847, 2],\n",
       " [13247, 2],\n",
       " [9777, 2],\n",
       " [15491, 2],\n",
       " [327, 2],\n",
       " [16677, 2],\n",
       " [18448, 2],\n",
       " [3625, 2],\n",
       " [24647, 2],\n",
       " [10288, 2],\n",
       " [22201, 2],\n",
       " [11399, 2],\n",
       " [13779, 2],\n",
       " [14552, 2],\n",
       " [18448, 2],\n",
       " [5493, 2],\n",
       " [20093, 2],\n",
       " [4708, 2],\n",
       " [12224, 2],\n",
       " [22096, 2],\n",
       " [4798, 2],\n",
       " [3549, 2],\n",
       " [20821, 2],\n",
       " [12666, 2],\n",
       " [21757, 2],\n",
       " [8508, 2],\n",
       " [10664, 2],\n",
       " [4836, 2],\n",
       " [3215, 2]]"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted_X[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(26259, 308)\n",
      "[  9.44000036e-02   1.11000001e-01   6.80000037e-02   1.65299997e-01\n",
      "   8.55000019e-02  -4.23999988e-02  -5.99000007e-02  -3.11999992e-02\n",
      "  -7.02000037e-02   7.76000023e-02  -3.64000015e-02  -1.39200002e-01\n",
      "  -8.72000009e-02   6.58999979e-02  -2.07399994e-01   3.66999991e-02\n",
      "  -7.49999983e-03  -2.63999999e-02   4.89999987e-02  -5.35999984e-02\n",
      "   6.32999986e-02   2.03500003e-01  -2.41999999e-02   8.55000019e-02\n",
      "   3.07999998e-02  -3.70000005e-02   2.54099995e-01  -1.09700002e-01\n",
      "   3.42000015e-02   2.53999997e-02   1.56399995e-01   3.51999998e-02\n",
      "  -7.63999969e-02   1.47000002e-02   3.53999995e-02   1.36000002e-02\n",
      "   2.52000000e-02   2.77999993e-02  -5.13999984e-02   3.62999998e-02\n",
      "   2.00900003e-01   7.23000020e-02  -2.93000005e-02   1.65999997e-02\n",
      "  -7.86999986e-02  -9.71999988e-02   1.26999998e-02  -3.61000001e-02\n",
      "   7.19999988e-03   2.89999996e-03   2.51000002e-02   8.98000002e-02\n",
      "  -6.87000006e-02  -2.42999997e-02  -4.12000008e-02  -1.07799999e-01\n",
      "   4.17000018e-02   6.62000030e-02   6.17000014e-02   7.69999996e-03\n",
      "  -2.78999992e-02  -3.79999988e-02  -9.52999964e-02   2.85999998e-02\n",
      "  -6.36000037e-02   5.73000014e-02   1.41000003e-02   4.10000002e-03\n",
      "  -4.43000011e-02   1.63000003e-02  -1.83000006e-02  -7.19999969e-02\n",
      "   8.79999995e-02  -1.04000000e-02  -5.48000000e-02   8.82999972e-02\n",
      "  -7.02000037e-02  -1.45099998e-01  -1.38899997e-01  -6.35000020e-02\n",
      "   1.62000004e-02   1.10999998e-02   1.09999999e-02   1.36999995e-01\n",
      "   1.42000001e-02   5.16000018e-02  -4.41999994e-02  -1.00000005e-03\n",
      "   1.07000005e-02   3.50000001e-02   2.17000004e-02   2.40000011e-03\n",
      "   6.53000027e-02  -9.39999986e-03  -9.70000029e-03   2.06000004e-02\n",
      "  -1.10999998e-02   5.38000017e-02   2.77999993e-02  -8.10000021e-03\n",
      "   1.59000009e-02   6.03000000e-02  -5.70000000e-02   6.30000001e-03\n",
      "  -9.09999982e-02  -1.37999998e-02   5.02999984e-02   6.12999983e-02\n",
      "   1.11999996e-02   1.10999998e-02   2.88999993e-02   1.44999996e-02\n",
      "   1.20000001e-02  -2.38000005e-02  -8.51999968e-02   1.45899996e-01\n",
      "  -1.70000002e-03   6.41999990e-02   4.25999984e-02  -6.71999976e-02\n",
      "   9.74000022e-02   3.62999998e-02  -5.73000014e-02  -5.59999980e-03\n",
      "  -1.26000000e-02   4.25999984e-02   4.45000008e-02   9.99999975e-05\n",
      "   1.37199998e-01   1.48699999e-01  -5.64999990e-02   8.79999995e-03\n",
      "   1.53000001e-02  -7.15000033e-02   7.82999992e-02   3.97000015e-02\n",
      "   1.60000008e-02  -2.06000004e-02  -7.72000030e-02  -1.93000007e-02\n",
      "  -2.73000002e-02  -1.59999996e-03  -6.93999976e-02   1.16999997e-02\n",
      "   3.29999998e-03  -1.22999996e-02  -2.83000004e-02   3.22000012e-02\n",
      "   2.60000005e-02  -9.43000019e-02  -8.20999965e-02   4.14000005e-02\n",
      "  -9.01999995e-02   3.48000005e-02  -9.75999981e-02   1.90999992e-02\n",
      "  -1.47000002e-02  -4.90000006e-03  -4.16000001e-02  -2.19999999e-02\n",
      "   1.16999997e-02  -5.05000018e-02  -5.13999984e-02   2.74000000e-02\n",
      "  -2.43999995e-02   2.73000002e-02  -6.06999993e-02  -2.47000009e-02\n",
      "  -8.86000022e-02  -1.70000009e-02   4.45000008e-02  -8.70999992e-02\n",
      "  -5.90000004e-02  -1.94000006e-02  -3.00000003e-03  -1.09999999e-02\n",
      "  -8.70000012e-03   9.69000012e-02  -8.70000012e-03  -7.50000030e-02\n",
      "  -2.82000005e-02   2.89999996e-03   6.99999975e-04  -4.30999994e-02\n",
      "  -0.00000000e+00  -8.33000019e-02   1.94000006e-02   3.29999998e-03\n",
      "   3.07999998e-02   9.43000019e-02   8.29999987e-03   5.31999990e-02\n",
      "  -1.30000003e-02  -6.99999975e-04  -3.03000007e-02   1.10999998e-02\n",
      "  -3.03000007e-02   2.09999997e-02   4.60000001e-02   6.00000028e-04\n",
      "   8.12000036e-02   4.89000008e-02  -3.90000008e-02  -3.06000002e-02\n",
      "  -3.18000019e-02   3.70000000e-03  -7.89999962e-03  -4.06000018e-02\n",
      "  -1.94000006e-02  -8.70000012e-03  -9.49999969e-03   1.86000001e-02\n",
      "   3.48000005e-02   5.04000001e-02  -5.79999993e-03  -3.66999991e-02\n",
      "  -2.09999997e-02  -1.29000004e-02  -9.39999986e-03   5.18000014e-02\n",
      "   5.53000011e-02   1.83000006e-02   9.49999969e-03   4.10999991e-02\n",
      "  -2.72000004e-02   4.65999991e-02  -2.30999999e-02  -3.53000015e-02\n",
      "   6.80000037e-02  -5.02999984e-02   5.29999984e-03   2.16000006e-02\n",
      "   7.30000017e-03   4.03999984e-02   6.39000013e-02   4.69999984e-02\n",
      "  -8.46000016e-02   1.04000000e-02  -3.84000018e-02   1.08400002e-01\n",
      "   3.48999985e-02   2.77999993e-02   6.99999975e-04  -2.09999993e-03\n",
      "   6.01999983e-02  -3.40000018e-02   3.20999995e-02   2.41999999e-02\n",
      "   7.07999989e-02   4.28000018e-02   3.75000015e-02   1.00999996e-02\n",
      "  -3.08999997e-02   8.16999972e-02   1.73000004e-02   2.61000004e-02\n",
      "  -9.71999988e-02   2.05000006e-02  -3.46999988e-02  -3.57000008e-02\n",
      "  -4.39999998e-03   2.33999994e-02  -2.39000004e-02  -2.72000004e-02\n",
      "  -4.19999985e-03  -2.09999997e-02   4.63999994e-02   3.75000015e-02\n",
      "   2.53999997e-02  -1.03299998e-01  -1.02000004e-02  -3.99999991e-02\n",
      "  -2.97999997e-02  -3.75000015e-02   4.05000001e-02  -1.61000006e-02\n",
      "   1.42000001e-02  -4.54999991e-02  -4.01000008e-02  -4.78999987e-02\n",
      "   3.59999985e-02   2.99999993e-02   3.18000019e-02  -2.56999992e-02\n",
      "  -1.02000004e-02  -2.47000009e-02   2.22999994e-02  -5.59999980e-03\n",
      "   1.26000000e-02  -5.10000018e-03   1.41000003e-02   3.17000002e-02\n",
      "  -7.78999999e-02   1.37000000e-02  -7.90999979e-02  -2.99999993e-02\n",
      "   5.24000004e-02  -7.51999989e-02   1.39999995e-03   4.16000001e-02\n",
      "   0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00\n",
      "   0.00000000e+00   0.00000000e+00   0.00000000e+00   0.00000000e+00]\n"
     ]
    }
   ],
   "source": [
    "print word_embedding_matrix.shape\n",
    "print word_embedding_matrix[-2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modeling Using tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SET Hyperparams at first !!!\n",
    "learning_rate = 0.001\n",
    "learning_rate_decay = 0.95\n",
    "min_learning_rate = 0.00005\n",
    "epochs = 100\n",
    "batch_size = 32\n",
    "keep_probability = 0.75\n",
    "# 1 - GradientDescentOptimizer\n",
    "# 2 - AdamOptimizer\n",
    "# 3 - RMSPropOptimizer\n",
    "model_optimizer = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyperparams for cells\n",
    "# 1 - Basic RNN\n",
    "# 2 - GRU\n",
    "# 3 - LSTM\n",
    "encoder_cell_type = 3\n",
    "decoder_cell_type = 3\n",
    "rnn_dim = 512\n",
    "encoder_forget_bias = 1.0\n",
    "decoder_forget_bias = 1.0\n",
    "\n",
    "# 1 - tf.random_uniform_initializer\n",
    "# 2 - tf.truncated_normal_initializer\n",
    "# 3 - tf.orthogonal_initializer\n",
    "initializer_type = 3\n",
    "\n",
    "# 1 - Relu\n",
    "# 2 - tanh\n",
    "activation = None\n",
    "num_layers = 1\n",
    "\n",
    "# Hyperparams for attentions\n",
    "# 1 - tf.contrib.seq2seq.BahdanauAttention()\n",
    "# 2 - tf.contrib.seq2seq.LuongAttetion()\n",
    "# 3 - no attention\n",
    "attention_type = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "# others \n",
    "# gradient clipping\n",
    "# the steep cliffs commonly occur in recurrent neural networks in the area where\n",
    "# the recurrent network behaves approximately linearly, SGD or other methods without\n",
    "# gradient clipping overshoots the landscape minimum, while the one with gradient\n",
    "# clipping descents into the minimum.\n",
    "# 1 - tf.clip_by_values(tensor, clip_value_min, clip_value_max)\n",
    "# value less than clip_value_min and greater than clip_value_max will constrained\n",
    "# 2 - tf.clip_by_norm(tensor, clip_norm, axes = None, name = None)\n",
    "# tensor = t * clip_norm / l2norm(t)\n",
    "model_gradient_clipping = 1\n",
    "clip_value_min = -3\n",
    "clip_value_max = 3\n",
    "clip_norm = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_inputs():\n",
    "    \n",
    "    input_data = tf.placeholder(tf.int32, [None, None], name='input')\n",
    "    targets = tf.placeholder(tf.int32, [None, None], name='targets')\n",
    "    lr = tf.placeholder(tf.float32, name='learning_rate')\n",
    "    keep_prob = tf.placeholder(tf.float32, name='keep_prob')\n",
    "    y_length = tf.placeholder(tf.int32, (None,), name='y_length')\n",
    "    max_y_length = tf.reduce_max(y_length, name='max_decoder_len')\n",
    "    X_length = tf.placeholder(tf.int32, (None,), name='X_length')\n",
    "\n",
    "    return input_data, targets, lr, keep_prob, y_length, max_y_length, X_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_decoding_input(target_data, word_to_int, batch_size):\n",
    "    \n",
    "    ending = tf.strided_slice(target_data, [0, 0], [batch_size, -1], [1, 1])\n",
    "    decoder_input = tf.concat([tf.fill([batch_size, 1], word_to_int['<GO>']), ending], axis=1)\n",
    "    return decoder_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "def encoding_layer(rnn_dim, sequence_length, num_layers, rnn_inputs, keep_prob):\n",
    "    \n",
    "    # choose the initializer to use in cells\n",
    "    if initializer_type == 1:\n",
    "        initializer = tf.random_uniform_initializer(1.0, 1.0, seed=2)\n",
    "    elif initializer_type == 2:\n",
    "        initializer = tf.truncated_normal_initializer(1.0, 1.0, seed=2)\n",
    "    else:\n",
    "        initializer = tf.orthogonal_initializer(gain=1.0, seed=2)\n",
    "    \n",
    "    # choose the cell type to use\n",
    "    if encoder_cell_type == 1:\n",
    "        tf_cell = tf.contrib.rnn.RNNCell(rnn_dim)\n",
    "    elif encoder_cell_type == 2:\n",
    "        tf_cell = tf.contrib.rnn.GRUCell(rnn_dim,\n",
    "            kernel_initializer=initializer,\n",
    "            activation=activation)\n",
    "    else:\n",
    "        tf_cell = tf.contrib.rnn.LSTMCell(rnn_dim,\n",
    "            initializer=initializer,\n",
    "            forget_bias=1.0, \n",
    "            activation=activation)\n",
    "\n",
    "    # multilayered bidirecitonal RNN\n",
    "    # https://stackoverflow.com/questions/44483560/multilayered-bi-directional-encoder-for-seq2seq-in-tensorflow\n",
    "    next_inputs = rnn_inputs\n",
    "    output_list = []\n",
    "    for layer in range(num_layers):\n",
    "        with tf.variable_scope('encoder_{}'.format(layer)):\n",
    "            cell_fw = tf_cell\n",
    "            # add dropout wrapper\n",
    "            cell_fw = tf.contrib.rnn.DropoutWrapper(cell_fw, \n",
    "                                input_keep_prob = keep_prob)\n",
    "            cell_bw = tf_cell\n",
    "            cell_bw = tf.contrib.rnn.DropoutWrapper(cell_bw, \n",
    "                                input_keep_prob = keep_prob)\n",
    "            \n",
    "            # add bidirectional wrapper\n",
    "            (encoder_output_fw, encoder_output_bw), encoder_state = tf.nn.bidirectional_dynamic_rnn(cell_fw, \n",
    "                                                                    cell_bw, \n",
    "                                                                    next_inputs,\n",
    "                                                                    sequence_length,\n",
    "                                                                    dtype=tf.float32)\n",
    "            # update the next_inputs from the output of current layer\n",
    "            next_inputs = tf.concat([encoder_output_fw,encoder_output_bw], axis=2)\n",
    "            output_list.append(next_inputs)\n",
    "            \n",
    "    # only take last one as encoder output\n",
    "    # encoder_output = next_inputs\n",
    "    # take all the outputs as encoder output\n",
    "    if num_layers == 1:\n",
    "        encoder_output = next_inputs\n",
    "    else:\n",
    "        encoder_output = tf.concat(output_list, axis=2)\n",
    "\n",
    "    return encoder_output, encoder_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the decoding layer used in training\n",
    "def training_decoding_layer(decoder_embed_input, y_length, decoder_cell, initial_state, \n",
    "                            output_layer, vocab_size, max_y_length):\n",
    "    '''Create the training logits'''\n",
    "    # 3 steps for decoding layer:\n",
    "    # helper: use argmax, go through embeddings\n",
    "    # basic decoder: decoder\n",
    "    # dynamic_decode: perform dynamic decoding with the decoder chose\n",
    "    training_helper = tf.contrib.seq2seq.TrainingHelper(inputs=decoder_embed_input,\n",
    "                                                        sequence_length=y_length,\n",
    "                                                        time_major=False)\n",
    "    # time major means:\n",
    "    # If time_major == False (default), this must be a Tensor of shape: [batch_size, max_time, ...], or a nested tuple of such elements.\n",
    "    # If time_major == True, this must be a Tensor of shape: [max_time, batch_size, ...], or a nested tuple of such elements.\n",
    "    training_decoder = tf.contrib.seq2seq.BasicDecoder(cell=decoder_cell,\n",
    "                                                       helper=training_helper,\n",
    "                                                       initial_state=initial_state,\n",
    "                                                       output_layer=output_layer) \n",
    "    \n",
    "    # impute_finished: Python boolean. If True, then states for batch entries which are marked as finished get copied through and the corresponding outputs get zeroed out. This causes some slowdown at each time step, but ensures that the final state and outputs have the correct values and that backprop ignores time steps that were marked as finished.\n",
    "    # set zero when finished and improve the performance\n",
    "    training_logits, _, _ = tf.contrib.seq2seq.dynamic_decode(decoder=training_decoder,\n",
    "                                                           output_time_major=False,\n",
    "                                                           impute_finished=True,\n",
    "                                                           maximum_iterations=max_y_length)\n",
    "    return training_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the decoding layer used in using\n",
    "def inference_decoding_layer(embeddings, start_token, end_token, decoder_cell, initial_state, \n",
    "                             output_layer, max_y_length, batch_size):\n",
    "    '''Create the inference logits'''\n",
    "    # <GO> symbol\n",
    "    start_tokens = tf.tile(tf.constant([start_token], dtype=tf.int32), [batch_size], name='start_tokens')\n",
    "    # A helper for use during inference\n",
    "    # use the argmax of the output and passes the result through an embedding layer to get the next input\n",
    "    inference_helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(embedding=embeddings,\n",
    "                                                                start_tokens=start_tokens,\n",
    "                                                                end_token=end_token)\n",
    "    # Basic sampling decoder            \n",
    "    inference_decoder = tf.contrib.seq2seq.BasicDecoder(cell=decoder_cell,\n",
    "                                                        helper=inference_helper,\n",
    "                                                        initial_state=initial_state,\n",
    "                                                        output_layer=output_layer)\n",
    "                \n",
    "    inference_logits, _, _ = tf.contrib.seq2seq.dynamic_decode(decoder=inference_decoder,\n",
    "                                                            output_time_major=False,\n",
    "                                                            impute_finished=True,\n",
    "                                                            maximum_iterations=max_y_length)\n",
    "    \n",
    "    return inference_logits\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decoding_layer(decoder_embed_input, embeddings, encoder_output, encoder_state, \n",
    "                   vocab_size, X_length, y_length, max_y_length, rnn_dim, word_to_int,\n",
    "                   keep_prob, batch_size, num_layers):\n",
    "    '''Create the decoding cell and attention for the training and inference decoding layers'''\n",
    "    # choose the initializer to use in cells\n",
    "    if initializer_type == 1:\n",
    "        initializer = tf.random_uniform_initializer(1.0, 1.0, seed=2)\n",
    "    elif initializer_type == 2:\n",
    "        initializer = tf.truncated_normal_initializer(1.0, 1.0, seed=2)\n",
    "    else:\n",
    "        initializer = tf.orthogonal_initializer(gain=1.0, seed=2)\n",
    "\n",
    "    # choose the cell type to use\n",
    "    if decoder_cell_type == 1:\n",
    "        tf_cell = tf.contrib.rnn.RNNCell(rnn_dim)\n",
    "    elif decoder_cell_type == 2:\n",
    "        tf_cell = tf.contrib.rnn.GRUCell(rnn_dim,\n",
    "            kernel_initializer=initializer,\n",
    "            activation=activation)\n",
    "    else:\n",
    "        tf_cell = tf.contrib.rnn.LSTMCell(rnn_dim,\n",
    "            initializer=initializer,\n",
    "            forget_bias=1.0, \n",
    "            activation=activation)\n",
    "\n",
    "    # create cells for decoder\n",
    "    # for layer in range(num_layers):\n",
    "    with tf.variable_scope('decoder'):\n",
    "        lstm = tf_cell\n",
    "        decoder_cell = tf.contrib.rnn.DropoutWrapper(lstm, \n",
    "                                                     input_keep_prob = keep_prob)\n",
    "\n",
    "    output_layer = Dense(vocab_size,\n",
    "                         kernel_initializer = tf.truncated_normal_initializer(mean = 0.0, stddev=0.25))\n",
    "    \n",
    "    # # implements Bahdanau-style (additive) attetion\n",
    "    if attention_type == 1:\n",
    "        attn_mech = tf.contrib.seq2seq.BahdanauAttention(rnn_dim,\n",
    "                                                         encoder_output,\n",
    "                                                         X_length,\n",
    "                                                         normalize=False,\n",
    "                                                         name='BahdanauAttention')\n",
    "    else:\n",
    "        #implements Luong-style(multiplication) attention\n",
    "        attn_mech = tf.contrib.seq2seq.LuongAttention(rnn_dim,\n",
    "                                                      encoder_output,\n",
    "                                                      X_length,\n",
    "                                                      name='LuongAttention')\n",
    "    # Wraps RNNCells with attention\n",
    "    decoder_cell = tf.contrib.seq2seq.AttentionWrapper(decoder_cell,\n",
    "                                                       attn_mech,\n",
    "                                                       rnn_dim)\n",
    "    \n",
    "    # initial_cell_state = the ending state of encoder\n",
    "    initial_state = decoder_cell.zero_state(batch_size, tf.float32).clone(cell_state=encoder_state[0])\n",
    "    \n",
    "    with tf.variable_scope(\"decode\"):\n",
    "        training_logits = training_decoding_layer(decoder_embed_input, \n",
    "                                                  y_length, \n",
    "                                                  decoder_cell, \n",
    "                                                  initial_state,\n",
    "                                                  output_layer,\n",
    "                                                  vocab_size, \n",
    "                                                  max_y_length)\n",
    "    with tf.variable_scope(\"decode\", reuse=True):\n",
    "        inference_logits = inference_decoding_layer(embeddings,  \n",
    "                                                    word_to_int['<GO>'], \n",
    "                                                    word_to_int['<EOS>'],\n",
    "                                                    decoder_cell, \n",
    "                                                    initial_state, \n",
    "                                                    output_layer,\n",
    "                                                    max_y_length,\n",
    "                                                    batch_size)\n",
    "\n",
    "    return training_logits, inference_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "def seq2seq_model(input_data, target_data, keep_prob, X_length, y_length, max_y_length, \n",
    "                  vocab_size, rnn_dim, num_layers, word_to_int, batch_size):\n",
    "    '''Use the previous functions to create the training and inference logits'''\n",
    "    # The whole flow graph of the seq2seq model\n",
    "    # embedding --> encoding layer(bidirectional lstm) --> output -->\n",
    "    # decoding layer (dynamic decoder with bahdanau style (additive) ttention)\n",
    "    \n",
    "    # Use Numberbatch's embeddings and the newly created ones as our embeddings\n",
    "    embeddings = word_embedding_matrix\n",
    "    \n",
    "    # convert the input int to vectors\n",
    "    encoder_embed_input = tf.nn.embedding_lookup(embeddings, input_data)\n",
    "    # encoding layer \n",
    "    encoder_output, encoder_state = encoding_layer(rnn_dim, \n",
    "                                                   X_length, \n",
    "                                                   num_layers, \n",
    "                                                   encoder_embed_input, \n",
    "                                                   keep_prob)\n",
    "    \n",
    "    decoder_input = process_decoding_input(target_data, word_to_int, batch_size)\n",
    "    decoder_embed_input = tf.nn.embedding_lookup(embeddings, decoder_input)\n",
    "    \n",
    "    training_logits, inference_logits  = decoding_layer(decoder_embed_input, \n",
    "                                                        embeddings,\n",
    "                                                        encoder_output,\n",
    "                                                        encoder_state, \n",
    "                                                        vocab_size, \n",
    "                                                        X_length, \n",
    "                                                        y_length, \n",
    "                                                        max_y_length,\n",
    "                                                        rnn_dim, \n",
    "                                                        word_to_int, \n",
    "                                                        keep_prob, \n",
    "                                                        batch_size,\n",
    "                                                        num_layers)\n",
    "    \n",
    "    return training_logits, inference_logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentences):\n",
    "    max_sentence = max([len(sentence) for sentence in sentences])\n",
    "    return [sentence + [word_to_int['<PAD>']] * (max_sentence - len(sentence)) for sentence in sentences]\n",
    "\n",
    "\n",
    "def reverse_sentence_batch(sentences):\n",
    "    # reverse the inputs\n",
    "    return [list(reversed(sentence)) for sentence in sentences]\n",
    "\n",
    "\n",
    "def get_batches(summaries, texts, batch_size):\n",
    "    \"\"\"Batch summaries, texts, and the lengths of their sentences together\"\"\"\n",
    "    for batch_i in range(0, len(texts)//batch_size):\n",
    "        start_i = batch_i * batch_size\n",
    "        summaries_batch = summaries[start_i:start_i + batch_size]\n",
    "        texts_batch = texts[start_i:start_i + batch_size]\n",
    "        pad_summaries_batch = np.array(pad_sentence_batch(summaries_batch))\n",
    "        pad_texts_batch = np.array(reverse_sentence_batch(pad_sentence_batch(texts_batch)))\n",
    "        \n",
    "        # Need the lengths for the _lengths parameters\n",
    "        pad_summaries_lengths = []\n",
    "        for summary in pad_summaries_batch:\n",
    "            pad_summaries_lengths.append(len(summary))\n",
    "        \n",
    "        pad_texts_lengths = []\n",
    "        for text in pad_texts_batch:\n",
    "            pad_texts_lengths.append(len(text))\n",
    "        \n",
    "        yield pad_summaries_batch, pad_texts_batch, pad_summaries_lengths, pad_texts_lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building the model\n",
      "Model is built\n"
     ]
    }
   ],
   "source": [
    "### Building the graph ###\n",
    "print \"Building the model\"\n",
    "train_graph = tf.Graph()\n",
    "\n",
    "with train_graph.as_default():\n",
    "    input_data, targets, lr, keep_prob, y_length, max_y_length, X_length = model_inputs()\n",
    "    \n",
    "    training_logits, inference_logits = seq2seq_model(tf.reverse(input_data, [-1]),\n",
    "                                                      targets,\n",
    "                                                      keep_prob,\n",
    "                                                      X_length,\n",
    "                                                      y_length,\n",
    "                                                      max_y_length,\n",
    "                                                      len(word_to_int) + 1,\n",
    "                                                      rnn_dim,\n",
    "                                                      num_layers,\n",
    "                                                      word_to_int,\n",
    "                                                      batch_size)\n",
    "    \n",
    "    training_logits = tf.identity(training_logits.rnn_output, 'logits')\n",
    "    inference_logits = tf.identity(inference_logits.sample_id, name='predictions')\n",
    "    \n",
    "    # Create the weights for sequence_loss\n",
    "    masks = tf.sequence_mask(y_length, max_y_length, dtype=tf.float32, name='mask')\n",
    "    \n",
    "    with tf.name_scope(\"optimization\"):\n",
    "        cost = tf.contrib.seq2seq.sequence_loss(training_logits, targets, masks)\n",
    "\n",
    "        # Here we can choose optimizer used in the model\n",
    "        if model_optimizer == 1:\n",
    "            optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "        elif model_optimizer == 2:    \n",
    "            optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "        else:\n",
    "            optimizer = tf.train.RMSPropOptimizer(learning_rate)\n",
    "\n",
    "        # Gradient Clipping\n",
    "        gradients = optimizer.compute_gradients(cost)\n",
    "        if model_gradient_clipping == 1:\n",
    "            capped_gradients = [(tf.clip_by_value(grad, clip_value_min, clip_value_max), var) for grad, var in gradients if grad is not None]\n",
    "        else:\n",
    "            capped_gradients = [(tf.clip_by_norm(grad, clip_norm), var) for grad, var in grfadients if grad is not None]\n",
    "        train_op = optimizer.apply_gradients(capped_gradients)\n",
    "\n",
    "print \"Model is built\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('The shortest X length:', 6)\n",
      "('The longest X length:', 7)\n"
     ]
    }
   ],
   "source": [
    "# cut the dataset to training set\n",
    "start = 200000\n",
    "end = start + 50000\n",
    "sorted_y_short = sorted_y[start:end]\n",
    "sorted_X_short = sorted_X[start:end]\n",
    "\n",
    "print(\"The shortest X length:\", len(sorted_X_short[0]))\n",
    "print(\"The longest X length:\",len(sorted_X_short[-1]))\n",
    "\n",
    "# parameters in training:\n",
    "display_step = 20 # Check training loss after every 20 batches\n",
    "stop_early = 0 \n",
    "# If the update loss does not decrease in num_to_stop consecutive update checks, stop training\n",
    "num_to_stop = 7 \n",
    "\n",
    "per_epoch = 3 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# update checking\n",
    "update_check = (len(sorted_X_short)//batch_size//per_epoch)-1\n",
    "update_loss = 0\n",
    "batch_loss = 0\n",
    "y_update_loss = [] # Record the update losses for saving improvements in the model\n",
    "\n",
    "checkpoint = \"best_model.ckpt\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch   1/100 Batch   20/1562 - Loss:  3.316, Seconds: 2.59\n",
      "Epoch   1/100 Batch   40/1562 - Loss:  2.419, Seconds: 2.06\n",
      "Epoch   1/100 Batch   60/1562 - Loss:  2.249, Seconds: 2.39\n",
      "Epoch   1/100 Batch   80/1562 - Loss:  2.211, Seconds: 3.07\n",
      "Epoch   1/100 Batch  100/1562 - Loss:  2.374, Seconds: 3.10\n",
      "Epoch   1/100 Batch  120/1562 - Loss:  2.116, Seconds: 3.04\n",
      "Epoch   1/100 Batch  140/1562 - Loss:  2.178, Seconds: 3.97\n",
      "Epoch   1/100 Batch  160/1562 - Loss:  2.118, Seconds: 2.31\n",
      "Epoch   1/100 Batch  180/1562 - Loss:  2.257, Seconds: 3.24\n",
      "Epoch   1/100 Batch  200/1562 - Loss:  2.071, Seconds: 2.53\n",
      "Epoch   1/100 Batch  220/1562 - Loss:  2.226, Seconds: 1.95\n",
      "Epoch   1/100 Batch  240/1562 - Loss:  2.152, Seconds: 2.69\n",
      "Epoch   1/100 Batch  260/1562 - Loss:  2.219, Seconds: 2.46\n",
      "Epoch   1/100 Batch  280/1562 - Loss:  1.923, Seconds: 3.63\n",
      "Epoch   1/100 Batch  300/1562 - Loss:  2.188, Seconds: 3.00\n",
      "Epoch   1/100 Batch  320/1562 - Loss:  2.248, Seconds: 2.92\n",
      "Epoch   1/100 Batch  340/1562 - Loss:  2.183, Seconds: 2.73\n",
      "Epoch   1/100 Batch  360/1562 - Loss:  2.162, Seconds: 3.81\n",
      "Epoch   1/100 Batch  380/1562 - Loss:  1.952, Seconds: 3.24\n",
      "Epoch   1/100 Batch  400/1562 - Loss:  2.195, Seconds: 3.24\n",
      "Epoch   1/100 Batch  420/1562 - Loss:  2.178, Seconds: 2.70\n",
      "Epoch   1/100 Batch  440/1562 - Loss:  2.184, Seconds: 2.35\n",
      "Epoch   1/100 Batch  460/1562 - Loss:  2.036, Seconds: 2.86\n",
      "Epoch   1/100 Batch  480/1562 - Loss:  2.111, Seconds: 2.32\n",
      "Epoch   1/100 Batch  500/1562 - Loss:  2.108, Seconds: 3.14\n",
      "('Average loss for this update:', 2.208)\n",
      "New Record!\n",
      "Epoch   1/100 Batch  520/1562 - Loss:  2.038, Seconds: 47.37\n",
      "Epoch   1/100 Batch  540/1562 - Loss:  2.110, Seconds: 3.37\n",
      "Epoch   1/100 Batch  560/1562 - Loss:  2.083, Seconds: 2.96\n",
      "Epoch   1/100 Batch  580/1562 - Loss:  2.069, Seconds: 3.08\n",
      "Epoch   1/100 Batch  600/1562 - Loss:  1.997, Seconds: 2.34\n",
      "Epoch   1/100 Batch  620/1562 - Loss:  2.010, Seconds: 2.45\n",
      "Epoch   1/100 Batch  640/1562 - Loss:  1.887, Seconds: 3.22\n",
      "Epoch   1/100 Batch  660/1562 - Loss:  1.895, Seconds: 4.48\n",
      "Epoch   1/100 Batch  680/1562 - Loss:  1.979, Seconds: 2.17\n",
      "Epoch   1/100 Batch  700/1562 - Loss:  1.920, Seconds: 3.46\n",
      "Epoch   1/100 Batch  720/1562 - Loss:  2.176, Seconds: 2.47\n",
      "Epoch   1/100 Batch  740/1562 - Loss:  1.867, Seconds: 2.70\n",
      "Epoch   1/100 Batch  760/1562 - Loss:  2.084, Seconds: 1.96\n",
      "Epoch   1/100 Batch  780/1562 - Loss:  2.220, Seconds: 4.43\n",
      "Epoch   1/100 Batch  800/1562 - Loss:  2.552, Seconds: 3.12\n",
      "Epoch   1/100 Batch  820/1562 - Loss:  2.232, Seconds: 1.86\n",
      "Epoch   1/100 Batch  840/1562 - Loss:  1.958, Seconds: 2.16\n",
      "Epoch   1/100 Batch  860/1562 - Loss:  1.988, Seconds: 2.90\n",
      "Epoch   1/100 Batch  880/1562 - Loss:  1.962, Seconds: 2.51\n",
      "Epoch   1/100 Batch  900/1562 - Loss:  1.931, Seconds: 3.33\n",
      "Epoch   1/100 Batch  920/1562 - Loss:  2.012, Seconds: 2.22\n",
      "Epoch   1/100 Batch  940/1562 - Loss:  2.007, Seconds: 2.64\n",
      "Epoch   1/100 Batch  960/1562 - Loss:  2.067, Seconds: 2.67\n",
      "Epoch   1/100 Batch  980/1562 - Loss:  1.981, Seconds: 2.56\n",
      "Epoch   1/100 Batch 1000/1562 - Loss:  1.849, Seconds: 2.89\n",
      "Epoch   1/100 Batch 1020/1562 - Loss:  1.922, Seconds: 2.88\n",
      "('Average loss for this update:', 2.031)\n",
      "New Record!\n",
      "Epoch   1/100 Batch 1040/1562 - Loss:  2.071, Seconds: 2.67\n",
      "Epoch   1/100 Batch 1060/1562 - Loss:  1.867, Seconds: 3.65\n",
      "Epoch   1/100 Batch 1080/1562 - Loss:  1.943, Seconds: 2.35\n",
      "Epoch   1/100 Batch 1100/1562 - Loss:  1.979, Seconds: 2.99\n",
      "Epoch   1/100 Batch 1120/1562 - Loss:  1.813, Seconds: 2.16\n",
      "Epoch   1/100 Batch 1140/1562 - Loss:  1.955, Seconds: 2.90\n",
      "Epoch   1/100 Batch 1160/1562 - Loss:  2.011, Seconds: 2.89\n",
      "Epoch   1/100 Batch 1180/1562 - Loss:  1.949, Seconds: 2.27\n",
      "Epoch   1/100 Batch 1200/1562 - Loss:  1.758, Seconds: 3.88\n",
      "Epoch   1/100 Batch 1220/1562 - Loss:  1.865, Seconds: 2.88\n",
      "Epoch   1/100 Batch 1240/1562 - Loss:  1.897, Seconds: 2.25\n",
      "Epoch   1/100 Batch 1260/1562 - Loss:  1.974, Seconds: 2.63\n",
      "Epoch   1/100 Batch 1280/1562 - Loss:  2.096, Seconds: 3.00\n",
      "Epoch   1/100 Batch 1300/1562 - Loss:  2.094, Seconds: 2.78\n",
      "Epoch   1/100 Batch 1320/1562 - Loss:  2.015, Seconds: 2.35\n",
      "Epoch   1/100 Batch 1340/1562 - Loss:  1.980, Seconds: 3.34\n",
      "Epoch   1/100 Batch 1360/1562 - Loss:  1.949, Seconds: 2.51\n",
      "Epoch   1/100 Batch 1380/1562 - Loss:  1.857, Seconds: 2.40\n",
      "Epoch   1/100 Batch 1400/1562 - Loss:  1.874, Seconds: 3.09\n",
      "Epoch   1/100 Batch 1420/1562 - Loss:  1.909, Seconds: 2.70\n",
      "Epoch   1/100 Batch 1440/1562 - Loss:  1.922, Seconds: 3.19\n",
      "Epoch   1/100 Batch 1460/1562 - Loss:  1.965, Seconds: 2.54\n",
      "Epoch   1/100 Batch 1480/1562 - Loss:  2.050, Seconds: 2.88\n",
      "Epoch   1/100 Batch 1500/1562 - Loss:  2.031, Seconds: 2.43\n",
      "Epoch   1/100 Batch 1520/1562 - Loss:  1.996, Seconds: 2.63\n",
      "Epoch   1/100 Batch 1540/1562 - Loss:  1.910, Seconds: 2.49\n",
      "('Average loss for this update:', 1.949)\n",
      "New Record!\n",
      "Epoch   1/100 Batch 1560/1562 - Loss:  1.971, Seconds: 2.66\n",
      "Epoch   2/100 Batch   20/1562 - Loss:  1.836, Seconds: 2.60\n",
      "Epoch   2/100 Batch   40/1562 - Loss:  1.888, Seconds: 2.06\n",
      "Epoch   2/100 Batch   60/1562 - Loss:  1.856, Seconds: 2.42\n",
      "Epoch   2/100 Batch   80/1562 - Loss:  1.863, Seconds: 3.05\n",
      "Epoch   2/100 Batch  100/1562 - Loss:  2.014, Seconds: 3.13\n",
      "Epoch   2/100 Batch  120/1562 - Loss:  1.808, Seconds: 3.08\n",
      "Epoch   2/100 Batch  140/1562 - Loss:  1.877, Seconds: 4.09\n",
      "Epoch   2/100 Batch  160/1562 - Loss:  1.823, Seconds: 2.58\n",
      "Epoch   2/100 Batch  180/1562 - Loss:  1.952, Seconds: 3.41\n",
      "Epoch   2/100 Batch  200/1562 - Loss:  1.803, Seconds: 2.54\n",
      "Epoch   2/100 Batch  220/1562 - Loss:  1.939, Seconds: 1.95\n",
      "Epoch   2/100 Batch  240/1562 - Loss:  1.897, Seconds: 2.70\n",
      "Epoch   2/100 Batch  260/1562 - Loss:  1.949, Seconds: 2.45\n",
      "Epoch   2/100 Batch  280/1562 - Loss:  1.688, Seconds: 3.62\n",
      "Epoch   2/100 Batch  300/1562 - Loss:  1.938, Seconds: 3.00\n",
      "Epoch   2/100 Batch  320/1562 - Loss:  1.998, Seconds: 2.87\n",
      "Epoch   2/100 Batch  340/1562 - Loss:  1.927, Seconds: 2.73\n",
      "Epoch   2/100 Batch  360/1562 - Loss:  1.923, Seconds: 3.83\n",
      "Epoch   2/100 Batch  380/1562 - Loss:  1.742, Seconds: 3.24\n",
      "Epoch   2/100 Batch  400/1562 - Loss:  1.974, Seconds: 3.27\n",
      "Epoch   2/100 Batch  420/1562 - Loss:  1.961, Seconds: 2.70\n",
      "Epoch   2/100 Batch  440/1562 - Loss:  1.972, Seconds: 2.33\n",
      "Epoch   2/100 Batch  460/1562 - Loss:  1.837, Seconds: 2.93\n",
      "Epoch   2/100 Batch  480/1562 - Loss:  1.915, Seconds: 2.36\n",
      "Epoch   2/100 Batch  500/1562 - Loss:  1.907, Seconds: 3.27\n",
      "('Average loss for this update:', 1.89)\n",
      "New Record!\n",
      "Epoch   2/100 Batch  520/1562 - Loss:  1.858, Seconds: 2.60\n",
      "Epoch   2/100 Batch  540/1562 - Loss:  1.917, Seconds: 3.35\n",
      "Epoch   2/100 Batch  560/1562 - Loss:  1.907, Seconds: 3.01\n",
      "Epoch   2/100 Batch  580/1562 - Loss:  1.898, Seconds: 3.04\n",
      "Epoch   2/100 Batch  600/1562 - Loss:  1.830, Seconds: 2.33\n",
      "Epoch   2/100 Batch  620/1562 - Loss:  1.842, Seconds: 2.45\n",
      "Epoch   2/100 Batch  640/1562 - Loss:  1.740, Seconds: 3.25\n",
      "Epoch   2/100 Batch  660/1562 - Loss:  1.750, Seconds: 4.48\n",
      "Epoch   2/100 Batch  680/1562 - Loss:  1.836, Seconds: 2.17\n",
      "Epoch   2/100 Batch  700/1562 - Loss:  1.784, Seconds: 3.45\n",
      "Epoch   2/100 Batch  720/1562 - Loss:  2.010, Seconds: 2.41\n",
      "Epoch   2/100 Batch  740/1562 - Loss:  1.736, Seconds: 2.72\n",
      "Epoch   2/100 Batch  760/1562 - Loss:  1.939, Seconds: 1.94\n",
      "Epoch   2/100 Batch  780/1562 - Loss:  1.836, Seconds: 4.47\n",
      "Epoch   2/100 Batch  800/1562 - Loss:  1.900, Seconds: 3.06\n",
      "Epoch   2/100 Batch  820/1562 - Loss:  1.975, Seconds: 1.89\n",
      "Epoch   2/100 Batch  840/1562 - Loss:  1.791, Seconds: 2.13\n",
      "Epoch   2/100 Batch  860/1562 - Loss:  1.821, Seconds: 2.89\n",
      "Epoch   2/100 Batch  880/1562 - Loss:  1.812, Seconds: 2.56\n",
      "Epoch   2/100 Batch  900/1562 - Loss:  1.789, Seconds: 3.34\n",
      "Epoch   2/100 Batch  920/1562 - Loss:  1.864, Seconds: 2.23\n",
      "Epoch   2/100 Batch  940/1562 - Loss:  1.872, Seconds: 2.57\n",
      "Epoch   2/100 Batch  960/1562 - Loss:  1.919, Seconds: 2.60\n",
      "Epoch   2/100 Batch  980/1562 - Loss:  1.839, Seconds: 2.52\n",
      "Epoch   2/100 Batch 1000/1562 - Loss:  1.727, Seconds: 2.92\n",
      "Epoch   2/100 Batch 1020/1562 - Loss:  1.799, Seconds: 2.90\n",
      "('Average loss for this update:', 1.848)\n",
      "New Record!\n",
      "Epoch   2/100 Batch 1040/1562 - Loss:  1.933, Seconds: 2.69\n",
      "Epoch   2/100 Batch 1060/1562 - Loss:  1.748, Seconds: 3.67\n",
      "Epoch   2/100 Batch 1080/1562 - Loss:  1.828, Seconds: 2.32\n",
      "Epoch   2/100 Batch 1100/1562 - Loss:  1.856, Seconds: 3.07\n",
      "Epoch   2/100 Batch 1120/1562 - Loss:  1.708, Seconds: 2.18\n",
      "Epoch   2/100 Batch 1140/1562 - Loss:  1.843, Seconds: 2.88\n",
      "Epoch   2/100 Batch 1160/1562 - Loss:  1.886, Seconds: 2.90\n",
      "Epoch   2/100 Batch 1180/1562 - Loss:  1.845, Seconds: 2.28\n",
      "Epoch   2/100 Batch 1200/1562 - Loss:  1.657, Seconds: 3.92\n",
      "Epoch   2/100 Batch 1220/1562 - Loss:  1.760, Seconds: 2.87\n",
      "Epoch   2/100 Batch 1240/1562 - Loss:  1.788, Seconds: 2.24\n",
      "Epoch   2/100 Batch 1260/1562 - Loss:  1.866, Seconds: 2.68\n",
      "Epoch   2/100 Batch 1280/1562 - Loss:  1.985, Seconds: 3.03\n",
      "Epoch   2/100 Batch 1300/1562 - Loss:  1.977, Seconds: 2.76\n",
      "Epoch   2/100 Batch 1320/1562 - Loss:  1.901, Seconds: 2.37\n",
      "Epoch   2/100 Batch 1340/1562 - Loss:  1.871, Seconds: 3.38\n",
      "Epoch   2/100 Batch 1360/1562 - Loss:  1.848, Seconds: 2.52\n",
      "Epoch   2/100 Batch 1380/1562 - Loss:  1.764, Seconds: 2.43\n",
      "Epoch   2/100 Batch 1400/1562 - Loss:  1.773, Seconds: 3.07\n",
      "Epoch   2/100 Batch 1420/1562 - Loss:  1.808, Seconds: 2.70\n",
      "Epoch   2/100 Batch 1440/1562 - Loss:  1.822, Seconds: 3.16\n",
      "Epoch   2/100 Batch 1460/1562 - Loss:  1.859, Seconds: 2.54\n",
      "Epoch   2/100 Batch 1480/1562 - Loss:  1.944, Seconds: 2.90\n",
      "Epoch   2/100 Batch 1500/1562 - Loss:  1.925, Seconds: 2.34\n",
      "Epoch   2/100 Batch 1520/1562 - Loss:  1.900, Seconds: 2.63\n",
      "Epoch   2/100 Batch 1540/1562 - Loss:  1.815, Seconds: 2.46\n",
      "('Average loss for this update:', 1.842)\n",
      "New Record!\n",
      "Epoch   2/100 Batch 1560/1562 - Loss:  1.870, Seconds: 2.65\n",
      "Epoch   3/100 Batch   20/1562 - Loss:  1.767, Seconds: 2.63\n",
      "Epoch   3/100 Batch   40/1562 - Loss:  1.815, Seconds: 2.05\n",
      "Epoch   3/100 Batch   60/1562 - Loss:  1.778, Seconds: 2.44\n",
      "Epoch   3/100 Batch   80/1562 - Loss:  1.784, Seconds: 3.09\n",
      "Epoch   3/100 Batch  100/1562 - Loss:  1.925, Seconds: 3.08\n",
      "Epoch   3/100 Batch  120/1562 - Loss:  1.727, Seconds: 3.07\n",
      "Epoch   3/100 Batch  140/1562 - Loss:  1.799, Seconds: 4.02\n",
      "Epoch   3/100 Batch  160/1562 - Loss:  1.747, Seconds: 2.32\n",
      "Epoch   3/100 Batch  180/1562 - Loss:  1.867, Seconds: 3.52\n",
      "Epoch   3/100 Batch  200/1562 - Loss:  1.732, Seconds: 2.51\n",
      "Epoch   3/100 Batch  220/1562 - Loss:  1.858, Seconds: 1.96\n",
      "Epoch   3/100 Batch  240/1562 - Loss:  1.822, Seconds: 2.70\n",
      "Epoch   3/100 Batch  260/1562 - Loss:  1.865, Seconds: 2.43\n",
      "Epoch   3/100 Batch  280/1562 - Loss:  1.618, Seconds: 3.65\n",
      "Epoch   3/100 Batch  300/1562 - Loss:  1.854, Seconds: 2.98\n",
      "Epoch   3/100 Batch  320/1562 - Loss:  1.920, Seconds: 2.87\n",
      "Epoch   3/100 Batch  340/1562 - Loss:  1.851, Seconds: 2.71\n",
      "Epoch   3/100 Batch  360/1562 - Loss:  1.837, Seconds: 3.82\n",
      "Epoch   3/100 Batch  380/1562 - Loss:  1.673, Seconds: 3.26\n",
      "Epoch   3/100 Batch  400/1562 - Loss:  1.895, Seconds: 3.28\n",
      "Epoch   3/100 Batch  420/1562 - Loss:  1.880, Seconds: 2.70\n",
      "Epoch   3/100 Batch  440/1562 - Loss:  1.893, Seconds: 2.33\n",
      "Epoch   3/100 Batch  460/1562 - Loss:  1.763, Seconds: 2.89\n",
      "Epoch   3/100 Batch  480/1562 - Loss:  1.841, Seconds: 2.33\n",
      "Epoch   3/100 Batch  500/1562 - Loss:  1.825, Seconds: 3.18\n",
      "('Average loss for this update:', 1.812)\n",
      "New Record!\n",
      "Epoch   3/100 Batch  520/1562 - Loss:  1.785, Seconds: 7.20\n",
      "Epoch   3/100 Batch  540/1562 - Loss:  1.834, Seconds: 3.42\n",
      "Epoch   3/100 Batch  560/1562 - Loss:  1.830, Seconds: 2.99\n",
      "Epoch   3/100 Batch  580/1562 - Loss:  1.820, Seconds: 3.08\n",
      "Epoch   3/100 Batch  600/1562 - Loss:  1.759, Seconds: 2.32\n",
      "Epoch   3/100 Batch  620/1562 - Loss:  1.772, Seconds: 2.45\n",
      "Epoch   3/100 Batch  640/1562 - Loss:  1.675, Seconds: 3.26\n",
      "Epoch   3/100 Batch  660/1562 - Loss:  1.682, Seconds: 4.50\n",
      "Epoch   3/100 Batch  680/1562 - Loss:  1.766, Seconds: 2.14\n",
      "Epoch   3/100 Batch  700/1562 - Loss:  1.718, Seconds: 3.44\n",
      "Epoch   3/100 Batch  720/1562 - Loss:  1.924, Seconds: 2.41\n",
      "Epoch   3/100 Batch  740/1562 - Loss:  1.676, Seconds: 2.72\n",
      "Epoch   3/100 Batch  760/1562 - Loss:  1.859, Seconds: 1.94\n",
      "Epoch   3/100 Batch  780/1562 - Loss:  1.754, Seconds: 4.44\n",
      "Epoch   3/100 Batch  800/1562 - Loss:  1.826, Seconds: 3.06\n",
      "Epoch   3/100 Batch  820/1562 - Loss:  1.896, Seconds: 1.86\n",
      "Epoch   3/100 Batch  840/1562 - Loss:  1.723, Seconds: 2.14\n",
      "Epoch   3/100 Batch  860/1562 - Loss:  1.749, Seconds: 2.90\n",
      "Epoch   3/100 Batch  880/1562 - Loss:  1.745, Seconds: 2.51\n",
      "Epoch   3/100 Batch  900/1562 - Loss:  1.717, Seconds: 3.37\n",
      "Epoch   3/100 Batch  920/1562 - Loss:  1.788, Seconds: 2.26\n",
      "Epoch   3/100 Batch  940/1562 - Loss:  1.802, Seconds: 2.65\n",
      "Epoch   3/100 Batch  960/1562 - Loss:  1.842, Seconds: 2.61\n",
      "Epoch   3/100 Batch  980/1562 - Loss:  1.767, Seconds: 2.50\n",
      "Epoch   3/100 Batch 1000/1562 - Loss:  1.661, Seconds: 2.93\n",
      "Epoch   3/100 Batch 1020/1562 - Loss:  1.730, Seconds: 2.89\n",
      "('Average loss for this update:', 1.775)\n",
      "New Record!\n",
      "Epoch   3/100 Batch 1040/1562 - Loss:  1.857, Seconds: 2.69\n",
      "Epoch   3/100 Batch 1060/1562 - Loss:  1.681, Seconds: 3.64\n",
      "Epoch   3/100 Batch 1080/1562 - Loss:  1.756, Seconds: 2.32\n",
      "Epoch   3/100 Batch 1100/1562 - Loss:  1.782, Seconds: 2.99\n",
      "Epoch   3/100 Batch 1120/1562 - Loss:  1.643, Seconds: 2.16\n",
      "Epoch   3/100 Batch 1140/1562 - Loss:  1.776, Seconds: 2.87\n",
      "Epoch   3/100 Batch 1160/1562 - Loss:  1.812, Seconds: 2.86\n",
      "Epoch   3/100 Batch 1180/1562 - Loss:  1.779, Seconds: 2.18\n",
      "Epoch   3/100 Batch 1200/1562 - Loss:  1.602, Seconds: 3.88\n",
      "Epoch   3/100 Batch 1220/1562 - Loss:  1.696, Seconds: 2.90\n",
      "Epoch   3/100 Batch 1240/1562 - Loss:  1.723, Seconds: 2.27\n",
      "Epoch   3/100 Batch 1260/1562 - Loss:  1.793, Seconds: 2.67\n",
      "Epoch   3/100 Batch 1280/1562 - Loss:  1.909, Seconds: 3.03\n",
      "Epoch   3/100 Batch 1300/1562 - Loss:  1.901, Seconds: 2.69\n",
      "Epoch   3/100 Batch 1320/1562 - Loss:  1.830, Seconds: 2.38\n",
      "Epoch   3/100 Batch 1340/1562 - Loss:  1.800, Seconds: 3.34\n",
      "Epoch   3/100 Batch 1360/1562 - Loss:  1.774, Seconds: 2.52\n",
      "Epoch   3/100 Batch 1380/1562 - Loss:  1.700, Seconds: 2.43\n",
      "Epoch   3/100 Batch 1400/1562 - Loss:  1.704, Seconds: 3.15\n",
      "Epoch   3/100 Batch 1420/1562 - Loss:  1.744, Seconds: 2.71\n",
      "Epoch   3/100 Batch 1440/1562 - Loss:  1.755, Seconds: 3.22\n",
      "Epoch   3/100 Batch 1460/1562 - Loss:  1.784, Seconds: 2.55\n",
      "Epoch   3/100 Batch 1480/1562 - Loss:  1.868, Seconds: 2.88\n",
      "Epoch   3/100 Batch 1500/1562 - Loss:  1.851, Seconds: 2.34\n",
      "Epoch   3/100 Batch 1520/1562 - Loss:  1.829, Seconds: 2.72\n",
      "Epoch   3/100 Batch 1540/1562 - Loss:  1.745, Seconds: 2.44\n",
      "('Average loss for this update:', 1.772)\n",
      "New Record!\n",
      "Epoch   3/100 Batch 1560/1562 - Loss:  1.801, Seconds: 2.70\n",
      "Epoch   4/100 Batch   20/1562 - Loss:  1.708, Seconds: 2.67\n",
      "Epoch   4/100 Batch   40/1562 - Loss:  1.746, Seconds: 2.02\n",
      "Epoch   4/100 Batch   60/1562 - Loss:  1.702, Seconds: 2.41\n",
      "Epoch   4/100 Batch   80/1562 - Loss:  1.716, Seconds: 3.07\n",
      "Epoch   4/100 Batch  100/1562 - Loss:  1.840, Seconds: 3.12\n",
      "Epoch   4/100 Batch  120/1562 - Loss:  1.663, Seconds: 3.07\n",
      "Epoch   4/100 Batch  140/1562 - Loss:  1.730, Seconds: 3.99\n",
      "Epoch   4/100 Batch  160/1562 - Loss:  1.679, Seconds: 2.45\n",
      "Epoch   4/100 Batch  180/1562 - Loss:  1.785, Seconds: 3.29\n",
      "Epoch   4/100 Batch  200/1562 - Loss:  1.665, Seconds: 2.52\n",
      "Epoch   4/100 Batch  220/1562 - Loss:  1.789, Seconds: 1.97\n",
      "Epoch   4/100 Batch  240/1562 - Loss:  1.750, Seconds: 2.70\n",
      "Epoch   4/100 Batch  260/1562 - Loss:  1.799, Seconds: 2.45\n",
      "Epoch   4/100 Batch  280/1562 - Loss:  1.555, Seconds: 3.63\n",
      "Epoch   4/100 Batch  300/1562 - Loss:  1.783, Seconds: 2.98\n",
      "Epoch   4/100 Batch  320/1562 - Loss:  1.845, Seconds: 2.90\n",
      "Epoch   4/100 Batch  340/1562 - Loss:  1.775, Seconds: 2.70\n",
      "Epoch   4/100 Batch  360/1562 - Loss:  1.761, Seconds: 3.83\n",
      "Epoch   4/100 Batch  380/1562 - Loss:  1.613, Seconds: 3.24\n",
      "Epoch   4/100 Batch  400/1562 - Loss:  1.826, Seconds: 3.26\n",
      "Epoch   4/100 Batch  420/1562 - Loss:  1.810, Seconds: 2.68\n",
      "Epoch   4/100 Batch  440/1562 - Loss:  1.822, Seconds: 2.33\n",
      "Epoch   4/100 Batch  460/1562 - Loss:  1.696, Seconds: 2.89\n",
      "Epoch   4/100 Batch  480/1562 - Loss:  1.771, Seconds: 2.31\n",
      "Epoch   4/100 Batch  500/1562 - Loss:  1.756, Seconds: 3.15\n",
      "('Average loss for this update:', 1.742)\n",
      "New Record!\n",
      "Epoch   4/100 Batch  520/1562 - Loss:  1.725, Seconds: 2.86\n",
      "Epoch   4/100 Batch  540/1562 - Loss:  1.768, Seconds: 3.37\n",
      "Epoch   4/100 Batch  560/1562 - Loss:  1.758, Seconds: 2.99\n",
      "Epoch   4/100 Batch  580/1562 - Loss:  1.750, Seconds: 3.07\n",
      "Epoch   4/100 Batch  600/1562 - Loss:  1.688, Seconds: 2.33\n",
      "Epoch   4/100 Batch  620/1562 - Loss:  1.707, Seconds: 2.41\n",
      "Epoch   4/100 Batch  640/1562 - Loss:  1.617, Seconds: 3.24\n",
      "Epoch   4/100 Batch  660/1562 - Loss:  1.624, Seconds: 4.50\n",
      "Epoch   4/100 Batch  680/1562 - Loss:  1.705, Seconds: 2.14\n",
      "Epoch   4/100 Batch  700/1562 - Loss:  1.657, Seconds: 3.47\n",
      "Epoch   4/100 Batch  720/1562 - Loss:  1.843, Seconds: 2.43\n",
      "Epoch   4/100 Batch  740/1562 - Loss:  1.625, Seconds: 2.70\n",
      "Epoch   4/100 Batch  760/1562 - Loss:  1.780, Seconds: 1.98\n",
      "Epoch   4/100 Batch  780/1562 - Loss:  1.680, Seconds: 4.46\n",
      "Epoch   4/100 Batch  800/1562 - Loss:  1.755, Seconds: 3.12\n",
      "Epoch   4/100 Batch  820/1562 - Loss:  1.823, Seconds: 1.91\n",
      "Epoch   4/100 Batch  840/1562 - Loss:  1.662, Seconds: 2.16\n",
      "Epoch   4/100 Batch  860/1562 - Loss:  1.686, Seconds: 2.92\n",
      "Epoch   4/100 Batch  880/1562 - Loss:  1.685, Seconds: 2.52\n",
      "Epoch   4/100 Batch  900/1562 - Loss:  1.658, Seconds: 3.37\n",
      "Epoch   4/100 Batch  920/1562 - Loss:  1.727, Seconds: 2.23\n",
      "Epoch   4/100 Batch  940/1562 - Loss:  1.738, Seconds: 2.64\n",
      "Epoch   4/100 Batch  960/1562 - Loss:  1.774, Seconds: 2.61\n",
      "Epoch   4/100 Batch  980/1562 - Loss:  1.703, Seconds: 2.49\n",
      "Epoch   4/100 Batch 1000/1562 - Loss:  1.602, Seconds: 2.89\n",
      "Epoch   4/100 Batch 1020/1562 - Loss:  1.663, Seconds: 2.89\n",
      "('Average loss for this update:', 1.71)\n",
      "New Record!\n",
      "Epoch   4/100 Batch 1040/1562 - Loss:  1.786, Seconds: 2.66\n",
      "Epoch   4/100 Batch 1060/1562 - Loss:  1.623, Seconds: 3.66\n",
      "Epoch   4/100 Batch 1080/1562 - Loss:  1.696, Seconds: 2.32\n",
      "Epoch   4/100 Batch 1100/1562 - Loss:  1.710, Seconds: 2.98\n",
      "Epoch   4/100 Batch 1120/1562 - Loss:  1.590, Seconds: 2.14\n",
      "Epoch   4/100 Batch 1140/1562 - Loss:  1.710, Seconds: 2.85\n",
      "Epoch   4/100 Batch 1160/1562 - Loss:  1.745, Seconds: 2.90\n",
      "Epoch   4/100 Batch 1180/1562 - Loss:  1.719, Seconds: 2.13\n",
      "Epoch   4/100 Batch 1200/1562 - Loss:  1.552, Seconds: 4.79\n",
      "Epoch   4/100 Batch 1220/1562 - Loss:  1.637, Seconds: 2.87\n",
      "Epoch   4/100 Batch 1240/1562 - Loss:  1.665, Seconds: 2.23\n",
      "Epoch   4/100 Batch 1260/1562 - Loss:  1.734, Seconds: 2.61\n",
      "Epoch   4/100 Batch 1280/1562 - Loss:  1.844, Seconds: 3.01\n",
      "Epoch   4/100 Batch 1300/1562 - Loss:  1.834, Seconds: 2.69\n",
      "Epoch   4/100 Batch 1320/1562 - Loss:  1.757, Seconds: 2.31\n",
      "Epoch   4/100 Batch 1340/1562 - Loss:  1.732, Seconds: 3.39\n",
      "Epoch   4/100 Batch 1360/1562 - Loss:  1.713, Seconds: 2.51\n",
      "Epoch   4/100 Batch 1380/1562 - Loss:  1.639, Seconds: 2.46\n",
      "Epoch   4/100 Batch 1400/1562 - Loss:  1.645, Seconds: 3.09\n",
      "Epoch   4/100 Batch 1420/1562 - Loss:  1.682, Seconds: 2.71\n",
      "Epoch   4/100 Batch 1440/1562 - Loss:  1.693, Seconds: 3.18\n",
      "Epoch   4/100 Batch 1460/1562 - Loss:  1.712, Seconds: 2.52\n",
      "Epoch   4/100 Batch 1480/1562 - Loss:  1.796, Seconds: 2.92\n",
      "Epoch   4/100 Batch 1500/1562 - Loss:  1.783, Seconds: 2.34\n",
      "Epoch   4/100 Batch 1520/1562 - Loss:  1.767, Seconds: 2.59\n",
      "Epoch   4/100 Batch 1540/1562 - Loss:  1.682, Seconds: 2.43\n",
      "('Average loss for this update:', 1.709)\n",
      "New Record!\n",
      "Epoch   4/100 Batch 1560/1562 - Loss:  1.732, Seconds: 2.70\n",
      "Epoch   5/100 Batch   20/1562 - Loss:  1.650, Seconds: 2.59\n",
      "Epoch   5/100 Batch   40/1562 - Loss:  1.684, Seconds: 2.05\n",
      "Epoch   5/100 Batch   60/1562 - Loss:  1.637, Seconds: 2.43\n",
      "Epoch   5/100 Batch   80/1562 - Loss:  1.651, Seconds: 3.03\n",
      "Epoch   5/100 Batch  100/1562 - Loss:  1.768, Seconds: 3.18\n",
      "Epoch   5/100 Batch  120/1562 - Loss:  1.596, Seconds: 3.12\n",
      "Epoch   5/100 Batch  140/1562 - Loss:  1.667, Seconds: 4.08\n",
      "Epoch   5/100 Batch  160/1562 - Loss:  1.613, Seconds: 2.34\n",
      "Epoch   5/100 Batch  180/1562 - Loss:  1.706, Seconds: 3.28\n",
      "Epoch   5/100 Batch  200/1562 - Loss:  1.604, Seconds: 2.49\n",
      "Epoch   5/100 Batch  220/1562 - Loss:  1.716, Seconds: 1.97\n",
      "Epoch   5/100 Batch  240/1562 - Loss:  1.680, Seconds: 2.68\n",
      "Epoch   5/100 Batch  260/1562 - Loss:  1.722, Seconds: 2.41\n",
      "Epoch   5/100 Batch  280/1562 - Loss:  1.498, Seconds: 3.65\n",
      "Epoch   5/100 Batch  300/1562 - Loss:  1.711, Seconds: 2.93\n",
      "Epoch   5/100 Batch  320/1562 - Loss:  1.768, Seconds: 2.88\n",
      "Epoch   5/100 Batch  340/1562 - Loss:  1.698, Seconds: 2.69\n",
      "Epoch   5/100 Batch  360/1562 - Loss:  1.680, Seconds: 3.80\n",
      "Epoch   5/100 Batch  380/1562 - Loss:  1.557, Seconds: 3.27\n",
      "Epoch   5/100 Batch  400/1562 - Loss:  1.757, Seconds: 3.23\n",
      "Epoch   5/100 Batch  420/1562 - Loss:  1.729, Seconds: 3.67\n",
      "Epoch   5/100 Batch  440/1562 - Loss:  1.737, Seconds: 2.33\n",
      "Epoch   5/100 Batch  460/1562 - Loss:  1.636, Seconds: 2.89\n",
      "Epoch   5/100 Batch  480/1562 - Loss:  1.698, Seconds: 2.32\n",
      "Epoch   5/100 Batch  500/1562 - Loss:  1.682, Seconds: 3.15\n",
      "('Average loss for this update:', 1.673)\n",
      "New Record!\n",
      "Epoch   5/100 Batch  520/1562 - Loss:  1.657, Seconds: 2.67\n",
      "Epoch   5/100 Batch  540/1562 - Loss:  1.688, Seconds: 3.32\n",
      "Epoch   5/100 Batch  560/1562 - Loss:  1.690, Seconds: 2.99\n",
      "Epoch   5/100 Batch  580/1562 - Loss:  1.675, Seconds: 3.09\n",
      "Epoch   5/100 Batch  600/1562 - Loss:  1.621, Seconds: 2.32\n",
      "Epoch   5/100 Batch  620/1562 - Loss:  1.647, Seconds: 2.46\n",
      "Epoch   5/100 Batch  640/1562 - Loss:  1.557, Seconds: 3.25\n",
      "Epoch   5/100 Batch  660/1562 - Loss:  1.562, Seconds: 4.51\n",
      "Epoch   5/100 Batch  680/1562 - Loss:  1.637, Seconds: 2.32\n",
      "Epoch   5/100 Batch  700/1562 - Loss:  1.600, Seconds: 3.45\n",
      "Epoch   5/100 Batch  720/1562 - Loss:  1.757, Seconds: 2.45\n",
      "Epoch   5/100 Batch  740/1562 - Loss:  1.563, Seconds: 2.70\n",
      "Epoch   5/100 Batch  760/1562 - Loss:  1.705, Seconds: 1.96\n",
      "Epoch   5/100 Batch  780/1562 - Loss:  1.610, Seconds: 4.62\n",
      "Epoch   5/100 Batch  800/1562 - Loss:  1.689, Seconds: 3.06\n",
      "Epoch   5/100 Batch  820/1562 - Loss:  1.750, Seconds: 1.86\n",
      "Epoch   5/100 Batch  840/1562 - Loss:  1.605, Seconds: 2.14\n",
      "Epoch   5/100 Batch  860/1562 - Loss:  1.620, Seconds: 2.88\n",
      "Epoch   5/100 Batch  880/1562 - Loss:  1.617, Seconds: 2.52\n",
      "Epoch   5/100 Batch  900/1562 - Loss:  1.596, Seconds: 3.36\n",
      "Epoch   5/100 Batch  920/1562 - Loss:  1.660, Seconds: 2.22\n",
      "Epoch   5/100 Batch  940/1562 - Loss:  1.666, Seconds: 2.66\n",
      "Epoch   5/100 Batch  960/1562 - Loss:  1.701, Seconds: 2.61\n",
      "Epoch   5/100 Batch  980/1562 - Loss:  1.636, Seconds: 2.51\n",
      "Epoch   5/100 Batch 1000/1562 - Loss:  1.542, Seconds: 2.90\n",
      "Epoch   5/100 Batch 1020/1562 - Loss:  1.599, Seconds: 3.62\n",
      "('Average loss for this update:', 1.642)\n",
      "New Record!\n",
      "Epoch   5/100 Batch 1040/1562 - Loss:  1.712, Seconds: 2.84\n",
      "Epoch   5/100 Batch 1060/1562 - Loss:  1.556, Seconds: 3.58\n",
      "Epoch   5/100 Batch 1080/1562 - Loss:  1.628, Seconds: 2.36\n",
      "Epoch   5/100 Batch 1100/1562 - Loss:  1.639, Seconds: 3.01\n",
      "Epoch   5/100 Batch 1120/1562 - Loss:  1.533, Seconds: 2.16\n",
      "Epoch   5/100 Batch 1140/1562 - Loss:  1.646, Seconds: 2.92\n",
      "Epoch   5/100 Batch 1160/1562 - Loss:  1.684, Seconds: 2.87\n",
      "Epoch   5/100 Batch 1180/1562 - Loss:  1.664, Seconds: 2.21\n",
      "Epoch   5/100 Batch 1200/1562 - Loss:  1.500, Seconds: 3.98\n",
      "Epoch   5/100 Batch 1220/1562 - Loss:  1.575, Seconds: 2.89\n",
      "Epoch   5/100 Batch 1240/1562 - Loss:  1.599, Seconds: 2.26\n",
      "Epoch   5/100 Batch 1260/1562 - Loss:  1.665, Seconds: 2.60\n",
      "Epoch   5/100 Batch 1280/1562 - Loss:  1.771, Seconds: 2.99\n",
      "Epoch   5/100 Batch 1300/1562 - Loss:  1.758, Seconds: 2.70\n",
      "Epoch   5/100 Batch 1320/1562 - Loss:  1.689, Seconds: 2.37\n",
      "Epoch   5/100 Batch 1340/1562 - Loss:  1.662, Seconds: 3.39\n",
      "Epoch   5/100 Batch 1360/1562 - Loss:  1.648, Seconds: 2.73\n",
      "Epoch   5/100 Batch 1380/1562 - Loss:  1.576, Seconds: 2.44\n",
      "Epoch   5/100 Batch 1400/1562 - Loss:  1.576, Seconds: 3.07\n",
      "Epoch   5/100 Batch 1420/1562 - Loss:  1.621, Seconds: 2.70\n",
      "Epoch   5/100 Batch 1440/1562 - Loss:  1.626, Seconds: 3.16\n",
      "Epoch   5/100 Batch 1460/1562 - Loss:  1.647, Seconds: 2.54\n",
      "Epoch   5/100 Batch 1480/1562 - Loss:  1.723, Seconds: 2.98\n",
      "Epoch   5/100 Batch 1500/1562 - Loss:  1.714, Seconds: 2.33\n",
      "Epoch   5/100 Batch 1520/1562 - Loss:  1.700, Seconds: 2.66\n",
      "Epoch   5/100 Batch 1540/1562 - Loss:  1.610, Seconds: 2.47\n",
      "('Average loss for this update:', 1.642)\n",
      "No Improvement.\n",
      "Epoch   5/100 Batch 1560/1562 - Loss:  1.657, Seconds: 2.61\n",
      "Epoch   6/100 Batch   20/1562 - Loss:  1.591, Seconds: 2.60\n",
      "Epoch   6/100 Batch   40/1562 - Loss:  1.613, Seconds: 2.05\n",
      "Epoch   6/100 Batch   60/1562 - Loss:  1.563, Seconds: 2.42\n",
      "Epoch   6/100 Batch   80/1562 - Loss:  1.584, Seconds: 3.07\n",
      "Epoch   6/100 Batch  100/1562 - Loss:  1.683, Seconds: 3.05\n",
      "Epoch   6/100 Batch  120/1562 - Loss:  1.528, Seconds: 3.11\n",
      "Epoch   6/100 Batch  140/1562 - Loss:  1.597, Seconds: 3.98\n",
      "Epoch   6/100 Batch  160/1562 - Loss:  1.544, Seconds: 2.31\n",
      "Epoch   6/100 Batch  180/1562 - Loss:  1.624, Seconds: 3.25\n",
      "Epoch   6/100 Batch  200/1562 - Loss:  1.541, Seconds: 2.53\n",
      "Epoch   6/100 Batch  220/1562 - Loss:  1.636, Seconds: 1.94\n",
      "Epoch   6/100 Batch  240/1562 - Loss:  1.609, Seconds: 2.84\n",
      "Epoch   6/100 Batch  260/1562 - Loss:  1.646, Seconds: 2.44\n",
      "Epoch   6/100 Batch  280/1562 - Loss:  1.436, Seconds: 3.67\n",
      "Epoch   6/100 Batch  300/1562 - Loss:  1.636, Seconds: 2.98\n",
      "Epoch   6/100 Batch  320/1562 - Loss:  1.699, Seconds: 3.00\n",
      "Epoch   6/100 Batch  340/1562 - Loss:  1.612, Seconds: 2.71\n",
      "Epoch   6/100 Batch  360/1562 - Loss:  1.601, Seconds: 3.80\n",
      "Epoch   6/100 Batch  380/1562 - Loss:  1.493, Seconds: 3.27\n",
      "Epoch   6/100 Batch  400/1562 - Loss:  1.687, Seconds: 3.25\n",
      "Epoch   6/100 Batch  420/1562 - Loss:  1.646, Seconds: 2.71\n",
      "Epoch   6/100 Batch  440/1562 - Loss:  1.648, Seconds: 2.31\n",
      "Epoch   6/100 Batch  460/1562 - Loss:  1.564, Seconds: 2.89\n",
      "Epoch   6/100 Batch  480/1562 - Loss:  1.627, Seconds: 2.32\n",
      "Epoch   6/100 Batch  500/1562 - Loss:  1.605, Seconds: 3.20\n",
      "('Average loss for this update:', 1.599)\n",
      "New Record!\n",
      "Epoch   6/100 Batch  520/1562 - Loss:  1.580, Seconds: 2.74\n",
      "Epoch   6/100 Batch  540/1562 - Loss:  1.609, Seconds: 3.34\n",
      "Epoch   6/100 Batch  560/1562 - Loss:  1.614, Seconds: 2.97\n",
      "Epoch   6/100 Batch  580/1562 - Loss:  1.600, Seconds: 3.09\n",
      "Epoch   6/100 Batch  600/1562 - Loss:  1.547, Seconds: 2.37\n",
      "Epoch   6/100 Batch  620/1562 - Loss:  1.576, Seconds: 2.42\n",
      "Epoch   6/100 Batch  640/1562 - Loss:  1.496, Seconds: 3.25\n",
      "Epoch   6/100 Batch  660/1562 - Loss:  1.502, Seconds: 4.56\n",
      "Epoch   6/100 Batch  680/1562 - Loss:  1.565, Seconds: 2.15\n",
      "Epoch   6/100 Batch  700/1562 - Loss:  1.530, Seconds: 3.46\n",
      "Epoch   6/100 Batch  720/1562 - Loss:  1.668, Seconds: 2.43\n",
      "Epoch   6/100 Batch  740/1562 - Loss:  1.502, Seconds: 2.72\n",
      "Epoch   6/100 Batch  760/1562 - Loss:  1.617, Seconds: 1.96\n",
      "Epoch   6/100 Batch  780/1562 - Loss:  1.529, Seconds: 4.50\n",
      "Epoch   6/100 Batch  800/1562 - Loss:  1.601, Seconds: 3.06\n",
      "Epoch   6/100 Batch  820/1562 - Loss:  1.674, Seconds: 1.87\n",
      "Epoch   6/100 Batch  840/1562 - Loss:  1.537, Seconds: 2.16\n",
      "Epoch   6/100 Batch  860/1562 - Loss:  1.548, Seconds: 2.88\n",
      "Epoch   6/100 Batch  880/1562 - Loss:  1.552, Seconds: 2.54\n",
      "Epoch   6/100 Batch  900/1562 - Loss:  1.532, Seconds: 3.32\n",
      "Epoch   6/100 Batch  920/1562 - Loss:  1.590, Seconds: 2.24\n",
      "Epoch   6/100 Batch  940/1562 - Loss:  1.595, Seconds: 2.63\n",
      "Epoch   6/100 Batch  960/1562 - Loss:  1.619, Seconds: 2.60\n",
      "Epoch   6/100 Batch  980/1562 - Loss:  1.563, Seconds: 2.52\n",
      "Epoch   6/100 Batch 1000/1562 - Loss:  1.475, Seconds: 2.90\n",
      "Epoch   6/100 Batch 1020/1562 - Loss:  1.531, Seconds: 2.91\n",
      "('Average loss for this update:', 1.569)\n",
      "New Record!\n",
      "Epoch   6/100 Batch 1040/1562 - Loss:  1.639, Seconds: 2.62\n",
      "Epoch   6/100 Batch 1060/1562 - Loss:  1.489, Seconds: 3.74\n",
      "Epoch   6/100 Batch 1080/1562 - Loss:  1.562, Seconds: 2.33\n",
      "Epoch   6/100 Batch 1100/1562 - Loss:  1.566, Seconds: 2.99\n",
      "Epoch   6/100 Batch 1120/1562 - Loss:  1.471, Seconds: 2.14\n",
      "Epoch   6/100 Batch 1140/1562 - Loss:  1.577, Seconds: 2.94\n",
      "Epoch   6/100 Batch 1160/1562 - Loss:  1.612, Seconds: 2.90\n",
      "Epoch   6/100 Batch 1180/1562 - Loss:  1.599, Seconds: 2.14\n",
      "Epoch   6/100 Batch 1200/1562 - Loss:  1.444, Seconds: 4.10\n",
      "Epoch   6/100 Batch 1220/1562 - Loss:  1.509, Seconds: 2.90\n",
      "Epoch   6/100 Batch 1240/1562 - Loss:  1.527, Seconds: 2.25\n",
      "Epoch   6/100 Batch 1260/1562 - Loss:  1.592, Seconds: 2.62\n",
      "Epoch   6/100 Batch 1280/1562 - Loss:  1.694, Seconds: 2.97\n",
      "Epoch   6/100 Batch 1300/1562 - Loss:  1.683, Seconds: 2.70\n",
      "Epoch   6/100 Batch 1320/1562 - Loss:  1.614, Seconds: 2.34\n",
      "Epoch   6/100 Batch 1340/1562 - Loss:  1.584, Seconds: 3.45\n",
      "Epoch   6/100 Batch 1360/1562 - Loss:  1.579, Seconds: 2.51\n",
      "Epoch   6/100 Batch 1380/1562 - Loss:  1.514, Seconds: 2.41\n",
      "Epoch   6/100 Batch 1400/1562 - Loss:  1.498, Seconds: 3.08\n",
      "Epoch   6/100 Batch 1420/1562 - Loss:  1.547, Seconds: 2.70\n",
      "Epoch   6/100 Batch 1440/1562 - Loss:  1.565, Seconds: 3.23\n",
      "Epoch   6/100 Batch 1460/1562 - Loss:  1.571, Seconds: 2.51\n",
      "Epoch   6/100 Batch 1480/1562 - Loss:  1.639, Seconds: 2.91\n",
      "Epoch   6/100 Batch 1500/1562 - Loss:  1.637, Seconds: 2.31\n",
      "Epoch   6/100 Batch 1520/1562 - Loss:  1.625, Seconds: 2.60\n",
      "Epoch   6/100 Batch 1540/1562 - Loss:  1.544, Seconds: 2.46\n",
      "('Average loss for this update:', 1.572)\n",
      "No Improvement.\n",
      "Epoch   6/100 Batch 1560/1562 - Loss:  1.578, Seconds: 2.67\n",
      "Epoch   7/100 Batch   20/1562 - Loss:  1.522, Seconds: 2.62\n",
      "Epoch   7/100 Batch   40/1562 - Loss:  1.539, Seconds: 2.07\n",
      "Epoch   7/100 Batch   60/1562 - Loss:  1.500, Seconds: 2.42\n",
      "Epoch   7/100 Batch   80/1562 - Loss:  1.518, Seconds: 3.07\n",
      "Epoch   7/100 Batch  100/1562 - Loss:  1.604, Seconds: 3.14\n",
      "Epoch   7/100 Batch  120/1562 - Loss:  1.465, Seconds: 3.07\n",
      "Epoch   7/100 Batch  140/1562 - Loss:  1.531, Seconds: 3.97\n",
      "Epoch   7/100 Batch  160/1562 - Loss:  1.474, Seconds: 2.32\n",
      "Epoch   7/100 Batch  180/1562 - Loss:  1.550, Seconds: 3.24\n",
      "Epoch   7/100 Batch  200/1562 - Loss:  1.471, Seconds: 2.51\n",
      "Epoch   7/100 Batch  220/1562 - Loss:  1.554, Seconds: 1.94\n",
      "Epoch   7/100 Batch  240/1562 - Loss:  1.529, Seconds: 2.70\n",
      "Epoch   7/100 Batch  260/1562 - Loss:  1.569, Seconds: 2.44\n",
      "Epoch   7/100 Batch  280/1562 - Loss:  1.373, Seconds: 3.60\n",
      "Epoch   7/100 Batch  300/1562 - Loss:  1.561, Seconds: 2.98\n",
      "Epoch   7/100 Batch  320/1562 - Loss:  1.620, Seconds: 2.88\n",
      "Epoch   7/100 Batch  340/1562 - Loss:  1.521, Seconds: 2.68\n",
      "Epoch   7/100 Batch  360/1562 - Loss:  1.515, Seconds: 3.82\n",
      "Epoch   7/100 Batch  380/1562 - Loss:  1.433, Seconds: 3.29\n",
      "Epoch   7/100 Batch  400/1562 - Loss:  1.616, Seconds: 3.27\n",
      "Epoch   7/100 Batch  420/1562 - Loss:  1.558, Seconds: 2.69\n",
      "Epoch   7/100 Batch  440/1562 - Loss:  1.555, Seconds: 2.34\n",
      "Epoch   7/100 Batch  460/1562 - Loss:  1.487, Seconds: 2.90\n",
      "Epoch   7/100 Batch  480/1562 - Loss:  1.557, Seconds: 2.42\n",
      "Epoch   7/100 Batch  500/1562 - Loss:  1.524, Seconds: 3.19\n",
      "('Average loss for this update:', 1.524)\n",
      "New Record!\n",
      "Epoch   7/100 Batch  520/1562 - Loss:  1.502, Seconds: 2.88\n",
      "Epoch   7/100 Batch  540/1562 - Loss:  1.523, Seconds: 3.34\n",
      "Epoch   7/100 Batch  560/1562 - Loss:  1.533, Seconds: 2.98\n",
      "Epoch   7/100 Batch  580/1562 - Loss:  1.522, Seconds: 3.10\n",
      "Epoch   7/100 Batch  600/1562 - Loss:  1.477, Seconds: 2.34\n",
      "Epoch   7/100 Batch  620/1562 - Loss:  1.502, Seconds: 2.41\n",
      "Epoch   7/100 Batch  640/1562 - Loss:  1.435, Seconds: 3.27\n",
      "Epoch   7/100 Batch  660/1562 - Loss:  1.439, Seconds: 4.53\n",
      "Epoch   7/100 Batch  680/1562 - Loss:  1.481, Seconds: 2.22\n",
      "Epoch   7/100 Batch  700/1562 - Loss:  1.468, Seconds: 3.46\n",
      "Epoch   7/100 Batch  720/1562 - Loss:  1.586, Seconds: 2.41\n",
      "Epoch   7/100 Batch  740/1562 - Loss:  1.439, Seconds: 2.73\n",
      "Epoch   7/100 Batch  760/1562 - Loss:  1.533, Seconds: 1.97\n",
      "Epoch   7/100 Batch  780/1562 - Loss:  1.443, Seconds: 4.45\n",
      "Epoch   7/100 Batch  800/1562 - Loss:  1.512, Seconds: 3.15\n",
      "Epoch   7/100 Batch  820/1562 - Loss:  1.588, Seconds: 1.90\n",
      "Epoch   7/100 Batch  840/1562 - Loss:  1.468, Seconds: 2.14\n",
      "Epoch   7/100 Batch  860/1562 - Loss:  1.480, Seconds: 2.88\n",
      "Epoch   7/100 Batch  880/1562 - Loss:  1.476, Seconds: 2.48\n",
      "Epoch   7/100 Batch  900/1562 - Loss:  1.465, Seconds: 3.37\n",
      "Epoch   7/100 Batch  920/1562 - Loss:  1.515, Seconds: 2.25\n",
      "Epoch   7/100 Batch  940/1562 - Loss:  1.509, Seconds: 2.66\n",
      "Epoch   7/100 Batch  960/1562 - Loss:  1.529, Seconds: 2.61\n",
      "Epoch   7/100 Batch  980/1562 - Loss:  1.488, Seconds: 2.53\n",
      "Epoch   7/100 Batch 1000/1562 - Loss:  1.411, Seconds: 2.88\n",
      "Epoch   7/100 Batch 1020/1562 - Loss:  1.462, Seconds: 2.89\n",
      "('Average loss for this update:', 1.493)\n",
      "New Record!\n",
      "Epoch   7/100 Batch 1040/1562 - Loss:  1.553, Seconds: 2.75\n",
      "Epoch   7/100 Batch 1060/1562 - Loss:  1.414, Seconds: 3.63\n",
      "Epoch   7/100 Batch 1080/1562 - Loss:  1.489, Seconds: 2.33\n",
      "Epoch   7/100 Batch 1100/1562 - Loss:  1.484, Seconds: 3.03\n",
      "Epoch   7/100 Batch 1120/1562 - Loss:  1.401, Seconds: 2.16\n",
      "Epoch   7/100 Batch 1140/1562 - Loss:  1.511, Seconds: 2.89\n",
      "Epoch   7/100 Batch 1160/1562 - Loss:  1.534, Seconds: 3.03\n",
      "Epoch   7/100 Batch 1180/1562 - Loss:  1.523, Seconds: 2.16\n",
      "Epoch   7/100 Batch 1200/1562 - Loss:  1.388, Seconds: 3.94\n",
      "Epoch   7/100 Batch 1220/1562 - Loss:  1.447, Seconds: 2.92\n",
      "Epoch   7/100 Batch 1240/1562 - Loss:  1.455, Seconds: 2.26\n",
      "Epoch   7/100 Batch 1260/1562 - Loss:  1.513, Seconds: 2.61\n",
      "Epoch   7/100 Batch 1280/1562 - Loss:  1.607, Seconds: 3.04\n",
      "Epoch   7/100 Batch 1300/1562 - Loss:  1.601, Seconds: 2.72\n",
      "Epoch   7/100 Batch 1320/1562 - Loss:  1.528, Seconds: 2.34\n",
      "Epoch   7/100 Batch 1340/1562 - Loss:  1.500, Seconds: 3.33\n",
      "Epoch   7/100 Batch 1360/1562 - Loss:  1.503, Seconds: 2.53\n",
      "Epoch   7/100 Batch 1380/1562 - Loss:  1.445, Seconds: 2.45\n",
      "Epoch   7/100 Batch 1400/1562 - Loss:  1.423, Seconds: 3.08\n",
      "Epoch   7/100 Batch 1420/1562 - Loss:  1.464, Seconds: 2.72\n",
      "Epoch   7/100 Batch 1440/1562 - Loss:  1.483, Seconds: 3.30\n",
      "Epoch   7/100 Batch 1460/1562 - Loss:  1.502, Seconds: 2.53\n",
      "Epoch   7/100 Batch 1480/1562 - Loss:  1.547, Seconds: 2.92\n",
      "Epoch   7/100 Batch 1500/1562 - Loss:  1.551, Seconds: 2.34\n",
      "Epoch   7/100 Batch 1520/1562 - Loss:  1.548, Seconds: 2.61\n",
      "Epoch   7/100 Batch 1540/1562 - Loss:  1.466, Seconds: 2.43\n",
      "('Average loss for this update:', 1.495)\n",
      "No Improvement.\n",
      "Epoch   7/100 Batch 1560/1562 - Loss:  1.502, Seconds: 2.62\n",
      "Epoch   8/100 Batch   20/1562 - Loss:  1.455, Seconds: 2.63\n",
      "Epoch   8/100 Batch   40/1562 - Loss:  1.468, Seconds: 2.05\n",
      "Epoch   8/100 Batch   60/1562 - Loss:  1.429, Seconds: 2.42\n",
      "Epoch   8/100 Batch   80/1562 - Loss:  1.443, Seconds: 3.05\n",
      "Epoch   8/100 Batch  100/1562 - Loss:  1.526, Seconds: 3.08\n",
      "Epoch   8/100 Batch  120/1562 - Loss:  1.404, Seconds: 3.06\n",
      "Epoch   8/100 Batch  140/1562 - Loss:  1.462, Seconds: 3.97\n",
      "Epoch   8/100 Batch  160/1562 - Loss:  1.405, Seconds: 2.30\n",
      "Epoch   8/100 Batch  180/1562 - Loss:  1.469, Seconds: 3.28\n",
      "Epoch   8/100 Batch  200/1562 - Loss:  1.402, Seconds: 2.50\n",
      "Epoch   8/100 Batch  220/1562 - Loss:  1.475, Seconds: 1.97\n",
      "Epoch   8/100 Batch  240/1562 - Loss:  1.452, Seconds: 2.75\n",
      "Epoch   8/100 Batch  260/1562 - Loss:  1.486, Seconds: 2.39\n",
      "Epoch   8/100 Batch  280/1562 - Loss:  1.312, Seconds: 3.66\n",
      "Epoch   8/100 Batch  300/1562 - Loss:  1.488, Seconds: 3.07\n",
      "Epoch   8/100 Batch  320/1562 - Loss:  1.532, Seconds: 2.89\n",
      "Epoch   8/100 Batch  340/1562 - Loss:  1.442, Seconds: 2.70\n",
      "Epoch   8/100 Batch  360/1562 - Loss:  1.432, Seconds: 3.79\n",
      "Epoch   8/100 Batch  380/1562 - Loss:  1.363, Seconds: 3.23\n",
      "Epoch   8/100 Batch  400/1562 - Loss:  1.535, Seconds: 3.31\n",
      "Epoch   8/100 Batch  420/1562 - Loss:  1.475, Seconds: 2.85\n",
      "Epoch   8/100 Batch  440/1562 - Loss:  1.467, Seconds: 2.33\n",
      "Epoch   8/100 Batch  460/1562 - Loss:  1.404, Seconds: 2.92\n",
      "Epoch   8/100 Batch  480/1562 - Loss:  1.483, Seconds: 2.34\n",
      "Epoch   8/100 Batch  500/1562 - Loss:  1.450, Seconds: 3.15\n",
      "('Average loss for this update:', 1.449)\n",
      "New Record!\n",
      "Epoch   8/100 Batch  520/1562 - Loss:  1.427, Seconds: 2.71\n",
      "Epoch   8/100 Batch  540/1562 - Loss:  1.439, Seconds: 3.33\n",
      "Epoch   8/100 Batch  560/1562 - Loss:  1.454, Seconds: 2.99\n",
      "Epoch   8/100 Batch  580/1562 - Loss:  1.429, Seconds: 3.08\n",
      "Epoch   8/100 Batch  600/1562 - Loss:  1.398, Seconds: 2.35\n",
      "Epoch   8/100 Batch  620/1562 - Loss:  1.426, Seconds: 2.42\n",
      "Epoch   8/100 Batch  640/1562 - Loss:  1.367, Seconds: 3.22\n",
      "Epoch   8/100 Batch  660/1562 - Loss:  1.373, Seconds: 4.46\n",
      "Epoch   8/100 Batch  680/1562 - Loss:  1.403, Seconds: 2.15\n",
      "Epoch   8/100 Batch  700/1562 - Loss:  1.393, Seconds: 3.44\n",
      "Epoch   8/100 Batch  720/1562 - Loss:  1.507, Seconds: 2.50\n",
      "Epoch   8/100 Batch  740/1562 - Loss:  1.377, Seconds: 2.71\n",
      "Epoch   8/100 Batch  760/1562 - Loss:  1.452, Seconds: 1.95\n",
      "Epoch   8/100 Batch  780/1562 - Loss:  1.363, Seconds: 4.49\n",
      "Epoch   8/100 Batch  800/1562 - Loss:  1.431, Seconds: 3.08\n",
      "Epoch   8/100 Batch  820/1562 - Loss:  1.510, Seconds: 1.73\n",
      "Epoch   8/100 Batch  840/1562 - Loss:  1.400, Seconds: 2.14\n",
      "Epoch   8/100 Batch  860/1562 - Loss:  1.408, Seconds: 2.89\n",
      "Epoch   8/100 Batch  880/1562 - Loss:  1.402, Seconds: 2.55\n",
      "Epoch   8/100 Batch  900/1562 - Loss:  1.388, Seconds: 3.46\n",
      "Epoch   8/100 Batch  920/1562 - Loss:  1.444, Seconds: 2.21\n",
      "Epoch   8/100 Batch  940/1562 - Loss:  1.432, Seconds: 2.60\n",
      "Epoch   8/100 Batch  960/1562 - Loss:  1.446, Seconds: 2.60\n",
      "Epoch   8/100 Batch  980/1562 - Loss:  1.408, Seconds: 2.53\n",
      "Epoch   8/100 Batch 1000/1562 - Loss:  1.346, Seconds: 2.98\n",
      "Epoch   8/100 Batch 1020/1562 - Loss:  1.393, Seconds: 2.87\n",
      "('Average loss for this update:', 1.417)\n",
      "New Record!\n",
      "Epoch   8/100 Batch 1040/1562 - Loss:  1.465, Seconds: 2.71\n",
      "Epoch   8/100 Batch 1060/1562 - Loss:  1.344, Seconds: 3.67\n",
      "Epoch   8/100 Batch 1080/1562 - Loss:  1.415, Seconds: 2.33\n",
      "Epoch   8/100 Batch 1100/1562 - Loss:  1.402, Seconds: 3.03\n",
      "Epoch   8/100 Batch 1120/1562 - Loss:  1.336, Seconds: 2.12\n",
      "Epoch   8/100 Batch 1140/1562 - Loss:  1.436, Seconds: 2.96\n",
      "Epoch   8/100 Batch 1160/1562 - Loss:  1.463, Seconds: 2.90\n",
      "Epoch   8/100 Batch 1180/1562 - Loss:  1.451, Seconds: 2.13\n",
      "Epoch   8/100 Batch 1200/1562 - Loss:  1.332, Seconds: 3.90\n",
      "Epoch   8/100 Batch 1220/1562 - Loss:  1.382, Seconds: 2.90\n",
      "Epoch   8/100 Batch 1240/1562 - Loss:  1.379, Seconds: 2.25\n",
      "Epoch   8/100 Batch 1260/1562 - Loss:  1.433, Seconds: 2.60\n",
      "Epoch   8/100 Batch 1280/1562 - Loss:  1.521, Seconds: 2.96\n",
      "Epoch   8/100 Batch 1300/1562 - Loss:  1.517, Seconds: 2.71\n",
      "Epoch   8/100 Batch 1320/1562 - Loss:  1.442, Seconds: 2.33\n",
      "Epoch   8/100 Batch 1340/1562 - Loss:  1.414, Seconds: 3.34\n",
      "Epoch   8/100 Batch 1360/1562 - Loss:  1.426, Seconds: 2.54\n",
      "Epoch   8/100 Batch 1380/1562 - Loss:  1.369, Seconds: 2.41\n",
      "Epoch   8/100 Batch 1400/1562 - Loss:  1.342, Seconds: 3.11\n",
      "Epoch   8/100 Batch 1420/1562 - Loss:  1.391, Seconds: 2.73\n",
      "Epoch   8/100 Batch 1440/1562 - Loss:  1.415, Seconds: 3.25\n",
      "Epoch   8/100 Batch 1460/1562 - Loss:  1.422, Seconds: 2.51\n",
      "Epoch   8/100 Batch 1480/1562 - Loss:  1.468, Seconds: 3.18\n",
      "Epoch   8/100 Batch 1500/1562 - Loss:  1.475, Seconds: 2.45\n",
      "Epoch   8/100 Batch 1520/1562 - Loss:  1.474, Seconds: 2.60\n",
      "Epoch   8/100 Batch 1540/1562 - Loss:  1.393, Seconds: 2.42\n",
      "('Average loss for this update:', 1.419)\n",
      "No Improvement.\n",
      "Epoch   8/100 Batch 1560/1562 - Loss:  1.424, Seconds: 2.60\n",
      "Epoch   9/100 Batch   20/1562 - Loss:  1.383, Seconds: 2.62\n",
      "Epoch   9/100 Batch   40/1562 - Loss:  1.394, Seconds: 2.06\n",
      "Epoch   9/100 Batch   60/1562 - Loss:  1.350, Seconds: 2.43\n",
      "Epoch   9/100 Batch   80/1562 - Loss:  1.385, Seconds: 3.05\n",
      "Epoch   9/100 Batch  100/1562 - Loss:  1.451, Seconds: 3.16\n",
      "Epoch   9/100 Batch  120/1562 - Loss:  1.340, Seconds: 3.06\n",
      "Epoch   9/100 Batch  140/1562 - Loss:  1.394, Seconds: 4.04\n",
      "Epoch   9/100 Batch  160/1562 - Loss:  1.337, Seconds: 2.40\n",
      "Epoch   9/100 Batch  180/1562 - Loss:  1.396, Seconds: 3.35\n",
      "Epoch   9/100 Batch  200/1562 - Loss:  1.338, Seconds: 2.51\n",
      "Epoch   9/100 Batch  220/1562 - Loss:  1.400, Seconds: 1.93\n",
      "Epoch   9/100 Batch  240/1562 - Loss:  1.369, Seconds: 2.68\n",
      "Epoch   9/100 Batch  260/1562 - Loss:  1.413, Seconds: 2.41\n",
      "Epoch   9/100 Batch  280/1562 - Loss:  1.258, Seconds: 3.68\n",
      "Epoch   9/100 Batch  300/1562 - Loss:  1.422, Seconds: 3.15\n",
      "Epoch   9/100 Batch  320/1562 - Loss:  1.468, Seconds: 2.93\n",
      "Epoch   9/100 Batch  340/1562 - Loss:  1.367, Seconds: 2.69\n",
      "Epoch   9/100 Batch  360/1562 - Loss:  1.357, Seconds: 3.78\n",
      "Epoch   9/100 Batch  380/1562 - Loss:  1.299, Seconds: 3.29\n",
      "Epoch   9/100 Batch  400/1562 - Loss:  1.459, Seconds: 3.25\n",
      "Epoch   9/100 Batch  420/1562 - Loss:  1.405, Seconds: 2.73\n",
      "Epoch   9/100 Batch  440/1562 - Loss:  1.386, Seconds: 2.33\n",
      "Epoch   9/100 Batch  460/1562 - Loss:  1.326, Seconds: 2.89\n",
      "Epoch   9/100 Batch  480/1562 - Loss:  1.403, Seconds: 2.34\n",
      "Epoch   9/100 Batch  500/1562 - Loss:  1.371, Seconds: 3.16\n",
      "('Average loss for this update:', 1.378)\n",
      "New Record!\n",
      "Epoch   9/100 Batch  520/1562 - Loss:  1.356, Seconds: 2.60\n",
      "Epoch   9/100 Batch  540/1562 - Loss:  1.354, Seconds: 3.38\n",
      "Epoch   9/100 Batch  560/1562 - Loss:  1.380, Seconds: 2.96\n",
      "Epoch   9/100 Batch  580/1562 - Loss:  1.349, Seconds: 3.08\n",
      "Epoch   9/100 Batch  600/1562 - Loss:  1.324, Seconds: 2.33\n",
      "Epoch   9/100 Batch  620/1562 - Loss:  1.362, Seconds: 2.43\n",
      "Epoch   9/100 Batch  640/1562 - Loss:  1.306, Seconds: 3.35\n",
      "Epoch   9/100 Batch  660/1562 - Loss:  1.310, Seconds: 4.48\n",
      "Epoch   9/100 Batch  680/1562 - Loss:  1.331, Seconds: 2.17\n",
      "Epoch   9/100 Batch  700/1562 - Loss:  1.324, Seconds: 3.45\n",
      "Epoch   9/100 Batch  720/1562 - Loss:  1.421, Seconds: 2.45\n",
      "Epoch   9/100 Batch  740/1562 - Loss:  1.322, Seconds: 2.68\n",
      "Epoch   9/100 Batch  760/1562 - Loss:  1.374, Seconds: 1.93\n",
      "Epoch   9/100 Batch  780/1562 - Loss:  1.296, Seconds: 4.51\n",
      "Epoch   9/100 Batch  800/1562 - Loss:  1.352, Seconds: 3.09\n",
      "Epoch   9/100 Batch  820/1562 - Loss:  1.426, Seconds: 1.87\n",
      "Epoch   9/100 Batch  840/1562 - Loss:  1.336, Seconds: 2.14\n",
      "Epoch   9/100 Batch  860/1562 - Loss:  1.343, Seconds: 2.87\n",
      "Epoch   9/100 Batch  880/1562 - Loss:  1.332, Seconds: 2.51\n",
      "Epoch   9/100 Batch  900/1562 - Loss:  1.331, Seconds: 3.34\n",
      "Epoch   9/100 Batch  920/1562 - Loss:  1.372, Seconds: 2.24\n",
      "Epoch   9/100 Batch  940/1562 - Loss:  1.362, Seconds: 2.60\n",
      "Epoch   9/100 Batch  960/1562 - Loss:  1.359, Seconds: 2.69\n",
      "Epoch   9/100 Batch  980/1562 - Loss:  1.342, Seconds: 2.49\n",
      "Epoch   9/100 Batch 1000/1562 - Loss:  1.281, Seconds: 2.91\n",
      "Epoch   9/100 Batch 1020/1562 - Loss:  1.326, Seconds: 2.90\n",
      "('Average loss for this update:', 1.346)\n",
      "New Record!\n",
      "Epoch   9/100 Batch 1040/1562 - Loss:  1.384, Seconds: 2.71\n",
      "Epoch   9/100 Batch 1060/1562 - Loss:  1.280, Seconds: 3.64\n",
      "Epoch   9/100 Batch 1080/1562 - Loss:  1.344, Seconds: 2.33\n",
      "Epoch   9/100 Batch 1100/1562 - Loss:  1.328, Seconds: 2.98\n",
      "Epoch   9/100 Batch 1120/1562 - Loss:  1.280, Seconds: 2.20\n",
      "Epoch   9/100 Batch 1140/1562 - Loss:  1.372, Seconds: 2.88\n",
      "Epoch   9/100 Batch 1160/1562 - Loss:  1.391, Seconds: 2.88\n",
      "Epoch   9/100 Batch 1180/1562 - Loss:  1.383, Seconds: 2.18\n",
      "Epoch   9/100 Batch 1200/1562 - Loss:  1.277, Seconds: 3.91\n",
      "Epoch   9/100 Batch 1220/1562 - Loss:  1.318, Seconds: 2.90\n",
      "Epoch   9/100 Batch 1240/1562 - Loss:  1.318, Seconds: 2.28\n",
      "Epoch   9/100 Batch 1260/1562 - Loss:  1.359, Seconds: 2.66\n",
      "Epoch   9/100 Batch 1280/1562 - Loss:  1.433, Seconds: 3.01\n",
      "Epoch   9/100 Batch 1300/1562 - Loss:  1.446, Seconds: 2.74\n",
      "Epoch   9/100 Batch 1320/1562 - Loss:  1.358, Seconds: 2.35\n",
      "Epoch   9/100 Batch 1340/1562 - Loss:  1.343, Seconds: 3.35\n",
      "Epoch   9/100 Batch 1360/1562 - Loss:  1.359, Seconds: 2.52\n",
      "Epoch   9/100 Batch 1380/1562 - Loss:  1.314, Seconds: 2.73\n",
      "Epoch   9/100 Batch 1400/1562 - Loss:  1.267, Seconds: 3.08\n",
      "Epoch   9/100 Batch 1420/1562 - Loss:  1.321, Seconds: 2.69\n",
      "Epoch   9/100 Batch 1440/1562 - Loss:  1.332, Seconds: 3.19\n",
      "Epoch   9/100 Batch 1460/1562 - Loss:  1.357, Seconds: 2.52\n",
      "Epoch   9/100 Batch 1480/1562 - Loss:  1.379, Seconds: 2.97\n",
      "Epoch   9/100 Batch 1500/1562 - Loss:  1.390, Seconds: 2.46\n",
      "Epoch   9/100 Batch 1520/1562 - Loss:  1.403, Seconds: 2.60\n",
      "Epoch   9/100 Batch 1540/1562 - Loss:  1.327, Seconds: 2.48\n",
      "('Average loss for this update:', 1.348)\n",
      "No Improvement.\n",
      "Epoch   9/100 Batch 1560/1562 - Loss:  1.352, Seconds: 2.65\n",
      "Epoch  10/100 Batch   20/1562 - Loss:  1.316, Seconds: 2.68\n",
      "Epoch  10/100 Batch   40/1562 - Loss:  1.322, Seconds: 2.12\n",
      "Epoch  10/100 Batch   60/1562 - Loss:  1.295, Seconds: 2.48\n",
      "Epoch  10/100 Batch   80/1562 - Loss:  1.324, Seconds: 3.04\n",
      "Epoch  10/100 Batch  100/1562 - Loss:  1.388, Seconds: 3.14\n",
      "Epoch  10/100 Batch  120/1562 - Loss:  1.285, Seconds: 3.07\n",
      "Epoch  10/100 Batch  140/1562 - Loss:  1.334, Seconds: 4.00\n",
      "Epoch  10/100 Batch  160/1562 - Loss:  1.280, Seconds: 2.35\n",
      "Epoch  10/100 Batch  180/1562 - Loss:  1.327, Seconds: 3.26\n",
      "Epoch  10/100 Batch  200/1562 - Loss:  1.278, Seconds: 2.56\n",
      "Epoch  10/100 Batch  220/1562 - Loss:  1.324, Seconds: 1.97\n",
      "Epoch  10/100 Batch  240/1562 - Loss:  1.304, Seconds: 2.69\n",
      "Epoch  10/100 Batch  260/1562 - Loss:  1.336, Seconds: 2.42\n",
      "Epoch  10/100 Batch  280/1562 - Loss:  1.203, Seconds: 3.63\n",
      "Epoch  10/100 Batch  300/1562 - Loss:  1.358, Seconds: 2.99\n",
      "Epoch  10/100 Batch  320/1562 - Loss:  1.393, Seconds: 2.92\n",
      "Epoch  10/100 Batch  340/1562 - Loss:  1.295, Seconds: 2.71\n",
      "Epoch  10/100 Batch  360/1562 - Loss:  1.289, Seconds: 3.85\n",
      "Epoch  10/100 Batch  380/1562 - Loss:  1.238, Seconds: 3.28\n",
      "Epoch  10/100 Batch  400/1562 - Loss:  1.395, Seconds: 3.24\n",
      "Epoch  10/100 Batch  420/1562 - Loss:  1.333, Seconds: 2.70\n",
      "Epoch  10/100 Batch  440/1562 - Loss:  1.331, Seconds: 2.32\n",
      "Epoch  10/100 Batch  460/1562 - Loss:  1.267, Seconds: 2.90\n",
      "Epoch  10/100 Batch  480/1562 - Loss:  1.336, Seconds: 2.36\n",
      "Epoch  10/100 Batch  500/1562 - Loss:  1.298, Seconds: 3.19\n",
      "('Average loss for this update:', 1.312)\n",
      "New Record!\n",
      "Epoch  10/100 Batch  520/1562 - Loss:  1.279, Seconds: 2.79\n",
      "Epoch  10/100 Batch  540/1562 - Loss:  1.280, Seconds: 3.33\n",
      "Epoch  10/100 Batch  560/1562 - Loss:  1.302, Seconds: 2.98\n",
      "Epoch  10/100 Batch  580/1562 - Loss:  1.277, Seconds: 3.13\n",
      "Epoch  10/100 Batch  600/1562 - Loss:  1.270, Seconds: 2.35\n",
      "Epoch  10/100 Batch  620/1562 - Loss:  1.302, Seconds: 2.43\n",
      "Epoch  10/100 Batch  640/1562 - Loss:  1.251, Seconds: 3.23\n",
      "Epoch  10/100 Batch  660/1562 - Loss:  1.258, Seconds: 4.45\n",
      "Epoch  10/100 Batch  680/1562 - Loss:  1.273, Seconds: 2.13\n",
      "Epoch  10/100 Batch  700/1562 - Loss:  1.271, Seconds: 3.49\n",
      "Epoch  10/100 Batch  720/1562 - Loss:  1.350, Seconds: 2.63\n",
      "Epoch  10/100 Batch  740/1562 - Loss:  1.257, Seconds: 2.73\n",
      "Epoch  10/100 Batch  760/1562 - Loss:  1.312, Seconds: 1.98\n",
      "Epoch  10/100 Batch  780/1562 - Loss:  1.229, Seconds: 4.47\n",
      "Epoch  10/100 Batch  800/1562 - Loss:  1.291, Seconds: 3.08\n",
      "Epoch  10/100 Batch  820/1562 - Loss:  1.342, Seconds: 1.86\n",
      "Epoch  10/100 Batch  840/1562 - Loss:  1.277, Seconds: 2.15\n",
      "Epoch  10/100 Batch  860/1562 - Loss:  1.275, Seconds: 2.88\n",
      "Epoch  10/100 Batch  880/1562 - Loss:  1.270, Seconds: 2.52\n",
      "Epoch  10/100 Batch  900/1562 - Loss:  1.268, Seconds: 3.37\n",
      "Epoch  10/100 Batch  920/1562 - Loss:  1.303, Seconds: 2.24\n",
      "Epoch  10/100 Batch  940/1562 - Loss:  1.289, Seconds: 2.65\n",
      "Epoch  10/100 Batch  960/1562 - Loss:  1.291, Seconds: 2.59\n",
      "Epoch  10/100 Batch  980/1562 - Loss:  1.270, Seconds: 2.57\n",
      "Epoch  10/100 Batch 1000/1562 - Loss:  1.221, Seconds: 2.89\n",
      "Epoch  10/100 Batch 1020/1562 - Loss:  1.270, Seconds: 2.89\n",
      "('Average loss for this update:', 1.281)\n",
      "New Record!\n",
      "Epoch  10/100 Batch 1040/1562 - Loss:  1.305, Seconds: 2.97\n",
      "Epoch  10/100 Batch 1060/1562 - Loss:  1.210, Seconds: 3.70\n",
      "Epoch  10/100 Batch 1080/1562 - Loss:  1.283, Seconds: 2.33\n",
      "Epoch  10/100 Batch 1100/1562 - Loss:  1.260, Seconds: 3.16\n",
      "Epoch  10/100 Batch 1120/1562 - Loss:  1.223, Seconds: 2.15\n",
      "Epoch  10/100 Batch 1140/1562 - Loss:  1.296, Seconds: 2.97\n",
      "Epoch  10/100 Batch 1160/1562 - Loss:  1.329, Seconds: 2.91\n",
      "Epoch  10/100 Batch 1180/1562 - Loss:  1.318, Seconds: 2.16\n",
      "Epoch  10/100 Batch 1200/1562 - Loss:  1.224, Seconds: 3.93\n",
      "Epoch  10/100 Batch 1220/1562 - Loss:  1.268, Seconds: 2.87\n",
      "Epoch  10/100 Batch 1240/1562 - Loss:  1.249, Seconds: 2.25\n",
      "Epoch  10/100 Batch 1260/1562 - Loss:  1.296, Seconds: 2.62\n",
      "Epoch  10/100 Batch 1280/1562 - Loss:  1.361, Seconds: 3.00\n",
      "Epoch  10/100 Batch 1300/1562 - Loss:  1.370, Seconds: 2.69\n",
      "Epoch  10/100 Batch 1320/1562 - Loss:  1.293, Seconds: 2.34\n",
      "Epoch  10/100 Batch 1340/1562 - Loss:  1.272, Seconds: 3.38\n",
      "Epoch  10/100 Batch 1360/1562 - Loss:  1.298, Seconds: 2.55\n",
      "Epoch  10/100 Batch 1380/1562 - Loss:  1.254, Seconds: 2.42\n",
      "Epoch  10/100 Batch 1400/1562 - Loss:  1.210, Seconds: 3.06\n",
      "Epoch  10/100 Batch 1420/1562 - Loss:  1.262, Seconds: 2.75\n",
      "Epoch  10/100 Batch 1440/1562 - Loss:  1.269, Seconds: 3.19\n",
      "Epoch  10/100 Batch 1460/1562 - Loss:  1.288, Seconds: 2.51\n",
      "Epoch  10/100 Batch 1480/1562 - Loss:  1.311, Seconds: 2.89\n",
      "Epoch  10/100 Batch 1500/1562 - Loss:  1.322, Seconds: 2.41\n",
      "Epoch  10/100 Batch 1520/1562 - Loss:  1.337, Seconds: 2.61\n",
      "Epoch  10/100 Batch 1540/1562 - Loss:  1.268, Seconds: 2.46\n",
      "('Average loss for this update:', 1.284)\n",
      "No Improvement.\n",
      "Epoch  10/100 Batch 1560/1562 - Loss:  1.284, Seconds: 2.68\n",
      "Epoch  11/100 Batch   20/1562 - Loss:  1.267, Seconds: 2.63\n",
      "Epoch  11/100 Batch   40/1562 - Loss:  1.266, Seconds: 2.06\n",
      "Epoch  11/100 Batch   60/1562 - Loss:  1.230, Seconds: 2.49\n",
      "Epoch  11/100 Batch   80/1562 - Loss:  1.277, Seconds: 3.07\n",
      "Epoch  11/100 Batch  100/1562 - Loss:  1.309, Seconds: 3.20\n",
      "Epoch  11/100 Batch  120/1562 - Loss:  1.229, Seconds: 3.07\n",
      "Epoch  11/100 Batch  140/1562 - Loss:  1.278, Seconds: 4.02\n",
      "Epoch  11/100 Batch  160/1562 - Loss:  1.222, Seconds: 2.39\n",
      "Epoch  11/100 Batch  180/1562 - Loss:  1.269, Seconds: 3.31\n",
      "Epoch  11/100 Batch  200/1562 - Loss:  1.220, Seconds: 2.51\n",
      "Epoch  11/100 Batch  220/1562 - Loss:  1.272, Seconds: 2.01\n",
      "Epoch  11/100 Batch  240/1562 - Loss:  1.252, Seconds: 2.73\n",
      "Epoch  11/100 Batch  260/1562 - Loss:  1.282, Seconds: 2.41\n",
      "Epoch  11/100 Batch  280/1562 - Loss:  1.156, Seconds: 3.69\n",
      "Epoch  11/100 Batch  300/1562 - Loss:  1.299, Seconds: 2.98\n",
      "Epoch  11/100 Batch  320/1562 - Loss:  1.332, Seconds: 2.92\n",
      "Epoch  11/100 Batch  340/1562 - Loss:  1.232, Seconds: 2.69\n",
      "Epoch  11/100 Batch  360/1562 - Loss:  1.222, Seconds: 3.83\n",
      "Epoch  11/100 Batch  380/1562 - Loss:  1.173, Seconds: 3.25\n",
      "Epoch  11/100 Batch  400/1562 - Loss:  1.324, Seconds: 3.25\n",
      "Epoch  11/100 Batch  420/1562 - Loss:  1.276, Seconds: 2.71\n",
      "Epoch  11/100 Batch  440/1562 - Loss:  1.255, Seconds: 2.36\n",
      "Epoch  11/100 Batch  460/1562 - Loss:  1.217, Seconds: 2.99\n",
      "Epoch  11/100 Batch  480/1562 - Loss:  1.282, Seconds: 2.34\n",
      "Epoch  11/100 Batch  500/1562 - Loss:  1.241, Seconds: 3.17\n",
      "('Average loss for this update:', 1.254)\n",
      "New Record!\n",
      "Epoch  11/100 Batch  520/1562 - Loss:  1.233, Seconds: 2.70\n",
      "Epoch  11/100 Batch  540/1562 - Loss:  1.219, Seconds: 3.34\n",
      "Epoch  11/100 Batch  560/1562 - Loss:  1.242, Seconds: 3.05\n",
      "Epoch  11/100 Batch  580/1562 - Loss:  1.208, Seconds: 3.10\n",
      "Epoch  11/100 Batch  600/1562 - Loss:  1.192, Seconds: 2.32\n",
      "Epoch  11/100 Batch  620/1562 - Loss:  1.244, Seconds: 2.44\n",
      "Epoch  11/100 Batch  640/1562 - Loss:  1.201, Seconds: 3.23\n",
      "Epoch  11/100 Batch  660/1562 - Loss:  1.202, Seconds: 4.42\n",
      "Epoch  11/100 Batch  680/1562 - Loss:  1.208, Seconds: 2.13\n",
      "Epoch  11/100 Batch  700/1562 - Loss:  1.214, Seconds: 3.44\n",
      "Epoch  11/100 Batch  720/1562 - Loss:  1.286, Seconds: 2.42\n",
      "Epoch  11/100 Batch  740/1562 - Loss:  1.202, Seconds: 2.75\n",
      "Epoch  11/100 Batch  760/1562 - Loss:  1.252, Seconds: 1.95\n",
      "Epoch  11/100 Batch  780/1562 - Loss:  1.186, Seconds: 4.46\n",
      "Epoch  11/100 Batch  800/1562 - Loss:  1.233, Seconds: 3.07\n",
      "Epoch  11/100 Batch  820/1562 - Loss:  1.283, Seconds: 1.85\n",
      "Epoch  11/100 Batch  840/1562 - Loss:  1.221, Seconds: 2.15\n",
      "Epoch  11/100 Batch  860/1562 - Loss:  1.211, Seconds: 2.89\n",
      "Epoch  11/100 Batch  880/1562 - Loss:  1.211, Seconds: 2.51\n",
      "Epoch  11/100 Batch  900/1562 - Loss:  1.218, Seconds: 3.36\n",
      "Epoch  11/100 Batch  920/1562 - Loss:  1.248, Seconds: 2.24\n",
      "Epoch  11/100 Batch  940/1562 - Loss:  1.238, Seconds: 2.62\n",
      "Epoch  11/100 Batch  960/1562 - Loss:  1.219, Seconds: 2.64\n",
      "Epoch  11/100 Batch  980/1562 - Loss:  1.219, Seconds: 2.51\n",
      "Epoch  11/100 Batch 1000/1562 - Loss:  1.162, Seconds: 2.87\n",
      "Epoch  11/100 Batch 1020/1562 - Loss:  1.213, Seconds: 2.87\n",
      "('Average loss for this update:', 1.222)\n",
      "New Record!\n",
      "Epoch  11/100 Batch 1040/1562 - Loss:  1.246, Seconds: 2.44\n",
      "Epoch  11/100 Batch 1060/1562 - Loss:  1.161, Seconds: 3.69\n",
      "Epoch  11/100 Batch 1080/1562 - Loss:  1.225, Seconds: 2.32\n",
      "Epoch  11/100 Batch 1100/1562 - Loss:  1.201, Seconds: 2.99\n",
      "Epoch  11/100 Batch 1120/1562 - Loss:  1.170, Seconds: 2.17\n",
      "Epoch  11/100 Batch 1140/1562 - Loss:  1.242, Seconds: 2.86\n",
      "Epoch  11/100 Batch 1160/1562 - Loss:  1.269, Seconds: 2.90\n",
      "Epoch  11/100 Batch 1180/1562 - Loss:  1.274, Seconds: 2.15\n",
      "Epoch  11/100 Batch 1200/1562 - Loss:  1.175, Seconds: 3.90\n",
      "Epoch  11/100 Batch 1220/1562 - Loss:  1.219, Seconds: 2.88\n",
      "Epoch  11/100 Batch 1240/1562 - Loss:  1.197, Seconds: 2.24\n",
      "Epoch  11/100 Batch 1260/1562 - Loss:  1.241, Seconds: 2.61\n",
      "Epoch  11/100 Batch 1280/1562 - Loss:  1.299, Seconds: 3.00\n",
      "Epoch  11/100 Batch 1300/1562 - Loss:  1.308, Seconds: 2.74\n",
      "Epoch  11/100 Batch 1320/1562 - Loss:  1.223, Seconds: 2.33\n",
      "Epoch  11/100 Batch 1340/1562 - Loss:  1.215, Seconds: 3.38\n",
      "Epoch  11/100 Batch 1360/1562 - Loss:  1.225, Seconds: 2.51\n",
      "Epoch  11/100 Batch 1380/1562 - Loss:  1.199, Seconds: 2.46\n",
      "Epoch  11/100 Batch 1400/1562 - Loss:  1.157, Seconds: 3.07\n",
      "Epoch  11/100 Batch 1420/1562 - Loss:  1.205, Seconds: 2.73\n",
      "Epoch  11/100 Batch 1440/1562 - Loss:  1.213, Seconds: 3.17\n",
      "Epoch  11/100 Batch 1460/1562 - Loss:  1.224, Seconds: 2.49\n",
      "Epoch  11/100 Batch 1480/1562 - Loss:  1.252, Seconds: 2.89\n",
      "Epoch  11/100 Batch 1500/1562 - Loss:  1.258, Seconds: 2.35\n",
      "Epoch  11/100 Batch 1520/1562 - Loss:  1.275, Seconds: 2.65\n",
      "Epoch  11/100 Batch 1540/1562 - Loss:  1.212, Seconds: 2.49\n",
      "('Average loss for this update:', 1.226)\n",
      "No Improvement.\n",
      "Epoch  11/100 Batch 1560/1562 - Loss:  1.222, Seconds: 2.64\n",
      "Epoch  12/100 Batch   20/1562 - Loss:  1.204, Seconds: 2.67\n",
      "Epoch  12/100 Batch   40/1562 - Loss:  1.204, Seconds: 2.06\n",
      "Epoch  12/100 Batch   60/1562 - Loss:  1.180, Seconds: 2.42\n",
      "Epoch  12/100 Batch   80/1562 - Loss:  1.220, Seconds: 3.08\n",
      "Epoch  12/100 Batch  100/1562 - Loss:  1.270, Seconds: 3.06\n",
      "Epoch  12/100 Batch  120/1562 - Loss:  1.187, Seconds: 3.11\n",
      "Epoch  12/100 Batch  140/1562 - Loss:  1.230, Seconds: 3.97\n",
      "Epoch  12/100 Batch  160/1562 - Loss:  1.180, Seconds: 2.34\n",
      "Epoch  12/100 Batch  180/1562 - Loss:  1.215, Seconds: 3.26\n",
      "Epoch  12/100 Batch  200/1562 - Loss:  1.171, Seconds: 2.53\n",
      "Epoch  12/100 Batch  220/1562 - Loss:  1.211, Seconds: 1.95\n",
      "Epoch  12/100 Batch  240/1562 - Loss:  1.202, Seconds: 2.70\n",
      "Epoch  12/100 Batch  260/1562 - Loss:  1.231, Seconds: 2.44\n",
      "Epoch  12/100 Batch  280/1562 - Loss:  1.114, Seconds: 3.64\n",
      "Epoch  12/100 Batch  300/1562 - Loss:  1.238, Seconds: 3.00\n",
      "Epoch  12/100 Batch  320/1562 - Loss:  1.268, Seconds: 2.92\n",
      "Epoch  12/100 Batch  340/1562 - Loss:  1.186, Seconds: 2.69\n",
      "Epoch  12/100 Batch  360/1562 - Loss:  1.168, Seconds: 3.81\n",
      "Epoch  12/100 Batch  380/1562 - Loss:  1.130, Seconds: 3.29\n",
      "Epoch  12/100 Batch  400/1562 - Loss:  1.277, Seconds: 3.25\n",
      "Epoch  12/100 Batch  420/1562 - Loss:  1.218, Seconds: 2.69\n",
      "Epoch  12/100 Batch  440/1562 - Loss:  1.201, Seconds: 2.31\n",
      "Epoch  12/100 Batch  460/1562 - Loss:  1.177, Seconds: 2.90\n",
      "Epoch  12/100 Batch  480/1562 - Loss:  1.215, Seconds: 2.32\n",
      "Epoch  12/100 Batch  500/1562 - Loss:  1.193, Seconds: 3.13\n",
      "('Average loss for this update:', 1.203)\n",
      "New Record!\n",
      "Epoch  12/100 Batch  520/1562 - Loss:  1.182, Seconds: 2.71\n",
      "Epoch  12/100 Batch  540/1562 - Loss:  1.163, Seconds: 3.34\n",
      "Epoch  12/100 Batch  560/1562 - Loss:  1.195, Seconds: 3.12\n",
      "Epoch  12/100 Batch  580/1562 - Loss:  1.152, Seconds: 3.05\n",
      "Epoch  12/100 Batch  600/1562 - Loss:  1.145, Seconds: 2.33\n",
      "Epoch  12/100 Batch  620/1562 - Loss:  1.194, Seconds: 2.39\n",
      "Epoch  12/100 Batch  640/1562 - Loss:  1.157, Seconds: 3.27\n",
      "Epoch  12/100 Batch  660/1562 - Loss:  1.151, Seconds: 4.46\n",
      "Epoch  12/100 Batch  680/1562 - Loss:  1.162, Seconds: 2.14\n",
      "Epoch  12/100 Batch  700/1562 - Loss:  1.168, Seconds: 3.46\n",
      "Epoch  12/100 Batch  720/1562 - Loss:  1.223, Seconds: 2.42\n",
      "Epoch  12/100 Batch  740/1562 - Loss:  1.154, Seconds: 2.73\n",
      "Epoch  12/100 Batch  760/1562 - Loss:  1.204, Seconds: 1.95\n",
      "Epoch  12/100 Batch  780/1562 - Loss:  1.120, Seconds: 4.47\n",
      "Epoch  12/100 Batch  800/1562 - Loss:  1.167, Seconds: 3.20\n",
      "Epoch  12/100 Batch  820/1562 - Loss:  1.220, Seconds: 1.97\n",
      "Epoch  12/100 Batch  840/1562 - Loss:  1.172, Seconds: 2.27\n",
      "Epoch  12/100 Batch  860/1562 - Loss:  1.162, Seconds: 2.87\n",
      "Epoch  12/100 Batch  880/1562 - Loss:  1.161, Seconds: 2.64\n",
      "Epoch  12/100 Batch  900/1562 - Loss:  1.172, Seconds: 3.36\n",
      "Epoch  12/100 Batch  920/1562 - Loss:  1.197, Seconds: 2.22\n",
      "Epoch  12/100 Batch  940/1562 - Loss:  1.185, Seconds: 2.62\n",
      "Epoch  12/100 Batch  960/1562 - Loss:  1.176, Seconds: 2.61\n",
      "Epoch  12/100 Batch  980/1562 - Loss:  1.166, Seconds: 2.73\n",
      "Epoch  12/100 Batch 1000/1562 - Loss:  1.128, Seconds: 2.88\n",
      "Epoch  12/100 Batch 1020/1562 - Loss:  1.169, Seconds: 2.91\n",
      "('Average loss for this update:', 1.172)\n",
      "New Record!\n",
      "Epoch  12/100 Batch 1040/1562 - Loss:  1.195, Seconds: 2.73\n",
      "Epoch  12/100 Batch 1060/1562 - Loss:  1.118, Seconds: 3.63\n",
      "Epoch  12/100 Batch 1080/1562 - Loss:  1.167, Seconds: 2.41\n",
      "Epoch  12/100 Batch 1100/1562 - Loss:  1.148, Seconds: 2.98\n",
      "Epoch  12/100 Batch 1120/1562 - Loss:  1.118, Seconds: 2.16\n",
      "Epoch  12/100 Batch 1140/1562 - Loss:  1.193, Seconds: 2.87\n",
      "Epoch  12/100 Batch 1160/1562 - Loss:  1.214, Seconds: 2.90\n",
      "Epoch  12/100 Batch 1180/1562 - Loss:  1.202, Seconds: 2.15\n",
      "Epoch  12/100 Batch 1200/1562 - Loss:  1.133, Seconds: 3.93\n",
      "Epoch  12/100 Batch 1220/1562 - Loss:  1.169, Seconds: 2.90\n",
      "Epoch  12/100 Batch 1240/1562 - Loss:  1.149, Seconds: 2.25\n",
      "Epoch  12/100 Batch 1260/1562 - Loss:  1.183, Seconds: 2.63\n",
      "Epoch  12/100 Batch 1280/1562 - Loss:  1.225, Seconds: 2.99\n",
      "Epoch  12/100 Batch 1300/1562 - Loss:  1.255, Seconds: 2.76\n",
      "Epoch  12/100 Batch 1320/1562 - Loss:  1.179, Seconds: 2.33\n",
      "Epoch  12/100 Batch 1340/1562 - Loss:  1.163, Seconds: 3.63\n",
      "Epoch  12/100 Batch 1360/1562 - Loss:  1.180, Seconds: 2.55\n",
      "Epoch  12/100 Batch 1380/1562 - Loss:  1.146, Seconds: 2.45\n",
      "Epoch  12/100 Batch 1400/1562 - Loss:  1.110, Seconds: 3.12\n",
      "Epoch  12/100 Batch 1420/1562 - Loss:  1.156, Seconds: 2.78\n",
      "Epoch  12/100 Batch 1440/1562 - Loss:  1.153, Seconds: 3.18\n",
      "Epoch  12/100 Batch 1460/1562 - Loss:  1.159, Seconds: 2.51\n",
      "Epoch  12/100 Batch 1480/1562 - Loss:  1.188, Seconds: 2.88\n",
      "Epoch  12/100 Batch 1500/1562 - Loss:  1.202, Seconds: 2.33\n",
      "Epoch  12/100 Batch 1520/1562 - Loss:  1.219, Seconds: 2.60\n",
      "Epoch  12/100 Batch 1540/1562 - Loss:  1.168, Seconds: 2.60\n",
      "('Average loss for this update:', 1.172)\n",
      "No Improvement.\n",
      "Epoch  12/100 Batch 1560/1562 - Loss:  1.160, Seconds: 2.63\n",
      "Epoch  13/100 Batch   20/1562 - Loss:  1.156, Seconds: 2.71\n",
      "Epoch  13/100 Batch   40/1562 - Loss:  1.155, Seconds: 2.07\n",
      "Epoch  13/100 Batch   60/1562 - Loss:  1.122, Seconds: 2.48\n",
      "Epoch  13/100 Batch   80/1562 - Loss:  1.170, Seconds: 3.06\n",
      "Epoch  13/100 Batch  100/1562 - Loss:  1.209, Seconds: 3.07\n",
      "Epoch  13/100 Batch  120/1562 - Loss:  1.143, Seconds: 3.08\n",
      "Epoch  13/100 Batch  140/1562 - Loss:  1.178, Seconds: 4.02\n",
      "Epoch  13/100 Batch  160/1562 - Loss:  1.140, Seconds: 2.33\n",
      "Epoch  13/100 Batch  180/1562 - Loss:  1.173, Seconds: 3.26\n",
      "Epoch  13/100 Batch  200/1562 - Loss:  1.126, Seconds: 2.51\n",
      "Epoch  13/100 Batch  220/1562 - Loss:  1.160, Seconds: 1.94\n",
      "Epoch  13/100 Batch  240/1562 - Loss:  1.155, Seconds: 2.70\n",
      "Epoch  13/100 Batch  260/1562 - Loss:  1.175, Seconds: 2.51\n",
      "Epoch  13/100 Batch  280/1562 - Loss:  1.068, Seconds: 3.66\n",
      "Epoch  13/100 Batch  300/1562 - Loss:  1.184, Seconds: 3.01\n",
      "Epoch  13/100 Batch  320/1562 - Loss:  1.213, Seconds: 2.88\n",
      "Epoch  13/100 Batch  340/1562 - Loss:  1.128, Seconds: 2.77\n",
      "Epoch  13/100 Batch  360/1562 - Loss:  1.113, Seconds: 3.83\n",
      "Epoch  13/100 Batch  380/1562 - Loss:  1.078, Seconds: 3.29\n",
      "Epoch  13/100 Batch  400/1562 - Loss:  1.218, Seconds: 3.26\n",
      "Epoch  13/100 Batch  420/1562 - Loss:  1.173, Seconds: 2.77\n",
      "Epoch  13/100 Batch  440/1562 - Loss:  1.159, Seconds: 2.37\n",
      "Epoch  13/100 Batch  460/1562 - Loss:  1.124, Seconds: 2.87\n",
      "Epoch  13/100 Batch  480/1562 - Loss:  1.157, Seconds: 2.31\n",
      "Epoch  13/100 Batch  500/1562 - Loss:  1.126, Seconds: 3.16\n",
      "('Average loss for this update:', 1.151)\n",
      "New Record!\n",
      "Epoch  13/100 Batch  520/1562 - Loss:  1.126, Seconds: 2.88\n",
      "Epoch  13/100 Batch  540/1562 - Loss:  1.115, Seconds: 3.35\n",
      "Epoch  13/100 Batch  560/1562 - Loss:  1.143, Seconds: 2.96\n",
      "Epoch  13/100 Batch  580/1562 - Loss:  1.096, Seconds: 3.06\n",
      "Epoch  13/100 Batch  600/1562 - Loss:  1.098, Seconds: 2.34\n",
      "Epoch  13/100 Batch  620/1562 - Loss:  1.145, Seconds: 2.41\n",
      "Epoch  13/100 Batch  640/1562 - Loss:  1.109, Seconds: 3.28\n",
      "Epoch  13/100 Batch  660/1562 - Loss:  1.116, Seconds: 4.51\n",
      "Epoch  13/100 Batch  680/1562 - Loss:  1.111, Seconds: 2.13\n",
      "Epoch  13/100 Batch  700/1562 - Loss:  1.122, Seconds: 3.45\n",
      "Epoch  13/100 Batch  720/1562 - Loss:  1.177, Seconds: 2.44\n",
      "Epoch  13/100 Batch  740/1562 - Loss:  1.105, Seconds: 2.70\n",
      "Epoch  13/100 Batch  760/1562 - Loss:  1.138, Seconds: 1.96\n",
      "Epoch  13/100 Batch  780/1562 - Loss:  1.082, Seconds: 4.91\n",
      "Epoch  13/100 Batch  800/1562 - Loss:  1.123, Seconds: 3.03\n",
      "Epoch  13/100 Batch  820/1562 - Loss:  1.166, Seconds: 1.86\n",
      "Epoch  13/100 Batch  840/1562 - Loss:  1.119, Seconds: 2.14\n",
      "Epoch  13/100 Batch  860/1562 - Loss:  1.113, Seconds: 2.87\n",
      "Epoch  13/100 Batch  880/1562 - Loss:  1.112, Seconds: 2.52\n",
      "Epoch  13/100 Batch  900/1562 - Loss:  1.117, Seconds: 3.33\n",
      "Epoch  13/100 Batch  920/1562 - Loss:  1.140, Seconds: 2.23\n",
      "Epoch  13/100 Batch  940/1562 - Loss:  1.140, Seconds: 2.76\n",
      "Epoch  13/100 Batch  960/1562 - Loss:  1.116, Seconds: 2.60\n",
      "Epoch  13/100 Batch  980/1562 - Loss:  1.113, Seconds: 2.53\n",
      "Epoch  13/100 Batch 1000/1562 - Loss:  1.083, Seconds: 2.87\n",
      "Epoch  13/100 Batch 1020/1562 - Loss:  1.127, Seconds: 2.86\n",
      "('Average loss for this update:', 1.122)\n",
      "New Record!\n",
      "Epoch  13/100 Batch 1040/1562 - Loss:  1.145, Seconds: 2.44\n",
      "Epoch  13/100 Batch 1060/1562 - Loss:  1.079, Seconds: 3.61\n",
      "Epoch  13/100 Batch 1080/1562 - Loss:  1.129, Seconds: 2.33\n",
      "Epoch  13/100 Batch 1100/1562 - Loss:  1.084, Seconds: 3.00\n",
      "Epoch  13/100 Batch 1120/1562 - Loss:  1.073, Seconds: 2.17\n",
      "Epoch  13/100 Batch 1140/1562 - Loss:  1.137, Seconds: 2.87\n",
      "Epoch  13/100 Batch 1160/1562 - Loss:  1.162, Seconds: 2.93\n",
      "Epoch  13/100 Batch 1180/1562 - Loss:  1.162, Seconds: 2.13\n",
      "Epoch  13/100 Batch 1200/1562 - Loss:  1.089, Seconds: 3.90\n",
      "Epoch  13/100 Batch 1220/1562 - Loss:  1.120, Seconds: 2.91\n",
      "Epoch  13/100 Batch 1240/1562 - Loss:  1.099, Seconds: 2.26\n",
      "Epoch  13/100 Batch 1260/1562 - Loss:  1.128, Seconds: 2.61\n",
      "Epoch  13/100 Batch 1280/1562 - Loss:  1.179, Seconds: 2.97\n",
      "Epoch  13/100 Batch 1300/1562 - Loss:  1.198, Seconds: 2.69\n",
      "Epoch  13/100 Batch 1320/1562 - Loss:  1.119, Seconds: 2.32\n",
      "Epoch  13/100 Batch 1340/1562 - Loss:  1.111, Seconds: 3.35\n",
      "Epoch  13/100 Batch 1360/1562 - Loss:  1.129, Seconds: 2.58\n",
      "Epoch  13/100 Batch 1380/1562 - Loss:  1.095, Seconds: 2.42\n",
      "Epoch  13/100 Batch 1400/1562 - Loss:  1.060, Seconds: 3.09\n",
      "Epoch  13/100 Batch 1420/1562 - Loss:  1.104, Seconds: 2.71\n",
      "Epoch  13/100 Batch 1440/1562 - Loss:  1.110, Seconds: 3.18\n",
      "Epoch  13/100 Batch 1460/1562 - Loss:  1.111, Seconds: 2.50\n",
      "Epoch  13/100 Batch 1480/1562 - Loss:  1.140, Seconds: 2.90\n",
      "Epoch  13/100 Batch 1500/1562 - Loss:  1.144, Seconds: 2.32\n",
      "Epoch  13/100 Batch 1520/1562 - Loss:  1.166, Seconds: 2.61\n",
      "Epoch  13/100 Batch 1540/1562 - Loss:  1.122, Seconds: 2.43\n",
      "('Average loss for this update:', 1.122)\n",
      "New Record!\n",
      "Epoch  13/100 Batch 1560/1562 - Loss:  1.111, Seconds: 2.67\n",
      "Epoch  14/100 Batch   20/1562 - Loss:  1.117, Seconds: 2.62\n",
      "Epoch  14/100 Batch   40/1562 - Loss:  1.106, Seconds: 2.03\n",
      "Epoch  14/100 Batch   60/1562 - Loss:  1.086, Seconds: 2.41\n",
      "Epoch  14/100 Batch   80/1562 - Loss:  1.123, Seconds: 3.08\n",
      "Epoch  14/100 Batch  100/1562 - Loss:  1.162, Seconds: 3.05\n",
      "Epoch  14/100 Batch  120/1562 - Loss:  1.111, Seconds: 3.11\n",
      "Epoch  14/100 Batch  140/1562 - Loss:  1.140, Seconds: 4.11\n",
      "Epoch  14/100 Batch  160/1562 - Loss:  1.099, Seconds: 2.34\n",
      "Epoch  14/100 Batch  180/1562 - Loss:  1.120, Seconds: 3.25\n",
      "Epoch  14/100 Batch  200/1562 - Loss:  1.088, Seconds: 2.50\n",
      "Epoch  14/100 Batch  220/1562 - Loss:  1.124, Seconds: 2.00\n",
      "Epoch  14/100 Batch  240/1562 - Loss:  1.115, Seconds: 2.68\n",
      "Epoch  14/100 Batch  260/1562 - Loss:  1.146, Seconds: 2.43\n",
      "Epoch  14/100 Batch  280/1562 - Loss:  1.031, Seconds: 3.64\n",
      "Epoch  14/100 Batch  300/1562 - Loss:  1.158, Seconds: 2.97\n",
      "Epoch  14/100 Batch  320/1562 - Loss:  1.165, Seconds: 2.87\n",
      "Epoch  14/100 Batch  340/1562 - Loss:  1.096, Seconds: 2.74\n",
      "Epoch  14/100 Batch  360/1562 - Loss:  1.082, Seconds: 3.95\n",
      "Epoch  14/100 Batch  380/1562 - Loss:  1.043, Seconds: 3.28\n",
      "Epoch  14/100 Batch  400/1562 - Loss:  1.165, Seconds: 3.26\n",
      "Epoch  14/100 Batch  420/1562 - Loss:  1.129, Seconds: 2.71\n",
      "Epoch  14/100 Batch  440/1562 - Loss:  1.106, Seconds: 2.33\n",
      "Epoch  14/100 Batch  460/1562 - Loss:  1.087, Seconds: 2.88\n",
      "Epoch  14/100 Batch  480/1562 - Loss:  1.127, Seconds: 2.32\n",
      "Epoch  14/100 Batch  500/1562 - Loss:  1.091, Seconds: 3.21\n",
      "('Average loss for this update:', 1.112)\n",
      "New Record!\n",
      "Epoch  14/100 Batch  520/1562 - Loss:  1.093, Seconds: 2.79\n",
      "Epoch  14/100 Batch  540/1562 - Loss:  1.069, Seconds: 3.32\n",
      "Epoch  14/100 Batch  560/1562 - Loss:  1.087, Seconds: 2.97\n",
      "Epoch  14/100 Batch  580/1562 - Loss:  1.056, Seconds: 3.13\n",
      "Epoch  14/100 Batch  600/1562 - Loss:  1.059, Seconds: 2.35\n",
      "Epoch  14/100 Batch  620/1562 - Loss:  1.097, Seconds: 2.45\n",
      "Epoch  14/100 Batch  640/1562 - Loss:  1.071, Seconds: 3.35\n",
      "Epoch  14/100 Batch  660/1562 - Loss:  1.082, Seconds: 4.50\n",
      "Epoch  14/100 Batch  680/1562 - Loss:  1.065, Seconds: 2.13\n",
      "Epoch  14/100 Batch  700/1562 - Loss:  1.084, Seconds: 3.42\n",
      "Epoch  14/100 Batch  720/1562 - Loss:  1.120, Seconds: 2.46\n",
      "Epoch  14/100 Batch  740/1562 - Loss:  1.063, Seconds: 2.69\n",
      "Epoch  14/100 Batch  760/1562 - Loss:  1.098, Seconds: 1.97\n",
      "Epoch  14/100 Batch  780/1562 - Loss:  1.041, Seconds: 4.49\n",
      "Epoch  14/100 Batch  800/1562 - Loss:  1.086, Seconds: 3.06\n",
      "Epoch  14/100 Batch  820/1562 - Loss:  1.118, Seconds: 1.86\n",
      "Epoch  14/100 Batch  840/1562 - Loss:  1.091, Seconds: 2.13\n",
      "Epoch  14/100 Batch  860/1562 - Loss:  1.087, Seconds: 2.90\n",
      "Epoch  14/100 Batch  880/1562 - Loss:  1.067, Seconds: 2.91\n",
      "Epoch  14/100 Batch  900/1562 - Loss:  1.081, Seconds: 4.03\n",
      "Epoch  14/100 Batch  920/1562 - Loss:  1.111, Seconds: 2.25\n",
      "Epoch  14/100 Batch  940/1562 - Loss:  1.084, Seconds: 2.61\n",
      "Epoch  14/100 Batch  960/1562 - Loss:  1.072, Seconds: 2.72\n",
      "Epoch  14/100 Batch  980/1562 - Loss:  1.077, Seconds: 2.50\n",
      "Epoch  14/100 Batch 1000/1562 - Loss:  1.045, Seconds: 2.90\n",
      "Epoch  14/100 Batch 1020/1562 - Loss:  1.080, Seconds: 3.38\n",
      "('Average loss for this update:', 1.081)\n",
      "New Record!\n",
      "Epoch  14/100 Batch 1040/1562 - Loss:  1.107, Seconds: 2.65\n",
      "Epoch  14/100 Batch 1060/1562 - Loss:  1.040, Seconds: 3.60\n",
      "Epoch  14/100 Batch 1080/1562 - Loss:  1.087, Seconds: 2.35\n",
      "Epoch  14/100 Batch 1100/1562 - Loss:  1.046, Seconds: 2.99\n",
      "Epoch  14/100 Batch 1120/1562 - Loss:  1.047, Seconds: 2.15\n",
      "Epoch  14/100 Batch 1140/1562 - Loss:  1.110, Seconds: 2.90\n",
      "Epoch  14/100 Batch 1160/1562 - Loss:  1.124, Seconds: 2.87\n",
      "Epoch  14/100 Batch 1180/1562 - Loss:  1.120, Seconds: 2.13\n",
      "Epoch  14/100 Batch 1200/1562 - Loss:  1.052, Seconds: 3.91\n",
      "Epoch  14/100 Batch 1220/1562 - Loss:  1.087, Seconds: 2.92\n",
      "Epoch  14/100 Batch 1240/1562 - Loss:  1.063, Seconds: 2.25\n",
      "Epoch  14/100 Batch 1260/1562 - Loss:  1.092, Seconds: 2.64\n",
      "Epoch  14/100 Batch 1280/1562 - Loss:  1.135, Seconds: 3.01\n",
      "Epoch  14/100 Batch 1300/1562 - Loss:  1.145, Seconds: 2.69\n",
      "Epoch  14/100 Batch 1320/1562 - Loss:  1.072, Seconds: 2.57\n",
      "Epoch  14/100 Batch 1340/1562 - Loss:  1.069, Seconds: 3.34\n",
      "Epoch  14/100 Batch 1360/1562 - Loss:  1.086, Seconds: 2.53\n",
      "Epoch  14/100 Batch 1380/1562 - Loss:  1.057, Seconds: 2.46\n",
      "Epoch  14/100 Batch 1400/1562 - Loss:  1.025, Seconds: 3.11\n",
      "Epoch  14/100 Batch 1420/1562 - Loss:  1.070, Seconds: 2.77\n",
      "Epoch  14/100 Batch 1440/1562 - Loss:  1.067, Seconds: 3.19\n",
      "Epoch  14/100 Batch 1460/1562 - Loss:  1.065, Seconds: 2.53\n",
      "Epoch  14/100 Batch 1480/1562 - Loss:  1.089, Seconds: 2.90\n",
      "Epoch  14/100 Batch 1500/1562 - Loss:  1.105, Seconds: 2.35\n",
      "Epoch  14/100 Batch 1520/1562 - Loss:  1.129, Seconds: 2.60\n",
      "Epoch  14/100 Batch 1540/1562 - Loss:  1.072, Seconds: 2.43\n",
      "('Average loss for this update:', 1.082)\n",
      "No Improvement.\n",
      "Epoch  14/100 Batch 1560/1562 - Loss:  1.056, Seconds: 2.62\n",
      "Epoch  15/100 Batch   20/1562 - Loss:  1.075, Seconds: 2.60\n",
      "Epoch  15/100 Batch   40/1562 - Loss:  1.062, Seconds: 2.03\n",
      "Epoch  15/100 Batch   60/1562 - Loss:  1.032, Seconds: 2.43\n",
      "Epoch  15/100 Batch   80/1562 - Loss:  1.087, Seconds: 3.05\n",
      "Epoch  15/100 Batch  100/1562 - Loss:  1.119, Seconds: 3.09\n",
      "Epoch  15/100 Batch  120/1562 - Loss:  1.063, Seconds: 3.05\n",
      "Epoch  15/100 Batch  140/1562 - Loss:  1.105, Seconds: 4.03\n",
      "Epoch  15/100 Batch  160/1562 - Loss:  1.064, Seconds: 2.36\n",
      "Epoch  15/100 Batch  180/1562 - Loss:  1.086, Seconds: 3.36\n",
      "Epoch  15/100 Batch  200/1562 - Loss:  1.051, Seconds: 2.53\n",
      "Epoch  15/100 Batch  220/1562 - Loss:  1.087, Seconds: 2.24\n",
      "Epoch  15/100 Batch  240/1562 - Loss:  1.072, Seconds: 2.74\n",
      "Epoch  15/100 Batch  260/1562 - Loss:  1.108, Seconds: 2.41\n",
      "Epoch  15/100 Batch  280/1562 - Loss:  0.993, Seconds: 3.62\n",
      "Epoch  15/100 Batch  300/1562 - Loss:  1.109, Seconds: 2.98\n",
      "Epoch  15/100 Batch  320/1562 - Loss:  1.125, Seconds: 2.88\n",
      "Epoch  15/100 Batch  340/1562 - Loss:  1.065, Seconds: 2.68\n",
      "Epoch  15/100 Batch  360/1562 - Loss:  1.041, Seconds: 3.81\n",
      "Epoch  15/100 Batch  380/1562 - Loss:  1.005, Seconds: 3.28\n",
      "Epoch  15/100 Batch  400/1562 - Loss:  1.111, Seconds: 3.26\n",
      "Epoch  15/100 Batch  420/1562 - Loss:  1.082, Seconds: 2.73\n",
      "Epoch  15/100 Batch  440/1562 - Loss:  1.075, Seconds: 2.32\n",
      "Epoch  15/100 Batch  460/1562 - Loss:  1.055, Seconds: 2.90\n",
      "Epoch  15/100 Batch  480/1562 - Loss:  1.086, Seconds: 2.33\n",
      "Epoch  15/100 Batch  500/1562 - Loss:  1.052, Seconds: 3.18\n",
      "('Average loss for this update:', 1.071)\n",
      "New Record!\n",
      "Epoch  15/100 Batch  520/1562 - Loss:  1.041, Seconds: 10.24\n",
      "Epoch  15/100 Batch  540/1562 - Loss:  1.018, Seconds: 3.40\n",
      "Epoch  15/100 Batch  560/1562 - Loss:  1.057, Seconds: 2.97\n",
      "Epoch  15/100 Batch  580/1562 - Loss:  1.005, Seconds: 3.07\n",
      "Epoch  15/100 Batch  600/1562 - Loss:  1.014, Seconds: 2.33\n",
      "Epoch  15/100 Batch  620/1562 - Loss:  1.060, Seconds: 2.41\n",
      "Epoch  15/100 Batch  640/1562 - Loss:  1.037, Seconds: 3.25\n",
      "Epoch  15/100 Batch  660/1562 - Loss:  1.047, Seconds: 4.49\n",
      "Epoch  15/100 Batch  680/1562 - Loss:  1.022, Seconds: 2.11\n",
      "Epoch  15/100 Batch  700/1562 - Loss:  1.056, Seconds: 3.44\n",
      "Epoch  15/100 Batch  720/1562 - Loss:  1.082, Seconds: 2.40\n",
      "Epoch  15/100 Batch  740/1562 - Loss:  1.031, Seconds: 2.71\n",
      "Epoch  15/100 Batch  760/1562 - Loss:  1.055, Seconds: 1.96\n",
      "Epoch  15/100 Batch  780/1562 - Loss:  1.000, Seconds: 4.46\n",
      "Epoch  15/100 Batch  800/1562 - Loss:  1.042, Seconds: 3.10\n",
      "Epoch  15/100 Batch  820/1562 - Loss:  1.081, Seconds: 1.86\n",
      "Epoch  15/100 Batch  840/1562 - Loss:  1.035, Seconds: 2.14\n",
      "Epoch  15/100 Batch  860/1562 - Loss:  1.035, Seconds: 2.90\n",
      "Epoch  15/100 Batch  880/1562 - Loss:  1.035, Seconds: 2.55\n",
      "Epoch  15/100 Batch  900/1562 - Loss:  1.052, Seconds: 3.36\n",
      "Epoch  15/100 Batch  920/1562 - Loss:  1.070, Seconds: 2.24\n",
      "Epoch  15/100 Batch  940/1562 - Loss:  1.057, Seconds: 2.60\n",
      "Epoch  15/100 Batch  960/1562 - Loss:  1.035, Seconds: 2.66\n",
      "Epoch  15/100 Batch  980/1562 - Loss:  1.046, Seconds: 2.50\n",
      "Epoch  15/100 Batch 1000/1562 - Loss:  1.011, Seconds: 2.89\n",
      "Epoch  15/100 Batch 1020/1562 - Loss:  1.049, Seconds: 2.89\n",
      "('Average loss for this update:', 1.042)\n",
      "New Record!\n",
      "Epoch  15/100 Batch 1040/1562 - Loss:  1.054, Seconds: 2.67\n",
      "Epoch  15/100 Batch 1060/1562 - Loss:  1.005, Seconds: 3.65\n",
      "Epoch  15/100 Batch 1080/1562 - Loss:  1.045, Seconds: 2.34\n",
      "Epoch  15/100 Batch 1100/1562 - Loss:  1.009, Seconds: 2.99\n",
      "Epoch  15/100 Batch 1120/1562 - Loss:  1.005, Seconds: 2.14\n",
      "Epoch  15/100 Batch 1140/1562 - Loss:  1.063, Seconds: 2.87\n",
      "Epoch  15/100 Batch 1160/1562 - Loss:  1.082, Seconds: 2.85\n",
      "Epoch  15/100 Batch 1180/1562 - Loss:  1.083, Seconds: 2.20\n",
      "Epoch  15/100 Batch 1200/1562 - Loss:  1.018, Seconds: 3.90\n",
      "Epoch  15/100 Batch 1220/1562 - Loss:  1.042, Seconds: 2.89\n",
      "Epoch  15/100 Batch 1240/1562 - Loss:  1.027, Seconds: 2.24\n",
      "Epoch  15/100 Batch 1260/1562 - Loss:  1.048, Seconds: 2.63\n",
      "Epoch  15/100 Batch 1280/1562 - Loss:  1.087, Seconds: 2.98\n",
      "Epoch  15/100 Batch 1300/1562 - Loss:  1.108, Seconds: 2.71\n",
      "Epoch  15/100 Batch 1320/1562 - Loss:  1.036, Seconds: 2.32\n",
      "Epoch  15/100 Batch 1340/1562 - Loss:  1.028, Seconds: 3.36\n",
      "Epoch  15/100 Batch 1360/1562 - Loss:  1.040, Seconds: 2.57\n",
      "Epoch  15/100 Batch 1380/1562 - Loss:  1.021, Seconds: 2.50\n",
      "Epoch  15/100 Batch 1400/1562 - Loss:  0.990, Seconds: 3.10\n",
      "Epoch  15/100 Batch 1420/1562 - Loss:  1.031, Seconds: 2.69\n",
      "Epoch  15/100 Batch 1440/1562 - Loss:  1.026, Seconds: 3.22\n",
      "Epoch  15/100 Batch 1460/1562 - Loss:  1.031, Seconds: 2.52\n",
      "Epoch  15/100 Batch 1480/1562 - Loss:  1.053, Seconds: 2.91\n",
      "Epoch  15/100 Batch 1500/1562 - Loss:  1.052, Seconds: 2.35\n",
      "Epoch  15/100 Batch 1520/1562 - Loss:  1.085, Seconds: 2.63\n",
      "Epoch  15/100 Batch 1540/1562 - Loss:  1.034, Seconds: 2.42\n",
      "('Average loss for this update:', 1.042)\n",
      "New Record!\n",
      "Epoch  15/100 Batch 1560/1562 - Loss:  1.034, Seconds: 2.65\n",
      "Epoch  16/100 Batch   20/1562 - Loss:  1.036, Seconds: 2.60\n",
      "Epoch  16/100 Batch   40/1562 - Loss:  1.026, Seconds: 2.04\n",
      "Epoch  16/100 Batch   60/1562 - Loss:  1.009, Seconds: 2.43\n",
      "Epoch  16/100 Batch   80/1562 - Loss:  1.058, Seconds: 3.19\n",
      "Epoch  16/100 Batch  100/1562 - Loss:  1.073, Seconds: 3.06\n",
      "Epoch  16/100 Batch  120/1562 - Loss:  1.020, Seconds: 3.07\n",
      "Epoch  16/100 Batch  140/1562 - Loss:  1.057, Seconds: 4.00\n",
      "Epoch  16/100 Batch  160/1562 - Loss:  1.038, Seconds: 2.32\n",
      "Epoch  16/100 Batch  180/1562 - Loss:  1.054, Seconds: 3.24\n",
      "Epoch  16/100 Batch  200/1562 - Loss:  1.007, Seconds: 2.54\n",
      "Epoch  16/100 Batch  220/1562 - Loss:  1.047, Seconds: 1.95\n",
      "Epoch  16/100 Batch  240/1562 - Loss:  1.040, Seconds: 2.72\n",
      "Epoch  16/100 Batch  260/1562 - Loss:  1.057, Seconds: 2.41\n",
      "Epoch  16/100 Batch  280/1562 - Loss:  0.964, Seconds: 3.63\n",
      "Epoch  16/100 Batch  300/1562 - Loss:  1.075, Seconds: 3.02\n",
      "Epoch  16/100 Batch  320/1562 - Loss:  1.090, Seconds: 2.94\n",
      "Epoch  16/100 Batch  340/1562 - Loss:  1.029, Seconds: 2.69\n",
      "Epoch  16/100 Batch  360/1562 - Loss:  1.012, Seconds: 3.80\n",
      "Epoch  16/100 Batch  380/1562 - Loss:  0.974, Seconds: 3.30\n",
      "Epoch  16/100 Batch  400/1562 - Loss:  1.090, Seconds: 3.26\n",
      "Epoch  16/100 Batch  420/1562 - Loss:  1.050, Seconds: 2.70\n",
      "Epoch  16/100 Batch  440/1562 - Loss:  1.039, Seconds: 2.32\n",
      "Epoch  16/100 Batch  460/1562 - Loss:  1.014, Seconds: 2.89\n",
      "Epoch  16/100 Batch  480/1562 - Loss:  1.055, Seconds: 2.36\n",
      "Epoch  16/100 Batch  500/1562 - Loss:  1.025, Seconds: 3.22\n",
      "('Average loss for this update:', 1.037)\n",
      "New Record!\n",
      "Epoch  16/100 Batch  520/1562 - Loss:  1.015, Seconds: 2.69\n",
      "Epoch  16/100 Batch  540/1562 - Loss:  0.994, Seconds: 3.37\n",
      "Epoch  16/100 Batch  560/1562 - Loss:  1.022, Seconds: 2.96\n",
      "Epoch  16/100 Batch  580/1562 - Loss:  0.968, Seconds: 3.06\n",
      "Epoch  16/100 Batch  600/1562 - Loss:  0.968, Seconds: 2.33\n",
      "Epoch  16/100 Batch  620/1562 - Loss:  1.035, Seconds: 2.40\n",
      "Epoch  16/100 Batch  640/1562 - Loss:  1.009, Seconds: 3.26\n",
      "Epoch  16/100 Batch  660/1562 - Loss:  1.013, Seconds: 4.52\n",
      "Epoch  16/100 Batch  680/1562 - Loss:  0.984, Seconds: 2.15\n",
      "Epoch  16/100 Batch  700/1562 - Loss:  1.011, Seconds: 3.45\n",
      "Epoch  16/100 Batch  720/1562 - Loss:  1.046, Seconds: 2.43\n",
      "Epoch  16/100 Batch  740/1562 - Loss:  0.997, Seconds: 2.71\n",
      "Epoch  16/100 Batch  760/1562 - Loss:  1.030, Seconds: 1.97\n",
      "Epoch  16/100 Batch  780/1562 - Loss:  0.977, Seconds: 4.46\n",
      "Epoch  16/100 Batch  800/1562 - Loss:  1.019, Seconds: 3.10\n",
      "Epoch  16/100 Batch  820/1562 - Loss:  1.041, Seconds: 1.87\n",
      "Epoch  16/100 Batch  840/1562 - Loss:  1.018, Seconds: 2.13\n",
      "Epoch  16/100 Batch  860/1562 - Loss:  1.007, Seconds: 2.90\n",
      "Epoch  16/100 Batch  880/1562 - Loss:  0.988, Seconds: 2.51\n",
      "Epoch  16/100 Batch  900/1562 - Loss:  1.018, Seconds: 3.35\n",
      "Epoch  16/100 Batch  920/1562 - Loss:  1.042, Seconds: 2.22\n",
      "Epoch  16/100 Batch  940/1562 - Loss:  1.014, Seconds: 2.60\n",
      "Epoch  16/100 Batch  960/1562 - Loss:  0.990, Seconds: 2.64\n",
      "Epoch  16/100 Batch  980/1562 - Loss:  1.003, Seconds: 2.52\n",
      "Epoch  16/100 Batch 1000/1562 - Loss:  0.975, Seconds: 3.01\n",
      "Epoch  16/100 Batch 1020/1562 - Loss:  1.012, Seconds: 2.92\n",
      "('Average loss for this update:', 1.008)\n",
      "New Record!\n",
      "Epoch  16/100 Batch 1040/1562 - Loss:  1.027, Seconds: 2.69\n",
      "Epoch  16/100 Batch 1060/1562 - Loss:  0.969, Seconds: 3.75\n",
      "Epoch  16/100 Batch 1080/1562 - Loss:  1.015, Seconds: 2.31\n",
      "Epoch  16/100 Batch 1100/1562 - Loss:  0.981, Seconds: 2.98\n",
      "Epoch  16/100 Batch 1120/1562 - Loss:  0.971, Seconds: 2.15\n",
      "Epoch  16/100 Batch 1140/1562 - Loss:  1.029, Seconds: 2.89\n",
      "Epoch  16/100 Batch 1160/1562 - Loss:  1.053, Seconds: 2.93\n",
      "Epoch  16/100 Batch 1180/1562 - Loss:  1.041, Seconds: 2.13\n",
      "Epoch  16/100 Batch 1200/1562 - Loss:  0.989, Seconds: 3.92\n",
      "Epoch  16/100 Batch 1220/1562 - Loss:  1.016, Seconds: 2.90\n",
      "Epoch  16/100 Batch 1240/1562 - Loss:  0.986, Seconds: 2.25\n",
      "Epoch  16/100 Batch 1260/1562 - Loss:  1.015, Seconds: 2.63\n",
      "Epoch  16/100 Batch 1280/1562 - Loss:  1.034, Seconds: 3.00\n",
      "Epoch  16/100 Batch 1300/1562 - Loss:  1.067, Seconds: 2.72\n",
      "Epoch  16/100 Batch 1320/1562 - Loss:  1.007, Seconds: 2.54\n",
      "Epoch  16/100 Batch 1340/1562 - Loss:  0.989, Seconds: 3.35\n",
      "Epoch  16/100 Batch 1360/1562 - Loss:  1.002, Seconds: 2.53\n",
      "Epoch  16/100 Batch 1380/1562 - Loss:  0.983, Seconds: 2.45\n",
      "Epoch  16/100 Batch 1400/1562 - Loss:  0.957, Seconds: 3.11\n",
      "Epoch  16/100 Batch 1420/1562 - Loss:  0.994, Seconds: 2.69\n",
      "Epoch  16/100 Batch 1440/1562 - Loss:  0.986, Seconds: 3.19\n",
      "Epoch  16/100 Batch 1460/1562 - Loss:  0.989, Seconds: 2.49\n",
      "Epoch  16/100 Batch 1480/1562 - Loss:  1.018, Seconds: 2.89\n",
      "Epoch  16/100 Batch 1500/1562 - Loss:  1.005, Seconds: 2.34\n",
      "Epoch  16/100 Batch 1520/1562 - Loss:  1.045, Seconds: 2.65\n",
      "Epoch  16/100 Batch 1540/1562 - Loss:  1.001, Seconds: 2.41\n",
      "('Average loss for this update:', 1.005)\n",
      "New Record!\n",
      "Epoch  16/100 Batch 1560/1562 - Loss:  0.994, Seconds: 2.77\n",
      "Epoch  17/100 Batch   20/1562 - Loss:  1.012, Seconds: 2.65\n",
      "Epoch  17/100 Batch   40/1562 - Loss:  0.989, Seconds: 2.03\n",
      "Epoch  17/100 Batch   60/1562 - Loss:  0.975, Seconds: 2.43\n",
      "Epoch  17/100 Batch   80/1562 - Loss:  1.017, Seconds: 3.11\n",
      "Epoch  17/100 Batch  100/1562 - Loss:  1.032, Seconds: 3.05\n",
      "Epoch  17/100 Batch  120/1562 - Loss:  0.998, Seconds: 3.10\n",
      "Epoch  17/100 Batch  140/1562 - Loss:  1.025, Seconds: 4.01\n",
      "Epoch  17/100 Batch  160/1562 - Loss:  0.998, Seconds: 2.35\n",
      "Epoch  17/100 Batch  180/1562 - Loss:  1.012, Seconds: 3.24\n",
      "Epoch  17/100 Batch  200/1562 - Loss:  0.983, Seconds: 2.50\n",
      "Epoch  17/100 Batch  220/1562 - Loss:  1.017, Seconds: 1.95\n",
      "Epoch  17/100 Batch  240/1562 - Loss:  1.008, Seconds: 2.72\n",
      "Epoch  17/100 Batch  260/1562 - Loss:  1.032, Seconds: 2.52\n",
      "Epoch  17/100 Batch  280/1562 - Loss:  0.923, Seconds: 3.63\n",
      "Epoch  17/100 Batch  300/1562 - Loss:  1.030, Seconds: 3.00\n",
      "Epoch  17/100 Batch  320/1562 - Loss:  1.052, Seconds: 2.88\n",
      "Epoch  17/100 Batch  340/1562 - Loss:  1.000, Seconds: 2.69\n",
      "Epoch  17/100 Batch  360/1562 - Loss:  0.983, Seconds: 3.83\n",
      "Epoch  17/100 Batch  380/1562 - Loss:  0.939, Seconds: 3.27\n",
      "Epoch  17/100 Batch  400/1562 - Loss:  1.055, Seconds: 3.27\n",
      "Epoch  17/100 Batch  420/1562 - Loss:  1.013, Seconds: 2.70\n",
      "Epoch  17/100 Batch  440/1562 - Loss:  1.001, Seconds: 2.34\n",
      "Epoch  17/100 Batch  460/1562 - Loss:  0.985, Seconds: 2.86\n",
      "Epoch  17/100 Batch  480/1562 - Loss:  1.022, Seconds: 2.32\n",
      "Epoch  17/100 Batch  500/1562 - Loss:  0.986, Seconds: 3.17\n",
      "('Average loss for this update:', 1.003)\n",
      "New Record!\n",
      "Epoch  17/100 Batch  520/1562 - Loss:  0.989, Seconds: 2.84\n",
      "Epoch  17/100 Batch  540/1562 - Loss:  0.964, Seconds: 3.43\n",
      "Epoch  17/100 Batch  560/1562 - Loss:  0.989, Seconds: 2.99\n",
      "Epoch  17/100 Batch  580/1562 - Loss:  0.935, Seconds: 3.08\n",
      "Epoch  17/100 Batch  600/1562 - Loss:  0.949, Seconds: 2.32\n",
      "Epoch  17/100 Batch  620/1562 - Loss:  0.989, Seconds: 2.42\n",
      "Epoch  17/100 Batch  640/1562 - Loss:  0.974, Seconds: 3.29\n",
      "Epoch  17/100 Batch  660/1562 - Loss:  0.979, Seconds: 4.46\n",
      "Epoch  17/100 Batch  680/1562 - Loss:  0.956, Seconds: 2.13\n",
      "Epoch  17/100 Batch  700/1562 - Loss:  0.992, Seconds: 3.51\n",
      "Epoch  17/100 Batch  720/1562 - Loss:  1.013, Seconds: 2.51\n",
      "Epoch  17/100 Batch  740/1562 - Loss:  0.961, Seconds: 2.70\n",
      "Epoch  17/100 Batch  760/1562 - Loss:  0.993, Seconds: 1.95\n",
      "Epoch  17/100 Batch  780/1562 - Loss:  0.943, Seconds: 4.42\n",
      "Epoch  17/100 Batch  800/1562 - Loss:  0.978, Seconds: 3.06\n",
      "Epoch  17/100 Batch  820/1562 - Loss:  1.004, Seconds: 1.88\n",
      "Epoch  17/100 Batch  840/1562 - Loss:  0.990, Seconds: 2.13\n",
      "Epoch  17/100 Batch  860/1562 - Loss:  0.969, Seconds: 2.89\n",
      "Epoch  17/100 Batch  880/1562 - Loss:  0.959, Seconds: 2.53\n",
      "Epoch  17/100 Batch  900/1562 - Loss:  0.978, Seconds: 3.35\n",
      "Epoch  17/100 Batch  920/1562 - Loss:  1.007, Seconds: 2.26\n",
      "Epoch  17/100 Batch  940/1562 - Loss:  0.992, Seconds: 2.60\n",
      "Epoch  17/100 Batch  960/1562 - Loss:  0.955, Seconds: 2.60\n",
      "Epoch  17/100 Batch  980/1562 - Loss:  0.974, Seconds: 2.51\n",
      "Epoch  17/100 Batch 1000/1562 - Loss:  0.947, Seconds: 2.89\n",
      "Epoch  17/100 Batch 1020/1562 - Loss:  0.983, Seconds: 2.87\n",
      "('Average loss for this update:', 0.975)\n",
      "New Record!\n",
      "Epoch  17/100 Batch 1040/1562 - Loss:  0.982, Seconds: 2.62\n",
      "Epoch  17/100 Batch 1060/1562 - Loss:  0.937, Seconds: 3.66\n",
      "Epoch  17/100 Batch 1080/1562 - Loss:  0.973, Seconds: 2.33\n",
      "Epoch  17/100 Batch 1100/1562 - Loss:  0.946, Seconds: 2.96\n",
      "Epoch  17/100 Batch 1120/1562 - Loss:  0.936, Seconds: 2.13\n",
      "Epoch  17/100 Batch 1140/1562 - Loss:  0.991, Seconds: 2.87\n",
      "Epoch  17/100 Batch 1160/1562 - Loss:  1.026, Seconds: 2.88\n",
      "Epoch  17/100 Batch 1180/1562 - Loss:  1.005, Seconds: 2.14\n",
      "Epoch  17/100 Batch 1200/1562 - Loss:  0.952, Seconds: 3.90\n",
      "Epoch  17/100 Batch 1220/1562 - Loss:  0.981, Seconds: 2.88\n",
      "Epoch  17/100 Batch 1240/1562 - Loss:  0.954, Seconds: 2.29\n",
      "Epoch  17/100 Batch 1260/1562 - Loss:  0.986, Seconds: 2.62\n",
      "Epoch  17/100 Batch 1280/1562 - Loss:  1.000, Seconds: 3.00\n",
      "Epoch  17/100 Batch 1300/1562 - Loss:  1.022, Seconds: 2.70\n",
      "Epoch  17/100 Batch 1320/1562 - Loss:  0.967, Seconds: 2.36\n",
      "Epoch  17/100 Batch 1340/1562 - Loss:  0.966, Seconds: 3.45\n",
      "Epoch  17/100 Batch 1360/1562 - Loss:  0.973, Seconds: 2.52\n",
      "Epoch  17/100 Batch 1380/1562 - Loss:  0.947, Seconds: 2.42\n",
      "Epoch  17/100 Batch 1400/1562 - Loss:  0.920, Seconds: 3.09\n",
      "Epoch  17/100 Batch 1420/1562 - Loss:  0.969, Seconds: 2.72\n",
      "Epoch  17/100 Batch 1440/1562 - Loss:  0.943, Seconds: 3.17\n",
      "Epoch  17/100 Batch 1460/1562 - Loss:  0.955, Seconds: 2.55\n",
      "Epoch  17/100 Batch 1480/1562 - Loss:  0.970, Seconds: 2.88\n",
      "Epoch  17/100 Batch 1500/1562 - Loss:  0.971, Seconds: 2.38\n",
      "Epoch  17/100 Batch 1520/1562 - Loss:  1.018, Seconds: 2.63\n",
      "Epoch  17/100 Batch 1540/1562 - Loss:  0.970, Seconds: 2.45\n",
      "('Average loss for this update:', 0.971)\n",
      "New Record!\n",
      "Epoch  17/100 Batch 1560/1562 - Loss:  0.968, Seconds: 2.63\n",
      "Epoch  18/100 Batch   20/1562 - Loss:  0.978, Seconds: 2.61\n",
      "Epoch  18/100 Batch   40/1562 - Loss:  0.952, Seconds: 2.06\n",
      "Epoch  18/100 Batch   60/1562 - Loss:  0.943, Seconds: 2.46\n",
      "Epoch  18/100 Batch   80/1562 - Loss:  0.990, Seconds: 3.05\n",
      "Epoch  18/100 Batch  100/1562 - Loss:  1.013, Seconds: 3.12\n",
      "Epoch  18/100 Batch  120/1562 - Loss:  0.954, Seconds: 3.07\n",
      "Epoch  18/100 Batch  140/1562 - Loss:  0.995, Seconds: 4.05\n",
      "Epoch  18/100 Batch  160/1562 - Loss:  0.963, Seconds: 2.33\n",
      "Epoch  18/100 Batch  180/1562 - Loss:  0.990, Seconds: 3.29\n",
      "Epoch  18/100 Batch  200/1562 - Loss:  0.952, Seconds: 2.57\n",
      "Epoch  18/100 Batch  220/1562 - Loss:  0.981, Seconds: 1.96\n",
      "Epoch  18/100 Batch  240/1562 - Loss:  0.973, Seconds: 2.70\n",
      "Epoch  18/100 Batch  260/1562 - Loss:  1.001, Seconds: 2.45\n",
      "Epoch  18/100 Batch  280/1562 - Loss:  0.903, Seconds: 3.63\n",
      "Epoch  18/100 Batch  300/1562 - Loss:  0.999, Seconds: 2.97\n",
      "Epoch  18/100 Batch  320/1562 - Loss:  1.006, Seconds: 2.90\n",
      "Epoch  18/100 Batch  340/1562 - Loss:  0.971, Seconds: 2.69\n",
      "Epoch  18/100 Batch  360/1562 - Loss:  0.946, Seconds: 3.92\n",
      "Epoch  18/100 Batch  380/1562 - Loss:  0.910, Seconds: 3.29\n",
      "Epoch  18/100 Batch  400/1562 - Loss:  1.005, Seconds: 3.50\n",
      "Epoch  18/100 Batch  420/1562 - Loss:  0.985, Seconds: 2.69\n"
     ]
    }
   ],
   "source": [
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allocator_type ='BFC'\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.80\n",
    "\n",
    "with tf.Session(graph=train_graph) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    \n",
    "    # If we want to continue training a previous session\n",
    "    #loader = tf.train.import_meta_graph(\"./\" + checkpoint + '.meta')\n",
    "    #loader.restore(sess, checkpoint)\n",
    "    \n",
    "    for epoch_i in range(1, epochs+1):\n",
    "        update_loss = 0\n",
    "        batch_loss = 0\n",
    "        for batch_i, (y_batch, X_batch, y_lengths, X_lengths) in enumerate(\n",
    "                get_batches(sorted_y_short, sorted_X_short, batch_size)):\n",
    "            start_time = time.time()\n",
    "            _, loss = sess.run(\n",
    "                [train_op, cost],\n",
    "                {input_data: X_batch,\n",
    "                 targets: y_batch,\n",
    "                 lr: learning_rate,\n",
    "                 y_length: y_lengths,\n",
    "                 X_length: X_lengths,\n",
    "                 keep_prob: keep_probability})\n",
    "\n",
    "            batch_loss += loss\n",
    "            update_loss += loss\n",
    "            end_time = time.time()\n",
    "            batch_time = end_time - start_time\n",
    "\n",
    "            if batch_i % display_step == 0 and batch_i > 0:\n",
    "                print('Epoch {:>3}/{} Batch {:>4}/{} - Loss: {:>6.3f}, Seconds: {:>4.2f}'\n",
    "                      .format(epoch_i,\n",
    "                              epochs, \n",
    "                              batch_i, \n",
    "                              len(sorted_X_short) // batch_size, \n",
    "                              batch_loss / display_step, \n",
    "                              batch_time*display_step))\n",
    "                batch_loss = 0\n",
    "\n",
    "            if batch_i % update_check == 0 and batch_i > 0:\n",
    "                print(\"Average loss for this update:\", round(update_loss/update_check,3))\n",
    "                y_update_loss.append(update_loss)\n",
    "                \n",
    "                # If the update loss is at a new minimum, save the model\n",
    "                if update_loss <= min(y_update_loss):\n",
    "                    print('New Record!') \n",
    "                    stop_early = 0\n",
    "                    saver = tf.train.Saver() \n",
    "                    saver.save(sess, checkpoint)\n",
    "\n",
    "                else:\n",
    "                    print(\"No Improvement.\")\n",
    "                    stop_early += 1\n",
    "                    if stop_early == num_to_stop:\n",
    "                        break\n",
    "                update_loss = 0\n",
    "            \n",
    "                    \n",
    "        # Reduce learning rate, but not below its minimum value\n",
    "        learning_rate *= learning_rate_decay\n",
    "        if learning_rate < min_learning_rate:\n",
    "            learning_rate = min_learning_rate\n",
    "        \n",
    "        if stop_early == num_to_stop:\n",
    "            print(\"Stopping Training.\")\n",
    "            break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_to_seq(text):\n",
    "    '''Prepare the text for the model'''\n",
    "    \n",
    "    text = clean_text(text)\n",
    "    temp_list = [word_to_int.get(word, word_to_int['<UNK>']) for word in word_tokenize(text)]\n",
    "    return list(reversed(temp_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./best_model.ckpt\n",
      "('Original Text:', 'Do you like Joshua?')\n",
      "\n",
      "Text\n",
      "  Word Ids:    [7, 13162, 610, 2905, 17490]\n",
      "  Input Words: ? joshua like you do\n",
      "\n",
      "Summary\n",
      "  Word Ids:       [10795, 610, 22590, 5616, 7990, 5640, 4]\n",
      "  Response Words: i like a lot of drugs .\n"
     ]
    }
   ],
   "source": [
    "# Create your own review or use one from the dataset\n",
    "input_sentence = \"Do you like Joshua?\"\n",
    "text = text_to_seq(input_sentence)\n",
    "# random = np.random.randint(0,len(clean_texts))\n",
    "# input_sentence = clean_texts[random]\n",
    "# text = text_to_seq(clean_texts[random])\n",
    "\n",
    "checkpoint = \"./best_model.ckpt\"\n",
    "\n",
    "loaded_graph = tf.Graph()\n",
    "with tf.Session(graph=loaded_graph) as sess:\n",
    "    # Load saved model\n",
    "    loader = tf.train.import_meta_graph(checkpoint + '.meta')\n",
    "    loader.restore(sess, checkpoint)\n",
    "\n",
    "    input_data = loaded_graph.get_tensor_by_name('input:0')\n",
    "    logits = loaded_graph.get_tensor_by_name('predictions:0')\n",
    "    X_length = loaded_graph.get_tensor_by_name('X_length:0')\n",
    "    y_length = loaded_graph.get_tensor_by_name('y_length:0')\n",
    "    keep_prob = loaded_graph.get_tensor_by_name('keep_prob:0')\n",
    "    \n",
    "    #Multiply by batch_size to match the model's input parameters\n",
    "    answer_logits = sess.run(logits, {input_data: [text]*batch_size, \n",
    "                                      y_length: [np.random.randint(35,40)], \n",
    "                                      X_length: [len(text)]*batch_size,\n",
    "                                      keep_prob: 1.0})[0] \n",
    "\n",
    "# Remove the paddings\n",
    "pad = word_to_int[\"<PAD>\"] \n",
    "\n",
    "print('Original Text:', input_sentence)\n",
    "\n",
    "print('\\nText')\n",
    "print('  Word Ids:    {}'.format([i for i in text]))\n",
    "print('  Input Words: {}'.format(\" \".join([int_to_word[i] for i in text])))\n",
    "\n",
    "print('\\nSummary')\n",
    "print('  Word Ids:       {}'.format([i for i in answer_logits if i != pad]))\n",
    "print('  Response Words: {}'.format(\" \".join([int_to_word[i] for i in answer_logits if i != pad])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save other files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "f = open('int_to_word.pkl', 'wb')\n",
    "pickle.dump(int_to_word, f)\n",
    "f.close()\n",
    "\n",
    "f = open('word_to_int.pkl', 'wb')\n",
    "pickle.dump(word_to_int, f)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}