word-level-rnn-w-word2vec-w-boundaries.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Based on: https://machinelearningmastery.com/use-word-embedding-layers-deep-learning-keras/ and https://machinelearningmastery.com/how-to-develop-a-word-level-neural-language-model-in-keras/ and https://github.com/stanfordnlp/GloVe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.05.21 10:47\n",
      "Time to process: [1] seconds\n"
     ]
    }
   ],
   "source": [
    "#import time\n",
    "import datetime\n",
    "import string\n",
    "from pickle import dump\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils import to_categorical\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import LSTM\n",
    "from keras.layers import Embedding\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from random import randint\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "def get_time( output=True ):\n",
    "    \n",
    "    temp = time.time()\n",
    "    if output:\n",
    "        now = datetime.datetime.now()\n",
    "        print( now.strftime( \"%Y.%m.%d %H:%M\" ) )\n",
    "        \n",
    "    return temp\n",
    "\n",
    "foo = get_time()\n",
    "\n",
    "def print_time( start_time, end_time, interval=\"seconds\" ):\n",
    "    \n",
    "    if interval == \"hours\":\n",
    "        print ( \"Time to process: [%s] hours\" % ( str( ( end_time - start_time ) / 60 / 60 ) ) )\n",
    "    else:\n",
    "        print ( \"Time to process: [%s] seconds\" % ( str( end_time - start_time ) ) )\n",
    "\n",
    "print_time( 0, 1 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "﻿SPEECH 1\n",
      "\n",
      "\n",
      "...Thank you so much.  That's so nice.  Isn't he a great guy.  He doesn't get a fair press; he doesn't get it.  It's just not fair.  And I have to tell you I'm here, and very strongly here\n"
     ]
    }
   ],
   "source": [
    "# load doc into memory\n",
    "def load_doc( filename ):\n",
    "    \n",
    "    # open the file as read only\n",
    "    file = open( filename, 'r' )\n",
    "    # read all text\n",
    "    text = file.read()\n",
    "    # close the file\n",
    "    file.close()\n",
    "    \n",
    "    return text\n",
    "\n",
    "# load document\n",
    "#in_filename = \"../texts/alice-in-wonderland.txt\"\n",
    "#in_filename = \"../texts/dr-zeuss-compilation.txt\"\n",
    "in_filename = \"../texts/trump-speeches.txt\"\n",
    "doc = load_doc( in_filename )\n",
    "print( doc[ :200 ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\"#$%&\\'()*+,-/:;<=>@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# my_punctuation = string.punctuation\n",
    "# print( type( my_punctuation ) )\n",
    "# print( my_punctuation )\n",
    "my_punctuation = '\"#$%&\\'()*+,-/:;<=>@[\\\\]^_`{|}~'\n",
    "my_punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# turn a doc into clean tokens\n",
    "def clean_doc( doc, to_lower=True ):\n",
    "    \n",
    "    # replace '--' with a space ' '\n",
    "    doc = doc.replace( '--', ' ' )\n",
    "    # replace sentence simple sentence boundaries w/ unique token/markers\n",
    "    doc = doc.replace( '. ', ' endperiod ' )\n",
    "    doc = doc.replace( '! ', ' endexclamation ' )\n",
    "    doc = doc.replace( '? ', ' endquestion ' )\n",
    "    doc = doc.replace( ', ', ' pausecomma ' )\n",
    "    doc = doc.replace( ': ', ' pausecolon ' )\n",
    "    doc = doc.replace( '; ', ' pausesemicolon ' )\n",
    "    \n",
    "    # split into tokens by white space\n",
    "    tokens = doc.split()\n",
    "    \n",
    "    # remove punctuation from each token\n",
    "    table = str.maketrans( '', '', string.punctuation ) # will strip all .?!,:; that don't fit replace expr above.\n",
    "    #table = str.maketrans( '', '', my_punctuation )\n",
    "    tokens = [ w.translate( table ) for w in tokens ]\n",
    "    \n",
    "    # remove remaining tokens that are not alphabetic\n",
    "    if to_lower:\n",
    "        tokens = [ word for word in tokens if word.isalpha() ]\n",
    "    \n",
    "    # make lower case\n",
    "    tokens = [ word.lower() for word in tokens ] \n",
    "    \n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['thank', 'you', 'so', 'much', 'endperiod', 'thats', 'so', 'nice', 'endperiod', 'isnt', 'he', 'a', 'great', 'guy', 'endperiod', 'he', 'doesnt', 'get', 'a', 'fair', 'press', 'pausesemicolon', 'he', 'doesnt', 'get', 'it', 'endperiod', 'its', 'just', 'not', 'fair', 'endperiod', 'and', 'i', 'have', 'to', 'tell', 'you', 'im', 'here', 'pausecomma', 'and', 'very', 'strongly', 'here', 'pausecomma', 'because', 'i', 'have', 'great', 'respect', 'for', 'steve', 'king', 'and', 'have', 'great', 'respect', 'likewise', 'for', 'citizens', 'united', 'pausecomma', 'david', 'and', 'everybody', 'pausecomma', 'and', 'tremendous', 'resect', 'for', 'the', 'tea', 'party', 'endperiod', 'also', 'pausecomma', 'also', 'the', 'people', 'of', 'iowa', 'endperiod', 'they', 'have', 'something', 'in', 'common', 'endperiod', 'hardworking', 'people', 'endperiod', 'they', 'want', 'to', 'work', 'pausecomma', 'they', 'want', 'to', 'make', 'the', 'country', 'great', 'endperiod', 'i', 'love', 'the', 'people', 'of', 'iowa', 'endperiod', 'so', 'thats', 'the', 'way', 'it', 'is', 'endperiod', 'very', 'simple', 'with', 'that', 'said', 'pausecomma', 'our', 'country', 'is', 'really', 'headed', 'in', 'the', 'wrong', 'direction', 'with', 'a', 'president', 'who', 'is', 'doing', 'an', 'absolutely', 'terrible', 'job', 'endperiod', 'the', 'world', 'is', 'collapsing', 'around', 'us', 'pausecomma', 'and', 'many', 'of', 'the', 'problems', 'weve', 'caused', 'endperiod', 'our', 'president', 'is', 'either', 'grossly', 'incompetent', 'pausecomma', 'a', 'word', 'that', 'more', 'and', 'more', 'people', 'are', 'using', 'pausecomma', 'and', 'i', 'think', 'i', 'was', 'the', 'first', 'to', 'use', 'it', 'pausecomma', 'or', 'he', 'has', 'a', 'completely', 'different', 'agenda', 'than', 'you', 'want', 'to', 'know']\n",
      "Total Tokens: 176988\n",
      "Unique Tokens: 5848\n"
     ]
    }
   ],
   "source": [
    "# clean document\n",
    "tokens = clean_doc( doc )\n",
    "tokens_unique = list( set( tokens ) )\n",
    "print( tokens[ :200 ] )\n",
    "print( 'Total Tokens: %d' % len( tokens ) )\n",
    "print( 'Unique Tokens: %d' % len( tokens_unique ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Sequences: 176937\n"
     ]
    }
   ],
   "source": [
    "# organize into sequences of tokens\n",
    "sequence_len = 50 + 1\n",
    "sequences = list()\n",
    "\n",
    "for i in range( sequence_len, len( tokens ) ):\n",
    "    \n",
    "    # select sequence of tokens\n",
    "    seq = tokens[ i - sequence_len:i ]\n",
    "    \n",
    "    # convert into a line\n",
    "    line = ' '.join( seq )\n",
    "    \n",
    "    # store\n",
    "    sequences.append( line )\n",
    "    \n",
    "print( 'Total Sequences: %d' % len( sequences ) )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save tokens to file, one dialog per line\n",
    "def save_doc( lines, filename ):\n",
    "    \n",
    "    data = '\\n'.join( lines )\n",
    "    file = open( filename, 'w' )\n",
    "    file.write( data )\n",
    "    file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save sequences to file\n",
    "#out_filename = \"../texts/dr-zeuss-compilation-sequences.txt\"\n",
    "out_filename = \"../texts/trump-speeches-sequences-02.txt\"\n",
    "save_doc( sequences, out_filename )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['thank you so much endperiod thats so nice endperiod isnt he a great guy endperiod he doesnt get a fair press pausesemicolon he doesnt get it endperiod its just not fair endperiod and i have to tell you im here pausecomma and very strongly here pausecomma because i have great respect',\n",
       " 'you so much endperiod thats so nice endperiod isnt he a great guy endperiod he doesnt get a fair press pausesemicolon he doesnt get it endperiod its just not fair endperiod and i have to tell you im here pausecomma and very strongly here pausecomma because i have great respect for',\n",
       " 'so much endperiod thats so nice endperiod isnt he a great guy endperiod he doesnt get a fair press pausesemicolon he doesnt get it endperiod its just not fair endperiod and i have to tell you im here pausecomma and very strongly here pausecomma because i have great respect for steve',\n",
       " 'much endperiod thats so nice endperiod isnt he a great guy endperiod he doesnt get a fair press pausesemicolon he doesnt get it endperiod its just not fair endperiod and i have to tell you im here pausecomma and very strongly here pausecomma because i have great respect for steve king',\n",
       " 'endperiod thats so nice endperiod isnt he a great guy endperiod he doesnt get a fair press pausesemicolon he doesnt get it endperiod its just not fair endperiod and i have to tell you im here pausecomma and very strongly here pausecomma because i have great respect for steve king and',\n",
       " 'thats so nice endperiod isnt he a great guy endperiod he doesnt get a fair press pausesemicolon he doesnt get it endperiod its just not fair endperiod and i have to tell you im here pausecomma and very strongly here pausecomma because i have great respect for steve king and have',\n",
       " 'so nice endperiod isnt he a great guy endperiod he doesnt get a fair press pausesemicolon he doesnt get it endperiod its just not fair endperiod and i have to tell you im here pausecomma and very strongly here pausecomma because i have great respect for steve king and have great',\n",
       " 'nice endperiod isnt he a great guy endperiod he doesnt get a fair press pausesemicolon he doesnt get it endperiod its just not fair endperiod and i have to tell you im here pausecomma and very strongly here pausecomma because i have great respect for steve king and have great respect',\n",
       " 'endperiod isnt he a great guy endperiod he doesnt get a fair press pausesemicolon he doesnt get it endperiod its just not fair endperiod and i have to tell you im here pausecomma and very strongly here pausecomma because i have great respect for steve king and have great respect likewise',\n",
       " 'isnt he a great guy endperiod he doesnt get a fair press pausesemicolon he doesnt get it endperiod its just not fair endperiod and i have to tell you im here pausecomma and very strongly here pausecomma because i have great respect for steve king and have great respect likewise for']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#in_filename = \"../texts/dr-zeuss-compilation-sequences.txt\"\n",
    "in_filename = \"../texts/trump-speeches-sequences-02.txt\"\n",
    "doc = load_doc( in_filename )\n",
    "lines = doc.split( '\\n' )\n",
    "lines[ 0:10 ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert Words to Index Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# integer encode sequences of words\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts( lines )\n",
    "sequences = tokenizer.texts_to_sequences( lines )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "176937\n"
     ]
    }
   ],
   "source": [
    "print( len( sequences[ 0 ] ) == sequence_len )\n",
    "print( len( sequences ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5848\n",
      "<class 'dict'>\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "print( len( tokenizer.word_index ) )\n",
    "print( type( tokenizer.word_index ) )\n",
    "print( tokenizer.word_index[ \"pausecomma\" ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5849"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# vocabulary size\n",
    "vocab_size = len( tokenizer.word_index ) + 1\n",
    "vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# separate into input and output: for now it's 50 words input and 1 word output\n",
    "sequences = np.array( sequences )\n",
    "X = sequences[ :,:-1 ] # all rows, from word 0 up to, but not including, the last word\n",
    "y = sequences[ :,-1 ]  # all rows, last word only\n",
    "y = to_categorical( y, num_classes=vocab_size )\n",
    "seq_length = X.shape[ 1 ]\n",
    "seq_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and Filter GloVe Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "****************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************\n",
      "Loaded 5644 word vectors.\n",
      "\n",
      "Words not found 204.\n"
     ]
    }
   ],
   "source": [
    "# load the whole embedding into memory\n",
    "embeddings_index = dict()\n",
    "embeddings_dimension = 300 #must be 50, 100, 200, 300\n",
    "glove = open( \"../glove/glove.6B.\" + str( embeddings_dimension ) + \"d.txt\" )\n",
    "\n",
    "for line in glove:\n",
    "    \n",
    "    values = line.split()\n",
    "    # 1st string is word...\n",
    "    word = values[ 0 ]\n",
    "    \n",
    "    if word in tokens_unique:\n",
    "        \n",
    "        # ...the rest are coefficients\n",
    "        coefs = np.asarray( values[ 1: ], dtype='float32' )\n",
    "        embeddings_index[ word ] = coefs\n",
    "        print( \"*\", end=\"\" )\n",
    "    \n",
    "glove.close()\n",
    "print( '\\nLoaded %s word vectors.' % len( embeddings_index ) )\n",
    "print( '\\nWords not found %d.' % ( len( tokenizer.word_index ) - len( embeddings_index ) ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transform into Matrix That Maps Coefs by Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "204\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['endperiod',\n",
       " 'pausecomma',\n",
       " 'endquestion',\n",
       " 'pausesemicolon',\n",
       " 'pausecolon',\n",
       " 'endexclamation',\n",
       " 'selffunding',\n",
       " 'theyve',\n",
       " 'youve',\n",
       " 'beada',\n",
       " 'selfinspect',\n",
       " 'theyll',\n",
       " 'twoway',\n",
       " 'onethird',\n",
       " 'reince',\n",
       " 'lowenergy',\n",
       " 'bluecollar',\n",
       " 'ayeyeye',\n",
       " 'fbomb',\n",
       " 'obamaclinton',\n",
       " 'maralago',\n",
       " 'bigly',\n",
       " 'sixyear',\n",
       " 'escavators',\n",
       " 'nationbuilding',\n",
       " 'africanamerican',\n",
       " 'byebye',\n",
       " 'selfinspection',\n",
       " 'offmike',\n",
       " 'antiwoman',\n",
       " 'intelligencegathering',\n",
       " 'thatand',\n",
       " 'africanamericans',\n",
       " 'donaldjtrumpcom',\n",
       " 'mexicanamerican',\n",
       " 'hardhitting',\n",
       " 'braggadocious',\n",
       " 'selfpolice',\n",
       " 'komatsus',\n",
       " 'nobodys',\n",
       " 'taxexempt',\n",
       " 'indianas',\n",
       " 'fiveforone',\n",
       " 'oreilly',\n",
       " 'resect',\n",
       " 'everyones',\n",
       " 'theyd',\n",
       " 'cetain',\n",
       " 'romneycare',\n",
       " 'oneyard',\n",
       " 'threefooter',\n",
       " 'werent',\n",
       " 'clearsighted',\n",
       " 'nationstate',\n",
       " 'goodsized',\n",
       " 'airconditioner',\n",
       " 'disastertrump',\n",
       " 'nogood',\n",
       " 'peopletrump',\n",
       " 'exampletrump',\n",
       " 'brandnew',\n",
       " 'oldfashioned',\n",
       " 'wellover',\n",
       " 'trilliontrump',\n",
       " 'jobproducer',\n",
       " 'truthteller',\n",
       " 'expresident',\n",
       " 'middleincome',\n",
       " 'miniversion',\n",
       " 'lowlevel',\n",
       " 'smarttough',\n",
       " 'stronglooking',\n",
       " 'resonants',\n",
       " 'exofficials',\n",
       " 'incomptent',\n",
       " 'twentyfive',\n",
       " 'swenden',\n",
       " 'meand',\n",
       " 'friendand',\n",
       " 'hijackists',\n",
       " 'phenomenol',\n",
       " 'knifewielding',\n",
       " 'secondclass',\n",
       " 'orlandos',\n",
       " 'antiamerican',\n",
       " 'lawabiding',\n",
       " 'evergrowing',\n",
       " 'politicallycorrect',\n",
       " 'ityou',\n",
       " 'trumphe',\n",
       " 'leadingnevada',\n",
       " 'twoperson',\n",
       " 'nowand',\n",
       " 'securitybut',\n",
       " 'japanthey',\n",
       " 'showsi',\n",
       " 'gunhow',\n",
       " 'groupthey',\n",
       " 'peopleand',\n",
       " 'antiiowa',\n",
       " 'lovei',\n",
       " 'guyin',\n",
       " 'retweeted',\n",
       " 'realdonaldtrump',\n",
       " 'welleducated',\n",
       " 'underbudget',\n",
       " 'primarycaucus',\n",
       " 'informedthat',\n",
       " 'vetdogs',\n",
       " 'debatebecause',\n",
       " 'millionike',\n",
       " 'amongstand',\n",
       " 'startedi',\n",
       " 'oppositionwhat',\n",
       " 'pressi',\n",
       " 'gaveinvites',\n",
       " 'outsidego',\n",
       " 'waitlists',\n",
       " 'whyhe',\n",
       " 'landslidei',\n",
       " 'candidatewhen',\n",
       " 'thisif',\n",
       " 'halffrozen',\n",
       " 'themthis',\n",
       " 'debatesi',\n",
       " 'knowwe',\n",
       " 'betyou',\n",
       " 'landslidedoes',\n",
       " 'notesand',\n",
       " 'twoyear',\n",
       " 'nabsico',\n",
       " 'doddfrank',\n",
       " 'bewell',\n",
       " 'corruptness',\n",
       " 'countryyou',\n",
       " 'saidwhat',\n",
       " 'embargoing',\n",
       " 'sevenyearold',\n",
       " 'sevenyearsold',\n",
       " 'backtrillion',\n",
       " 'recordsetting',\n",
       " 'watchingwe',\n",
       " 'toothe',\n",
       " 'peoplebush',\n",
       " 'thereso',\n",
       " 'stateswhich',\n",
       " 'expresidents',\n",
       " 'runningthey',\n",
       " 'parttimers',\n",
       " 'nowthe',\n",
       " 'wellour',\n",
       " 'selffund',\n",
       " 'decisionsor',\n",
       " 'fantasticyou',\n",
       " 'semifairly',\n",
       " 'dday',\n",
       " 'permityou',\n",
       " 'scums',\n",
       " 'fairhaired',\n",
       " 'proisrael',\n",
       " 'lifechanging',\n",
       " 'mackeral',\n",
       " 'rollsroyce',\n",
       " 'schusters',\n",
       " 'oneminute',\n",
       " 'socialistcommunist',\n",
       " 'overbrimming',\n",
       " 'mortagage',\n",
       " 'deductability',\n",
       " 'eschelons',\n",
       " 'obselete',\n",
       " 'selffunded',\n",
       " 'ordierno',\n",
       " 'selffunder',\n",
       " 'bleh',\n",
       " 'ayeyayay',\n",
       " 'braggingly',\n",
       " 'sharons',\n",
       " 'fricking',\n",
       " 'itll',\n",
       " 'deathdefying',\n",
       " 'thirtythree',\n",
       " 'brexits',\n",
       " 'oversay',\n",
       " 'numberone',\n",
       " 'secondlowest',\n",
       " 'pences',\n",
       " 'dshtidnt',\n",
       " 'pliblg',\n",
       " 'maepgessenger',\n",
       " 'cameramans',\n",
       " 'nashltionally',\n",
       " 'kpzcompanies',\n",
       " 'thetory',\n",
       " 'thirdworld',\n",
       " 'alumisource',\n",
       " 'depressionlevel',\n",
       " 'politicianmade',\n",
       " 'upsidedown',\n",
       " 'taxfree',\n",
       " 'jobkilling',\n",
       " 'inflationadjusted',\n",
       " 'transpacificpartnership',\n",
       " 'americanproduced']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a weight matrix for words in training docs\n",
    "embedding_matrix = np.zeros( ( vocab_size, embeddings_dimension ) )\n",
    "missing_words = []\n",
    "\n",
    "# we need this to create empty coefficients array\n",
    "dummy_shape = embeddings_index[ \"the\" ].shape\n",
    "\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    \n",
    "    embedding_vector = embeddings_index.get( word )\n",
    "    \n",
    "    # not all words in our token list are in the wikipedia 400K set!\n",
    "    if embedding_vector is None:\n",
    "        \n",
    "        # report and create empty coefficients array\n",
    "        missing_words.append( word )\n",
    "        embedding_vector = np.zeros( dummy_shape )\n",
    "        \n",
    "    embedding_matrix[ i ] = embedding_vector\n",
    "    \n",
    "print( len( missing_words ) )\n",
    "missing_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300\n",
      "0.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "205"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# confirm visually that \n",
    "print( len( embedding_matrix[ 0 ] ) )\n",
    "print( sum( embedding_matrix[ 0 ] ) )\n",
    "empty_coefficients_count = 0\n",
    "\n",
    "for i in range( len( embedding_matrix ) ):\n",
    "    if sum( embedding_matrix[ i ] ) == 0:\n",
    "        empty_coefficients_count += 1\n",
    "        \n",
    "empty_coefficients_count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.3\n",
      "1.4.1\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "print( keras.__version__ )\n",
    "\n",
    "import tensorflow as tf\n",
    "print( tf.__version__ )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 50, 300)           1754700   \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 50, 100)           160400    \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, 100)               80400     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 100)               10100     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 5849)              590749    \n",
      "=================================================================\n",
      "Total params: 2,596,349\n",
      "Trainable params: 2,596,349\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# define model\n",
    "model = Sequential()\n",
    "\n",
    "# now using a pre-trained, non-trainable embedding from glove's wiki analysis\n",
    "model.add( Embedding( vocab_size, embeddings_dimension, weights=[embedding_matrix], input_length=seq_length, trainable=True ) )\n",
    "model.add( LSTM( seq_length * 2, return_sequences=True ) )\n",
    "model.add( LSTM( seq_length * 2 ) )\n",
    "model.add( Dense( seq_length * 2, activation='relu' ) )\n",
    "\n",
    "# fixed TypeError below, downgraded keras from 2.1.5 to 2.1.3: https://github.com/keras-team/keras/issues/9621\n",
    "# TypeError: softmax() got an unexpected keyword argument 'axis'\n",
    "model.add( Dense( vocab_size, activation='softmax' ) )\n",
    "\n",
    "print( model.summary() )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1382.3203125\n",
      "172.11770428015564\n"
     ]
    }
   ],
   "source": [
    "# calc batch size\n",
    "print( len( sequences ) / 128 )\n",
    "print( len( sequences ) / 1028 )\n",
    "# Was:\n",
    "#batch_size = 128\n",
    "batch_size = 1028\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.05.21 13:43\n",
      "Epoch 1/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.4837 - acc: 0.4610\n",
      "Epoch 2/300\n",
      "176937/176937 [==============================] - 27s 151us/step - loss: 2.4333 - acc: 0.4694\n",
      "Epoch 3/300\n",
      "176937/176937 [==============================] - 27s 152us/step - loss: 2.4164 - acc: 0.4738\n",
      "Epoch 4/300\n",
      "176937/176937 [==============================] - 27s 152us/step - loss: 2.4068 - acc: 0.4743\n",
      "Epoch 5/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.3940 - acc: 0.4763\n",
      "Epoch 6/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.3905 - acc: 0.4772\n",
      "Epoch 7/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.3739 - acc: 0.4802\n",
      "Epoch 8/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.3636 - acc: 0.4826\n",
      "Epoch 9/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.3536 - acc: 0.4836\n",
      "Epoch 10/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.3443 - acc: 0.4866\n",
      "Epoch 11/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.3284 - acc: 0.4888\n",
      "Epoch 12/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.3183 - acc: 0.4907\n",
      "Epoch 13/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.3107 - acc: 0.4914\n",
      "Epoch 14/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.2990 - acc: 0.4943\n",
      "Epoch 15/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.2933 - acc: 0.4945\n",
      "Epoch 16/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.2803 - acc: 0.4968\n",
      "Epoch 17/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.2759 - acc: 0.4978\n",
      "Epoch 18/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.2644 - acc: 0.4991\n",
      "Epoch 19/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.2605 - acc: 0.5004\n",
      "Epoch 20/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.2461 - acc: 0.5040\n",
      "Epoch 21/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.2403 - acc: 0.5040\n",
      "Epoch 22/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.2303 - acc: 0.5063\n",
      "Epoch 23/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.2156 - acc: 0.5092\n",
      "Epoch 24/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.2058 - acc: 0.5109\n",
      "Epoch 25/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.1994 - acc: 0.5114\n",
      "Epoch 26/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.1923 - acc: 0.5125\n",
      "Epoch 27/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.1824 - acc: 0.5148\n",
      "Epoch 28/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.1773 - acc: 0.5161\n",
      "Epoch 29/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.1653 - acc: 0.5188\n",
      "Epoch 30/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.1619 - acc: 0.5184\n",
      "Epoch 31/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.1518 - acc: 0.5215\n",
      "Epoch 32/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.1365 - acc: 0.5241\n",
      "Epoch 33/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.1262 - acc: 0.5255\n",
      "Epoch 34/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.1288 - acc: 0.5247\n",
      "Epoch 35/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.1103 - acc: 0.5288\n",
      "Epoch 36/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.1083 - acc: 0.5293\n",
      "Epoch 37/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.1063 - acc: 0.5299\n",
      "Epoch 38/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.0868 - acc: 0.5342\n",
      "Epoch 39/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.0860 - acc: 0.5333\n",
      "Epoch 40/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.0723 - acc: 0.5353\n",
      "Epoch 41/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.0658 - acc: 0.5379\n",
      "Epoch 42/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.0635 - acc: 0.5375\n",
      "Epoch 43/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.0605 - acc: 0.5385\n",
      "Epoch 44/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.0395 - acc: 0.5431\n",
      "Epoch 45/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.0300 - acc: 0.5436\n",
      "Epoch 46/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.0355 - acc: 0.5421\n",
      "Epoch 47/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.0319 - acc: 0.5437\n",
      "Epoch 48/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.0208 - acc: 0.5452\n",
      "Epoch 49/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.0072 - acc: 0.5484\n",
      "Epoch 50/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 2.0029 - acc: 0.5491\n",
      "Epoch 51/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.9927 - acc: 0.5511\n",
      "Epoch 52/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.9811 - acc: 0.5538\n",
      "Epoch 53/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.9775 - acc: 0.5547\n",
      "Epoch 54/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.9609 - acc: 0.5581\n",
      "Epoch 55/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.9608 - acc: 0.5574\n",
      "Epoch 56/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.9539 - acc: 0.5591\n",
      "Epoch 57/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.9557 - acc: 0.5588\n",
      "Epoch 58/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.9574 - acc: 0.5584\n",
      "Epoch 59/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.9480 - acc: 0.5589\n",
      "Epoch 60/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.9466 - acc: 0.5596\n",
      "Epoch 61/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.9415 - acc: 0.5605\n",
      "Epoch 62/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.9267 - acc: 0.5644\n",
      "Epoch 63/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.9095 - acc: 0.5679\n",
      "Epoch 64/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8976 - acc: 0.5706\n",
      "Epoch 65/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8923 - acc: 0.5708\n",
      "Epoch 66/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8818 - acc: 0.5731\n",
      "Epoch 67/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8836 - acc: 0.5735\n",
      "Epoch 68/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8750 - acc: 0.5746\n",
      "Epoch 69/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.9036 - acc: 0.5671\n",
      "Epoch 70/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8868 - acc: 0.5710\n",
      "Epoch 71/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8657 - acc: 0.5755\n",
      "Epoch 72/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8634 - acc: 0.5752\n",
      "Epoch 73/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8551 - acc: 0.5780\n",
      "Epoch 74/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8426 - acc: 0.5801\n",
      "Epoch 75/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8306 - acc: 0.5832\n",
      "Epoch 76/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8195 - acc: 0.5861\n",
      "Epoch 77/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8147 - acc: 0.5860\n",
      "Epoch 78/300\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8108 - acc: 0.5861\n",
      "Epoch 79/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8086 - acc: 0.5881\n",
      "Epoch 80/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8132 - acc: 0.5857\n",
      "Epoch 81/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8100 - acc: 0.5882\n",
      "Epoch 82/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8139 - acc: 0.5847\n",
      "Epoch 83/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.7986 - acc: 0.5888\n",
      "Epoch 84/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.7937 - acc: 0.5888\n",
      "Epoch 85/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.7723 - acc: 0.5953\n",
      "Epoch 86/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.7616 - acc: 0.5983\n",
      "Epoch 87/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.7664 - acc: 0.5961\n",
      "Epoch 88/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.7676 - acc: 0.5945\n",
      "Epoch 89/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.7588 - acc: 0.5967\n",
      "Epoch 90/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.8031 - acc: 0.5858\n",
      "Epoch 91/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.7703 - acc: 0.5928\n",
      "Epoch 92/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.7554 - acc: 0.5964\n",
      "Epoch 93/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.7467 - acc: 0.5990\n",
      "Epoch 94/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.7334 - acc: 0.6013\n",
      "Epoch 95/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.7332 - acc: 0.6019\n",
      "Epoch 96/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.7157 - acc: 0.6054\n",
      "Epoch 97/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.7133 - acc: 0.6064\n",
      "Epoch 98/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.7043 - acc: 0.6077\n",
      "Epoch 99/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.7061 - acc: 0.6070\n",
      "Epoch 100/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6994 - acc: 0.6092\n",
      "Epoch 101/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6989 - acc: 0.6084\n",
      "Epoch 102/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.7221 - acc: 0.6028\n",
      "Epoch 103/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.7031 - acc: 0.6071\n",
      "Epoch 104/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6982 - acc: 0.6082\n",
      "Epoch 105/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6837 - acc: 0.6115\n",
      "Epoch 106/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6636 - acc: 0.6161\n",
      "Epoch 107/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6486 - acc: 0.6206\n",
      "Epoch 108/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6413 - acc: 0.6215\n",
      "Epoch 109/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6487 - acc: 0.6198\n",
      "Epoch 110/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6486 - acc: 0.6186\n",
      "Epoch 111/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6497 - acc: 0.6194\n",
      "Epoch 112/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6549 - acc: 0.6174\n",
      "Epoch 113/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6470 - acc: 0.6187\n",
      "Epoch 114/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6333 - acc: 0.6226\n",
      "Epoch 115/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6340 - acc: 0.6215\n",
      "Epoch 116/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6278 - acc: 0.6226\n",
      "Epoch 117/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6301 - acc: 0.6221\n",
      "Epoch 118/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6755 - acc: 0.6107\n",
      "Epoch 119/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6348 - acc: 0.6205\n",
      "Epoch 120/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.6051 - acc: 0.6297\n",
      "Epoch 121/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5938 - acc: 0.6307\n",
      "Epoch 122/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5928 - acc: 0.6307\n",
      "Epoch 123/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5909 - acc: 0.6304\n",
      "Epoch 124/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5831 - acc: 0.6331\n",
      "Epoch 125/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5894 - acc: 0.6309\n",
      "Epoch 126/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5754 - acc: 0.6349\n",
      "Epoch 127/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5696 - acc: 0.6354\n",
      "Epoch 128/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5742 - acc: 0.6342\n",
      "Epoch 129/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5732 - acc: 0.6337\n",
      "Epoch 130/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5732 - acc: 0.6340\n",
      "Epoch 131/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5797 - acc: 0.6309\n",
      "Epoch 132/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5609 - acc: 0.6358\n",
      "Epoch 133/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5577 - acc: 0.6369\n",
      "Epoch 134/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5437 - acc: 0.6410\n",
      "Epoch 135/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5381 - acc: 0.6427\n",
      "Epoch 136/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5506 - acc: 0.6375\n",
      "Epoch 137/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5556 - acc: 0.6366\n",
      "Epoch 138/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5415 - acc: 0.6403\n",
      "Epoch 139/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5967 - acc: 0.6257\n",
      "Epoch 140/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5479 - acc: 0.6383\n",
      "Epoch 141/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5171 - acc: 0.6456\n",
      "Epoch 142/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5109 - acc: 0.6484\n",
      "Epoch 143/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5148 - acc: 0.6465\n",
      "Epoch 144/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5035 - acc: 0.6486\n",
      "Epoch 145/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5198 - acc: 0.6449\n",
      "Epoch 146/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5035 - acc: 0.6484\n",
      "Epoch 147/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5017 - acc: 0.6493\n",
      "Epoch 148/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.5002 - acc: 0.6484\n",
      "Epoch 149/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.4870 - acc: 0.6518\n",
      "Epoch 150/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.4892 - acc: 0.6504\n",
      "Epoch 151/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.4929 - acc: 0.6495\n",
      "Epoch 152/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.4997 - acc: 0.6488\n",
      "Epoch 153/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.4873 - acc: 0.6517\n",
      "Epoch 154/300\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.4649 - acc: 0.6568\n",
      "Epoch 155/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.4514 - acc: 0.6608\n",
      "Epoch 156/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4589 - acc: 0.6583\n",
      "Epoch 157/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4712 - acc: 0.6544\n",
      "Epoch 158/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4483 - acc: 0.6595\n",
      "Epoch 159/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4398 - acc: 0.6636\n",
      "Epoch 160/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.4418 - acc: 0.6622\n",
      "Epoch 161/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4583 - acc: 0.6580\n",
      "Epoch 162/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4762 - acc: 0.6532\n",
      "Epoch 163/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4650 - acc: 0.6551\n",
      "Epoch 164/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4365 - acc: 0.6619\n",
      "Epoch 165/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4333 - acc: 0.6640\n",
      "Epoch 166/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4366 - acc: 0.6632\n",
      "Epoch 167/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4330 - acc: 0.6631\n",
      "Epoch 168/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4365 - acc: 0.6616\n",
      "Epoch 169/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4423 - acc: 0.6598\n",
      "Epoch 170/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4414 - acc: 0.6598\n",
      "Epoch 171/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4545 - acc: 0.6573\n",
      "Epoch 172/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4307 - acc: 0.6630\n",
      "Epoch 173/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4017 - acc: 0.6714\n",
      "Epoch 174/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3965 - acc: 0.6724\n",
      "Epoch 175/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3888 - acc: 0.6741\n",
      "Epoch 176/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3969 - acc: 0.6716\n",
      "Epoch 177/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3928 - acc: 0.6719\n",
      "Epoch 178/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3965 - acc: 0.6710\n",
      "Epoch 179/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3975 - acc: 0.6711\n",
      "Epoch 180/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4013 - acc: 0.6692\n",
      "Epoch 181/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4004 - acc: 0.6695\n",
      "Epoch 182/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3970 - acc: 0.6689\n",
      "Epoch 183/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3899 - acc: 0.6723\n",
      "Epoch 184/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3722 - acc: 0.6774\n",
      "Epoch 185/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3653 - acc: 0.6784\n",
      "Epoch 186/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3713 - acc: 0.6767\n",
      "Epoch 187/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3644 - acc: 0.6781\n",
      "Epoch 188/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3799 - acc: 0.6734\n",
      "Epoch 189/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3921 - acc: 0.6701\n",
      "Epoch 190/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.4024 - acc: 0.6677\n",
      "Epoch 191/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3626 - acc: 0.6782\n",
      "Epoch 192/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3659 - acc: 0.6765\n",
      "Epoch 193/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3447 - acc: 0.6830\n",
      "Epoch 194/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3668 - acc: 0.6769\n",
      "Epoch 195/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3491 - acc: 0.6809\n",
      "Epoch 196/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3526 - acc: 0.6804\n",
      "Epoch 197/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3512 - acc: 0.6797\n",
      "Epoch 198/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3329 - acc: 0.6845\n",
      "Epoch 199/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3136 - acc: 0.6891\n",
      "Epoch 200/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3164 - acc: 0.6889\n",
      "Epoch 201/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3273 - acc: 0.6848\n",
      "Epoch 202/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3331 - acc: 0.6841\n",
      "Epoch 203/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3523 - acc: 0.6786\n",
      "Epoch 204/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.3850 - acc: 0.6717\n",
      "Epoch 205/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.3943 - acc: 0.6676\n",
      "Epoch 206/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3452 - acc: 0.6806\n",
      "Epoch 207/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3202 - acc: 0.6868\n",
      "Epoch 208/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3021 - acc: 0.6922\n",
      "Epoch 209/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3388 - acc: 0.6815\n",
      "Epoch 210/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3233 - acc: 0.6864\n",
      "Epoch 211/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.2953 - acc: 0.6931\n",
      "Epoch 212/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.2923 - acc: 0.6944\n",
      "Epoch 213/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.2991 - acc: 0.6917\n",
      "Epoch 214/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.2957 - acc: 0.6931\n",
      "Epoch 215/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3073 - acc: 0.6888\n",
      "Epoch 216/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.3011 - acc: 0.6902\n",
      "Epoch 217/300\n",
      "176937/176937 [==============================] - 27s 153us/step - loss: 1.2955 - acc: 0.6924\n",
      "Epoch 218/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2791 - acc: 0.6959\n",
      "Epoch 219/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2753 - acc: 0.6969\n",
      "Epoch 220/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2603 - acc: 0.7013\n",
      "Epoch 221/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2653 - acc: 0.7004\n",
      "Epoch 222/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2767 - acc: 0.6957\n",
      "Epoch 223/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2688 - acc: 0.6981\n",
      "Epoch 224/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2799 - acc: 0.6948\n",
      "Epoch 225/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2905 - acc: 0.6900\n",
      "Epoch 226/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2799 - acc: 0.6950\n",
      "Epoch 227/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2777 - acc: 0.6950\n",
      "Epoch 228/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2734 - acc: 0.6957\n",
      "Epoch 229/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2761 - acc: 0.6948\n",
      "Epoch 230/300\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2803 - acc: 0.6944\n",
      "Epoch 231/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2576 - acc: 0.7010\n",
      "Epoch 232/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2498 - acc: 0.7030\n",
      "Epoch 233/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2479 - acc: 0.7023\n",
      "Epoch 234/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2412 - acc: 0.7033\n",
      "Epoch 235/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2585 - acc: 0.7006\n",
      "Epoch 236/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2509 - acc: 0.7012\n",
      "Epoch 237/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2453 - acc: 0.7027\n",
      "Epoch 238/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2258 - acc: 0.7074\n",
      "Epoch 239/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2145 - acc: 0.7111\n",
      "Epoch 240/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2346 - acc: 0.7061\n",
      "Epoch 241/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2502 - acc: 0.7005\n",
      "Epoch 242/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2644 - acc: 0.6971\n",
      "Epoch 243/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2693 - acc: 0.6963\n",
      "Epoch 244/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2382 - acc: 0.7045\n",
      "Epoch 245/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2158 - acc: 0.7095\n",
      "Epoch 246/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2324 - acc: 0.7052\n",
      "Epoch 247/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2307 - acc: 0.7058\n",
      "Epoch 248/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2069 - acc: 0.7123\n",
      "Epoch 249/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1953 - acc: 0.7151\n",
      "Epoch 250/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2038 - acc: 0.7119\n",
      "Epoch 251/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2202 - acc: 0.7079\n",
      "Epoch 252/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2275 - acc: 0.7059\n",
      "Epoch 253/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2324 - acc: 0.7040\n",
      "Epoch 254/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2098 - acc: 0.7098\n",
      "Epoch 255/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2042 - acc: 0.7115\n",
      "Epoch 256/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2226 - acc: 0.7060\n",
      "Epoch 257/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2036 - acc: 0.7122\n",
      "Epoch 258/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1903 - acc: 0.7158\n",
      "Epoch 259/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1903 - acc: 0.7150\n",
      "Epoch 260/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1808 - acc: 0.7174\n",
      "Epoch 261/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1652 - acc: 0.7214\n",
      "Epoch 262/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1757 - acc: 0.7179\n",
      "Epoch 263/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1873 - acc: 0.7159\n",
      "Epoch 264/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2182 - acc: 0.7066\n",
      "Epoch 265/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.2140 - acc: 0.7073\n",
      "Epoch 266/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1792 - acc: 0.7163\n",
      "Epoch 267/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1771 - acc: 0.7170\n",
      "Epoch 268/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1784 - acc: 0.7177\n",
      "Epoch 269/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1999 - acc: 0.7104\n",
      "Epoch 270/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1648 - acc: 0.7199\n",
      "Epoch 271/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1762 - acc: 0.7180\n",
      "Epoch 272/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1578 - acc: 0.7226\n",
      "Epoch 273/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1692 - acc: 0.7184\n",
      "Epoch 274/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1783 - acc: 0.7161\n",
      "Epoch 275/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1934 - acc: 0.7119\n",
      "Epoch 276/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1630 - acc: 0.7202\n",
      "Epoch 277/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1440 - acc: 0.7266\n",
      "Epoch 278/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1684 - acc: 0.7191\n",
      "Epoch 279/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1558 - acc: 0.7223\n",
      "Epoch 280/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1482 - acc: 0.7253\n",
      "Epoch 281/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1388 - acc: 0.7262\n",
      "Epoch 282/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1242 - acc: 0.7308\n",
      "Epoch 283/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1389 - acc: 0.7264\n",
      "Epoch 284/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1541 - acc: 0.7222\n",
      "Epoch 285/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1565 - acc: 0.7203\n",
      "Epoch 286/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1247 - acc: 0.7301\n",
      "Epoch 287/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1321 - acc: 0.7282\n",
      "Epoch 288/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1214 - acc: 0.7298\n",
      "Epoch 289/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1400 - acc: 0.7260\n",
      "Epoch 290/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1860 - acc: 0.7130\n",
      "Epoch 291/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1847 - acc: 0.7134\n",
      "Epoch 292/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1925 - acc: 0.7121\n",
      "Epoch 293/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1777 - acc: 0.7156\n",
      "Epoch 294/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1649 - acc: 0.7193\n",
      "Epoch 295/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1202 - acc: 0.7304\n",
      "Epoch 296/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1057 - acc: 0.7353\n",
      "Epoch 297/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1152 - acc: 0.7324\n",
      "Epoch 298/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1058 - acc: 0.7345\n",
      "Epoch 299/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1243 - acc: 0.7296\n",
      "Epoch 300/300\n",
      "176937/176937 [==============================] - 27s 154us/step - loss: 1.1670 - acc: 0.7175\n",
      "2018.05.21 15:58\n",
      "Time to process: [8139.687075614929] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "# compile model\n",
    "model.compile( loss='categorical_crossentropy', optimizer='adam', metrics=[ 'accuracy' ] )\n",
    "# fit model\n",
    "model.fit( X, y, batch_size=batch_size, epochs=300 )\n",
    "end_time = get_time()\n",
    "print_time( start_time, end_time )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the model to file\n",
    "model.save( \"models/trump-speeches-03.keras\" )\n",
    "\n",
    "# save the tokenizer\n",
    "dump( tokenizer, open( \"tokenizers/trump-speeches-03.pkl\", 'wb' ) )\n",
    "\n",
    "# save embedding_matrix based on wiki embeddings, complete w/ missing coefficients array dummies\n",
    "dump( embedding_matrix, open( \"embeddings/trump-speeches-03.glove\", 'wb' ) )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use The Model to Generate Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seq_length = len( lines[ 0 ].split() ) - 1\n",
    "seq_length"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Encoded Punctuation to Punctuation Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "punctuation_dict = {}\n",
    "punctuation_dict[ \"endperiod\" ] = \".\"\n",
    "punctuation_dict[ \"endquestion\" ] = \"?\"\n",
    "punctuation_dict[ \"endexclamation\" ] = \"!\"\n",
    "punctuation_dict[ \"pausecomma\" ] = \",\"\n",
    "punctuation_dict[ \"pausecolon\" ] = \":\"\n",
    "punctuation_dict[ \"pausesemicolon\" ] = \";\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'bar'"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "punctuation_dict.get( \"foo\", \"bar\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_seq( model, tokenizer, seq_length, seed_text, n_words ):\n",
    "    \n",
    "    result = list()\n",
    "    in_text = seed_text\n",
    "    \n",
    "    # generate a fixed number of words\n",
    "    for _ in range( n_words ):\n",
    "        \n",
    "        # encode the text as integer\n",
    "        encoded = tokenizer.texts_to_sequences( [ in_text ] )[ 0 ] \n",
    "        \n",
    "        # truncate sequences to a fixed length\n",
    "        encoded = pad_sequences( [ encoded ], maxlen=seq_length, truncating='pre' ) \n",
    "        \n",
    "        # predict probabilities for each word\n",
    "        yhat = model.predict_classes( encoded, verbose=0 )\n",
    "        \n",
    "        # map predicted word index to word\n",
    "        out_word = ''\n",
    "        for word, index in tokenizer.word_index.items():\n",
    "            if index == yhat:\n",
    "                out_word = word\n",
    "                break \n",
    "                \n",
    "        # append to input\n",
    "        in_text += ' ' + out_word\n",
    "        \n",
    "#         # create output, conditionally\n",
    "#         if out_word in punctuation_dict:\n",
    "#             out_word = punctuation_dict.get( out_word, out_word )\n",
    "        #result.append( out_word )\n",
    "        result.append( punctuation_dict.get( out_word, out_word ) )\n",
    "        \n",
    "    return ' '.join( result )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reformat_punctuation( doc ):\n",
    "    \n",
    "    doc = doc.replace( ' . ', '. ' )\n",
    "    doc = doc.replace( ' ! ', '! ' )\n",
    "    doc = doc.replace( ' ? ', '? ' )\n",
    "    doc = doc.replace( ' , ', ', ' )\n",
    "    doc = doc.replace( ' : ', ': ' )\n",
    "    doc = doc.replace( ' ; ', '; ' )\n",
    "    \n",
    "    return doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "that has tremendous personal energy to get us back on track endperiod you do that when you have that i think marco is highly overrated endperiod highly overrated endperiod he have it and all you have to do is look at his stance on things jeb he lacks the quality that...\n",
      "\n",
      "...you will be great anymore. we have to do it and they said, they said, i think going to do it. i mean, not going to happen. going to be able to do well, going to be the wall, folks. going\n"
     ]
    }
   ],
   "source": [
    "# select a seed text\n",
    "seed_text = lines[ randint( 0, len( lines ) ) ]\n",
    "print( seed_text + '...\\n' )\n",
    "#print( len( seed_text.split( \" \" ) ) )\n",
    "\n",
    "# generate new text\n",
    "generated = generate_seq( model, tokenizer, seq_length, seed_text, 50 )\n",
    "print( \"...\" + reformat_punctuation( generated ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_seq_word_by_word( model, tokenizer, seq_length, seed_text, n_words ):\n",
    "    \n",
    "    print( \"...\", end='' )\n",
    "    #result = list()\n",
    "    in_text = seed_text\n",
    "    \n",
    "    # generate a fixed number of words\n",
    "    for _ in range( n_words ):\n",
    "        \n",
    "        # encode the text as integer\n",
    "        encoded = tokenizer.texts_to_sequences( [ in_text ] )[ 0 ] \n",
    "        \n",
    "        # truncate sequences to a fixed length\n",
    "        encoded = pad_sequences( [ encoded ], maxlen=seq_length, truncating='pre' ) \n",
    "        \n",
    "        # predict probabilities for each word\n",
    "        yhat = model.predict_classes( encoded, verbose=0 )\n",
    "        \n",
    "        # map predicted word index to word\n",
    "        out_word = ''\n",
    "        for word, index in tokenizer.word_index.items():\n",
    "            if index == yhat:\n",
    "                out_word = word\n",
    "                print( word, end=' ' )\n",
    "                break \n",
    "                \n",
    "        # append to input for next iteration\n",
    "        in_text += ' ' + out_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "take two minutes to vote now pausecomma yours is a little more complicated because the caucus system is a little bit different endperiod a very interesting system endperiod but it is different what i have to say about iowa endperiod very important so you know a movement to move iowa to...\n",
      "\n",
      "...the united states endperiod we have a lot of mine that are going through the border pausecomma and going to be stronger than ever before "
     ]
    }
   ],
   "source": [
    "# select a seed text\n",
    "seed_text = lines[ randint( 0, len( lines ) ) ]\n",
    "print( seed_text + '...\\n' )\n",
    "\n",
    "# generate new text\n",
    "generate_seq_word_by_word( model, tokenizer, seq_length, seed_text, 25 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trump walked down\n",
      "...a fortune up your company so trump is so you sit right and i just want to skip iowa i was and he said what "
     ]
    }
   ],
   "source": [
    "my_input = input()\n",
    "\n",
    "# generate new text\n",
    "generate_seq_word_by_word( model, tokenizer, seq_length, my_input, 25 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}