word-level-rnn-w-word2vec-speeches-take-III.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Based on: https://machinelearningmastery.com/use-word-embedding-layers-deep-learning-keras/ and https://machinelearningmastery.com/how-to-develop-a-word-level-neural-language-model-in-keras/ and https://github.com/stanfordnlp/GloVe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Choose GPU to Run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Select GPU [0 or 1]: 1\n"
     ]
    }
   ],
   "source": [
    "# From: https://github.com/keras-team/keras/issues/6031\n",
    "import os\n",
    "gpu_id = input( \"Select GPU [0 or 1]: \" )\n",
    "\n",
    "if gpu_id in [ \"0\", \"1\" ]:\n",
    "    os.environ[ \"CUDA_VISIBLE_DEVICES\" ] = gpu_id\n",
    "else:\n",
    "    print( \"Invalid GPU id.  Defaulting to '0,1'\" )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Choose CPU Cores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Share CPU cores w/ other models? [y/n]: y\n",
      "Allocating 6 cores to this notebook\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "cores = 12\n",
    "share_cores = input( \"Share CPU cores w/ other models? [y/n]: \" )\n",
    "\n",
    "if share_cores == \"y\":\n",
    "    \n",
    "    cores = int( cores / 2 )\n",
    "\n",
    "print( \"Allocating %d cores to this notebook\" % cores )\n",
    "\n",
    "# From: https://stackoverflow.com/questions/46421258/limit-number-of-cores-used-in-keras\n",
    "\n",
    "from keras import backend as K\n",
    "K.set_session(\n",
    "    K.tf.Session(\n",
    "        config=K.tf.ConfigProto(\n",
    "            intra_op_parallelism_threads=cores, inter_op_parallelism_threads=cores \n",
    "        )\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.06.27 11:01\n",
      "2018.06.27 11:01\n",
      "Time to process: [0.31827831268310547] seconds\n"
     ]
    }
   ],
   "source": [
    "import gc\n",
    "import re\n",
    "import time\n",
    "import datetime\n",
    "import string\n",
    "from pickle import dump\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils import to_categorical\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import LSTM\n",
    "from keras.layers import Bidirectional\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import Embedding\n",
    "from keras.models import load_model\n",
    "import collections\n",
    "from wordsegment import load, segment\n",
    "import numpy as np\n",
    "\n",
    "from random import randint\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "def get_time( output=True ):\n",
    "    \n",
    "    temp = time.time()\n",
    "    if output:\n",
    "        now = datetime.datetime.now()\n",
    "        print( now.strftime( \"%Y.%m.%d %H:%M\" ) )\n",
    "        \n",
    "    return temp\n",
    "\n",
    "start_time = get_time()\n",
    "\n",
    "def print_time( start_time, end_time, interval=\"seconds\" ):\n",
    "    \n",
    "    if interval == \"hours\":\n",
    "        print ( \"Time to process: [%s] hours\" % ( str( ( end_time - start_time ) / 60 / 60 ) ) )\n",
    "    else:\n",
    "        print ( \"Time to process: [%s] seconds\" % ( str( end_time - start_time ) ) )\n",
    "\n",
    "in_filename = \"../texts/trump-speeches.txt\"\n",
    "\n",
    "# load segmentation dictionary: http://www.grantjenks.com/docs/wordsegment/\n",
    "load()\n",
    "\n",
    "print_time( start_time, get_time() )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Doc, Line by Line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.06.27 11:02\n",
      "2018.06.27 11:02\n",
      "Time to process: [0.003679990768432617] seconds\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"\\ufeffSPEECH 1\\n \\n \\n ...Thank you so much.  That's so nice.  Isn't he a great guy.  He doesn't get a fair press; he doesn't get it.  It's just not fair.  And I have to tell you I'm here, and very strongly here, because I have great respect for Steve King and have great respect likewise for Citizens United, David and everybody, and tremendous resect for the Tea Party.  Also, also the people of Iowa.  The\""
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# http://cmdlinetips.com/2011/08/three-ways-to-read-a-text-file-line-by-line-in-python/\n",
    "def load_doc_by_line( filename ):\n",
    "    \n",
    "    # Open the file with read only permit\n",
    "    file = open( filename, \"r\" )\n",
    "    \n",
    "    # use readlines to read all lines in the file\n",
    "    # The variable \"lines\" is a list containing all lines in the file\n",
    "    lines = file.readlines()\n",
    "    \n",
    "    # close the file after reading the lines.\n",
    "    file.close()\n",
    "    \n",
    "    return lines\n",
    "\n",
    "start_time = get_time()\n",
    "lines = load_doc_by_line( in_filename )\n",
    "doc = \" \".join( lines )\n",
    "\n",
    "print_time( start_time, get_time() )\n",
    "\n",
    "doc[ :400 ]\n",
    "# tweets = None\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Encoded Punctuation to Punctuation Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "punctuation_dict = {}\n",
    "punctuation_dict[ \"endperiod\" ] = \".\"\n",
    "punctuation_dict[ \"endquestion\" ] = \"?\"\n",
    "punctuation_dict[ \"endexclamation\" ] = \"!\"\n",
    "punctuation_dict[ \"pausecomma\" ] = \",\"\n",
    "punctuation_dict[ \"pausecolon\" ] = \":\"\n",
    "punctuation_dict[ \"pausesemicolon\" ] = \";\"\n",
    "punctuation_dict[ \"pauseemdash\" ] = \"-\"\n",
    "punctuation_dict[ \"pausedash\" ] = \"-\"\n",
    "punctuation_dict[ \"smartquoteopen\" ] = '“'\n",
    "punctuation_dict[ \"smartquoteclose\" ] = '”'\n",
    "punctuation_dict[ \"quoteopen\" ] = '\"'\n",
    "punctuation_dict[ \"quoteclose\" ] = '\"'\n",
    "punctuation_dict[ \"attweetat\" ] = '@'\n",
    "punctuation_dict[ \"tweetlink\" ] = \"[link]\"\n",
    "punctuation_dict[ \"hashtweethash\" ] = '#'\n",
    "punctuation_dict[ \"opentweetopen\" ] = '[start]'\n",
    "punctuation_dict[ \"closetweetclose\" ] = '[end]'\n",
    "punctuation_dict[ \"ampersand\" ] = '&'\n",
    "punctuation_dict[ \"tweetelipsis\" ] = \"...\"\n",
    "\n",
    "punctuation_dict[ \"contractionopen\" ] = \"contractionopen\"\n",
    "punctuation_dict[ \"contractionclose\" ] = \"contractionclose\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# turn a doc into clean tokens\n",
    "punctuation_string = '!‘\"\\'#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~'\n",
    "\n",
    "def clean_doc( doc ):\n",
    "    \n",
    "    # multiple kinds of emdash!\n",
    "    doc = doc.replace( '--', ' pauseemdash ' )\n",
    "    doc = doc.replace( '—', ' pauseemdash ' )\n",
    "    doc = doc.replace( '–', ' pauseemdash ' )\n",
    "    doc = doc.replace( 'u.s.', ' US ' )\n",
    "    doc = doc.replace( 'U.S.', ' US ' )\n",
    "    doc = doc.replace( \"…\", \" tweetelipsis \" )\n",
    "    doc = doc.replace( \"...\", \" tweetelipsis \" )\n",
    "    doc = doc.replace( '. ', ' endperiod ' )\n",
    "    doc = doc.replace( '! ', ' endexclamation ' )\n",
    "    doc = doc.replace( '? ', ' endquestion ' )\n",
    "    doc = doc.replace( '•', ' endbullet ' )\n",
    "    doc = doc.replace( ', ', ' pausecomma ' )\n",
    "    doc = doc.replace( ': ', ' pausecolon ' )\n",
    "    doc = doc.replace( '; ', ' pausesemicolon ' )\n",
    "    doc = doc.replace( ' - ', ' pausedash ' )\n",
    "    doc = doc.replace( '“', ' smartquoteopen ' )\n",
    "    doc = doc.replace( '”', ' smartquoteclose ' )\n",
    "    doc = doc.replace( ' \"', ' quoteopen ' )\n",
    "    doc = doc.replace( '\" ', ' quoteclose ' )\n",
    "#     doc = doc.replace( \"@ \", \" \" ) # remove trailing @'s first...\n",
    "#     doc = doc.replace( \"@\", \"attweetat\" ) # ...then prefix 1st char @ as word\n",
    "#     doc = doc.replace( \"# \", \" \" ) # remove trailing #'s first...\n",
    "#     doc = doc.replace( \"#\", \"hashtweethash\" ) # ...then prefix 1st char # as word\n",
    "    doc = doc.replace( \"&amp;\", \" ampersand \" )\n",
    "    doc = doc.replace( \"\\ufeff\", \"\" )\n",
    "    \n",
    "    # From: https://stackoverflow.com/questions/33113338/how-to-replace-dash-between-characters-with-space-using-regex\n",
    "    # replace hyphenated words w/ spaces\n",
    "    doc = re.sub( r\"([a-z])\\-([a-z])\", r\"\\1 \\2\", doc , 0, re.IGNORECASE )\n",
    "    \n",
    "    # replace comma separated words w/0 spaces\n",
    "    doc = re.sub( r\"([a-z]),([a-z])\", r\"\\1 \\2\", doc , 0, re.IGNORECASE )\n",
    "    \n",
    "#     # replace links w/ \"tweetlink\"\n",
    "#     # basic regex here: https://bytes.com/topic/python/answers/741677-find-replace-hyperlinks-string\n",
    "#     http_pattern = r'http[^\\s\\n\\r]+'\n",
    "#     doc = re.sub( http_pattern , \"tweetlink\", doc )\n",
    "    \n",
    "#     # this overgenerates texttexttexttweetlink, so insert space where it occurs\n",
    "#     doc = re.sub( r\"([a-z])(tweetlink)\", r\"\\1 \\2\", doc , 0, re.IGNORECASE )\n",
    "    \n",
    "#     # above isn't catching all (wuh?!?), so just bruteforce it\n",
    "#     doc = doc.replace( \"tweetlink\", \" tweetlink\" )\n",
    "    \n",
    "#     # overgeneration of foooooooooooooooohashtweethash, so insert space where it occurs\n",
    "#     doc = re.sub( r\"([a-z])(hashtweethash)\", r\"\\1 \\2\", doc , 0, re.IGNORECASE )\n",
    "#     # above isn't catching all (wuh?!?), so just bruteforce it\n",
    "#     doc = doc.replace( \"hashtweethash\", \" hashtweethash\" )\n",
    "    \n",
    "#     # overgeneration of fooooooooooooooooattweetat, so insert space where it occurs\n",
    "#     doc = re.sub( r\"([a-z])(attweetat)\", r\"\\1 \\2\", doc , 0, re.IGNORECASE )\n",
    "#     # above isn't catching all (wuh?!?), so just bruteforce it\n",
    "#     doc = doc.replace( \"attweetat\", \" attweetat\" )\n",
    "    \n",
    "#     # tag all hyphenated words to protect from deletion\n",
    "#     doc = re.sub( r\"([a-z])-([a-z])\", r\"\\1hyphentweethyphen\\2\", doc , 0, re.IGNORECASE )\n",
    "    \n",
    "    # do big ad-hoc global replacement, instead of using maketrans and string.punctuation\n",
    "    # single quote is special case: don't leave a space, so that contractions are collapsed\n",
    "    doc = re.sub( r\"([a-z]+)'([a-z]+)\", r\"contractionopen \\1\\2 contractionclose\", doc , 0, re.IGNORECASE )\n",
    "    # some fool's using a DIFFERENT apostrophe... ’\n",
    "    doc = re.sub( r\"([a-z]+)’([a-z]+)\", r\"contractionopen \\1\\2 contractionclose\", doc , 0, re.IGNORECASE )\n",
    "    # replaced by above to wrap contractions\n",
    "    # doc = re.sub( r\"([a-z])'([a-z])\", r\"\\1\\2\", doc , 0, re.IGNORECASE )\n",
    "    # # some fool's using a DIFFERENT apostrophe... ’\n",
    "    # doc = re.sub( r\"([a-z])’([a-z])\", r\"\\1\\2\", doc , 0, re.IGNORECASE )\n",
    "    \n",
    "    # trailing deletion: goin' to goin\n",
    "    doc = re.sub( r\"([a-z])’ \", r\"\\1 \", doc , 0, re.IGNORECASE )\n",
    "    # not working?!?\n",
    "    #doc = re.sub( r\"’([a-z])\", r\"\\1\", doc , 0, re.IGNORECASE )\n",
    "    \n",
    "    # otherwise, delete all chars not already tagged as having semantic interest\n",
    "    # moved up!\n",
    "    #punctuation_string = '!‘\"\\'#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~'\n",
    "    for punctuation_char in punctuation_string:\n",
    "        doc = doc.replace( punctuation_char, ' ' )\n",
    "    \n",
    "    # ...now that global deletion of stragging dashes has been performed, replaced hyphenated words\n",
    "    #doc = doc.replace( \"hyphentweethyphen\", \"-\" )\n",
    "    \n",
    "    # finally, reduce duplicate spaces to just one: https://stackoverflow.com/questions/1546226/simple-way-to-remove-multiple-spaces-in-a-string/15913564\n",
    "    # doc = doc.replace( \"  \", ' ' )\n",
    "    doc = re.sub( ' +', ' ', doc )\n",
    "    \n",
    "    # split into tokens by white space\n",
    "    tokens = doc.split()\n",
    "    \n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Feels',\n",
       " 'like',\n",
       " 'contractionopen',\n",
       " 'Im',\n",
       " 'contractionclose',\n",
       " 'goin',\n",
       " 'crazy',\n",
       " 'pausecomma',\n",
       " 'this',\n",
       " 'contractionopen',\n",
       " 'isnt',\n",
       " 'contractionclose',\n",
       " 'some',\n",
       " 'contractionopen',\n",
       " 'fools',\n",
       " 'contractionclose',\n",
       " 'errand',\n",
       " 'pausecomma',\n",
       " 'contractionopen',\n",
       " 'wouldnt',\n",
       " 'contractionclose',\n",
       " 'you',\n",
       " 'agree']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_doc( \"\\ufeffFeels like I'm goin’ crazy, this isn’t some fool's errand, wouldn't you agree?\" )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Dictionary and Segment Doc Tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.06.27 11:06\n",
      "2018.06.27 11:06\n",
      "Time to process: [0.03485250473022461] seconds\n",
      "vocabulary_list 400000\n",
      "vocabulary_dict 400000\n",
      "False\n",
      "True\n",
      "True\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# load dictionary\n",
    "start_time = get_time()\n",
    "\n",
    "embeddings_dimension = 300 #must be 50, 100, 200, 300\n",
    "with open( \"output/vocabulary-glove.6B.\" + str( embeddings_dimension ) + \"d.txt\", 'r' ) as vocabulary_file:\n",
    "    \n",
    "    # omit newline char: https://stackoverflow.com/questions/12330522/reading-a-file-without-newlines\n",
    "    vocabulary_list = vocabulary_file.read().splitlines()\n",
    "\n",
    "print_time( start_time, get_time() )\n",
    "\n",
    "vocabulary_dict = dict.fromkeys( vocabulary_list )\n",
    "print( \"vocabulary_list\", len( vocabulary_list ) )\n",
    "print( \"vocabulary_dict\", len( vocabulary_dict ) )\n",
    "\n",
    "print( \"1234123412341234\" in vocabulary_dict )\n",
    "print( \"earth\" in vocabulary_dict )\n",
    "print( \"earth\" in vocabulary_list )\n",
    "print( vocabulary_dict[ \"earth\" ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.06.27 11:17\n",
      "\n",
      "Total tokens_raw: 209834\n",
      "Unique tokens: 7668\n",
      "Unique tokens_unique_lowercase: 7668\n",
      "2018.06.27 11:17\n",
      "Time to process: [0.45456862449645996] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "\n",
    "# clean document\n",
    "tokens_raw = clean_doc( doc )\n",
    "\n",
    " # 'compress' into list of unique tokens\n",
    "tokens_unique = list( set( tokens_raw ) )\n",
    "tokens_unique_lowercase = [ token.lower() for token in tokens_unique ]\n",
    "\n",
    "print()\n",
    "print( 'Total tokens_raw: %d' % len( tokens_raw ) )\n",
    "print( 'Unique tokens: %d' % len( tokens_unique ) )\n",
    "print( 'Unique tokens_unique_lowercase: %d' % len( tokens_unique_lowercase ) )\n",
    "\n",
    "print_time( start_time, get_time() )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Look at Token Frequencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'1': 84,\n",
       " 'SPEECH': 11,\n",
       " 'Thank': 148,\n",
       " 'Thats': 209,\n",
       " 'contractionopen': 9313,\n",
       " 'endperiod': 11080,\n",
       " 'much': 307,\n",
       " 'so': 781,\n",
       " 'tweetelipsis': 318,\n",
       " 'you': 2198}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_counts = collections.Counter( tokens_raw )\n",
    "pairs = { k: word_counts[ k ] for k in list( word_counts )[ :10 ] }\n",
    "pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "endperiod:   11080\n",
      "contractionopen:    9313\n",
      "contractionclose:    9313\n",
      "pausecomma:    8572\n",
      "the:    5187\n",
      "to:    5171\n",
      "I:    4873\n",
      "and:    3531\n",
      "a:    3351\n",
      "of:    2803\n",
      "you:    2198\n",
      "have:    2121\n",
      "that:    2109\n",
      "it:    2071\n",
      "pauseemdash:    1931\n",
      "going:    1922\n",
      "in:    1882\n",
      "And:    1643\n",
      "is:    1422\n",
      "we:    1405\n",
      "they:    1364\n",
      "know:    1278\n",
      "people:    1209\n",
      "be:    1127\n",
      "are:     987\n",
      "for:     977\n",
      "We:     975\n",
      "was:     932\n",
      "endquestion:     932\n",
      "But:     919\n"
     ]
    }
   ],
   "source": [
    "for word, count in word_counts.most_common( 30 ):\n",
    "    print( '%s: %7d' % ( word, count ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "i 0\n",
      "I 4873\n",
      "me 718\n",
      "you 2198\n",
      "we 1405\n",
      "us 309\n",
      "US 34\n",
      "they 1364\n",
      "them 548\n"
     ]
    }
   ],
   "source": [
    "print( \"i\", word_counts[ \"i\" ] )\n",
    "print( \"I\", word_counts[ \"I\" ] )\n",
    "print( \"me\", word_counts[ \"me\" ] )\n",
    "\n",
    "print( \"you\", word_counts[ \"you\" ] )\n",
    "\n",
    "print( \"we\", word_counts[ \"we\" ] )\n",
    "print( \"us\", word_counts[ \"us\" ] )\n",
    "print( \"US\", word_counts[ \"US\" ] )\n",
    "\n",
    "print( \"they\", word_counts[ \"they\" ] )\n",
    "print( \"them\", word_counts[ \"them\" ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.06.27 11:18\n",
      "Total Sequences: 209783\n",
      "2018.06.27 11:18\n",
      "Time to process: [0.171112060546875] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "\n",
    "# organize into sequences of tokens\n",
    "sequence_len = 50 + 1\n",
    "sequences = []\n",
    "\n",
    "for i in range( sequence_len, len( tokens_raw ) ):\n",
    "    \n",
    "    # select sequence of tokens\n",
    "    seq = tokens_raw[ i - sequence_len:i ]\n",
    "    \n",
    "    # convert into a line\n",
    "    line = ' '.join( seq )\n",
    "    \n",
    "    # store\n",
    "    sequences.append( line )\n",
    "    \n",
    "print( 'Total Sequences: %d' % len( sequences ) )\n",
    "print_time( start_time, get_time() )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save tokens to file, one dialog per line\n",
    "def save_doc( lines, filename ):\n",
    "    \n",
    "    data = '\\n'.join( lines )\n",
    "    file = open( filename, 'w' )\n",
    "    file.write( data )\n",
    "    file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save sequences to file\n",
    "out_filename = \"sequences/trump-speech-sequences-take-III.txt\"\n",
    "save_doc( sequences, out_filename )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_doc( filename ):\n",
    "    \n",
    "    # open the file as read only\n",
    "    file = open( filename, 'r' )\n",
    "    # read all text\n",
    "    text = file.read()\n",
    "    # close the file\n",
    "    file.close()\n",
    "    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['SPEECH 1 tweetelipsis Thank you so much endperiod contractionopen Thats contractionclose so nice endperiod contractionopen Isnt contractionclose he a great guy endperiod He contractionopen doesnt contractionclose get a fair press pausesemicolon he contractionopen doesnt contractionclose get it endperiod contractionopen Its contractionclose just not fair endperiod And I have to tell you',\n",
       " '1 tweetelipsis Thank you so much endperiod contractionopen Thats contractionclose so nice endperiod contractionopen Isnt contractionclose he a great guy endperiod He contractionopen doesnt contractionclose get a fair press pausesemicolon he contractionopen doesnt contractionclose get it endperiod contractionopen Its contractionclose just not fair endperiod And I have to tell you contractionopen',\n",
       " 'tweetelipsis Thank you so much endperiod contractionopen Thats contractionclose so nice endperiod contractionopen Isnt contractionclose he a great guy endperiod He contractionopen doesnt contractionclose get a fair press pausesemicolon he contractionopen doesnt contractionclose get it endperiod contractionopen Its contractionclose just not fair endperiod And I have to tell you contractionopen Im',\n",
       " 'Thank you so much endperiod contractionopen Thats contractionclose so nice endperiod contractionopen Isnt contractionclose he a great guy endperiod He contractionopen doesnt contractionclose get a fair press pausesemicolon he contractionopen doesnt contractionclose get it endperiod contractionopen Its contractionclose just not fair endperiod And I have to tell you contractionopen Im contractionclose',\n",
       " 'you so much endperiod contractionopen Thats contractionclose so nice endperiod contractionopen Isnt contractionclose he a great guy endperiod He contractionopen doesnt contractionclose get a fair press pausesemicolon he contractionopen doesnt contractionclose get it endperiod contractionopen Its contractionclose just not fair endperiod And I have to tell you contractionopen Im contractionclose here',\n",
       " 'so much endperiod contractionopen Thats contractionclose so nice endperiod contractionopen Isnt contractionclose he a great guy endperiod He contractionopen doesnt contractionclose get a fair press pausesemicolon he contractionopen doesnt contractionclose get it endperiod contractionopen Its contractionclose just not fair endperiod And I have to tell you contractionopen Im contractionclose here pausecomma',\n",
       " 'much endperiod contractionopen Thats contractionclose so nice endperiod contractionopen Isnt contractionclose he a great guy endperiod He contractionopen doesnt contractionclose get a fair press pausesemicolon he contractionopen doesnt contractionclose get it endperiod contractionopen Its contractionclose just not fair endperiod And I have to tell you contractionopen Im contractionclose here pausecomma and',\n",
       " 'endperiod contractionopen Thats contractionclose so nice endperiod contractionopen Isnt contractionclose he a great guy endperiod He contractionopen doesnt contractionclose get a fair press pausesemicolon he contractionopen doesnt contractionclose get it endperiod contractionopen Its contractionclose just not fair endperiod And I have to tell you contractionopen Im contractionclose here pausecomma and very',\n",
       " 'contractionopen Thats contractionclose so nice endperiod contractionopen Isnt contractionclose he a great guy endperiod He contractionopen doesnt contractionclose get a fair press pausesemicolon he contractionopen doesnt contractionclose get it endperiod contractionopen Its contractionclose just not fair endperiod And I have to tell you contractionopen Im contractionclose here pausecomma and very strongly',\n",
       " 'Thats contractionclose so nice endperiod contractionopen Isnt contractionclose he a great guy endperiod He contractionopen doesnt contractionclose get a fair press pausesemicolon he contractionopen doesnt contractionclose get it endperiod contractionopen Its contractionclose just not fair endperiod And I have to tell you contractionopen Im contractionclose here pausecomma and very strongly here']"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "in_filename = \"sequences/trump-speech-sequences-take-III.txt\"\n",
    "#doc = load_doc( in_filename )\n",
    "lines = load_doc( in_filename ).split( '\\n' )\n",
    "lines[ 0:10 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "51.0\n",
      "{51: 209783}\n"
     ]
    }
   ],
   "source": [
    "seq_len_sum = 0;\n",
    "line_len_dict = {}\n",
    "\n",
    "for line in lines:\n",
    "    \n",
    "    token_count = len( line.split( \" \" ) )\n",
    "    seq_len_sum += token_count\n",
    "    \n",
    "    if token_count in line_len_dict:\n",
    "        line_len_dict[ token_count ] += 1\n",
    "    else:\n",
    "        line_len_dict[ token_count ] = 1\n",
    "\n",
    "print( seq_len_sum / len( lines ) )\n",
    "print( line_len_dict )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert Words to Index Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.06.27 11:19\n",
      "sequences len *before* keras: 209783\n",
      "sequences len *after* keras: 209783\n",
      "3\n",
      "2\n",
      "endperiod\n",
      "is\n",
      "pausecomma\n",
      "vocab_size 7669\n",
      "2018.06.27 11:19\n",
      "Time to process: [7.721929311752319] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "\n",
    "# integer encode sequences of words\n",
    "# tokenizer = Tokenizer( lower=False, filters=punctuation_string )\n",
    "tokenizer = Tokenizer( lower=False, filters=\"\" )\n",
    "\n",
    "tokenizer.fit_on_texts( lines )\n",
    "print( \"sequences len *before* keras:\", len( sequences ) )\n",
    "sequences = tokenizer.texts_to_sequences( lines )\n",
    "print( \"sequences len *after* keras:\", len( sequences ) )\n",
    "\n",
    "# elegant! https://stackoverflow.com/questions/41971587/how-to-convert-predicted-sequence-back-to-text-in-keras\n",
    "words_by_id = dict( map( reversed, tokenizer.word_index.items() ) ) \n",
    "\n",
    "# Check to and from of words and idx\n",
    "print( tokenizer.word_index[ \"contractionopen\" ] )\n",
    "print( tokenizer.word_index[ \"contractionclose\" ] )\n",
    "\n",
    "print( words_by_id[ 2] )\n",
    "print( words_by_id[ 3 ] )\n",
    "\n",
    "# vocabulary size\n",
    "# discrepancy between these two lengths, of by a few words...\n",
    "#vocab_size = len( tokens_unique ) + 1\n",
    "vocab_size = len( tokenizer.word_index ) + 1\n",
    "print( \"vocab_size\", vocab_size )\n",
    "\n",
    "print_time( start_time, get_time() )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This KLUDGE Works, But Is No Longer Needed\n",
    "*(Using an empty filter string in the tokenizer obviates the need for this workaround)*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "# start_time = get_time()\n",
    "\n",
    "# sequences_foo = np.zeros( ( len( sequences ), len( sequences[ 0 ] ) ), dtype=int )\n",
    "# print( sequences_foo.shape )\n",
    "\n",
    "# row_count = 0\n",
    "\n",
    "# for row_idx, row in enumerate( sequences ):\n",
    "    \n",
    "#     for col_idx, col in enumerate( row ):\n",
    "        \n",
    "#         if col_idx > 26:\n",
    "#             print( row_idx, col_idx )\n",
    "#         else:\n",
    "#             sequences_foo[ row_idx, col_idx ] = col\n",
    "        \n",
    "        \n",
    "# #     if row_count == 30:\n",
    "# #         break\n",
    "# #     row_count += 1\n",
    "\n",
    "# print( sequences_foo.shape )\n",
    "# print( sequences_foo[ 0 ])\n",
    "\n",
    "# print_time( start_time, get_time() )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## We Need to Convert List of Lists into Array of Arrays\n",
    "_(Tokenizer's output is different when asked to leave case as is!?!)_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.06.27 11:21\n",
      "<class 'list'>\n",
      "<class 'numpy.ndarray'>\n",
      "(209783, 51)\n",
      "<class 'numpy.ndarray'>\n",
      "[1472  301   93  185   11   40  101    1    3  144    2   40  193    1    3\n",
      " 1691    2   52    9   53  168    1  110    3  216    2   56    9  641  312\n",
      "  215   52    3  216    2   56   14    1    3   61    2   66   35  641    1\n",
      "   18    7   12    6  123   11]\n",
      "2018.06.27 11:21\n",
      "Time to process: [1.0177383422851562] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "\n",
    "sequences_np = np.array( sequences )\n",
    "\n",
    "for i in range( len( sequences ) ):\n",
    "    sequences_np[ i ] = np.array( sequences[ i ] )\n",
    "\n",
    "print( type( sequences ) )\n",
    "print( type( sequences_np ) )\n",
    "print( sequences_np.shape )\n",
    "print( type( sequences_np[ 0 ] ) )\n",
    "print( sequences_np[ 0 ] )\n",
    "\n",
    "print_time( start_time, get_time() )\n",
    "\n",
    "# sequences = None\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7669\n",
      "7669\n",
      "7669\n"
     ]
    }
   ],
   "source": [
    "print( len( tokenizer.word_index ) + 1 )\n",
    "print( len( tokens_unique ) + 1 )\n",
    "print( len( set( tokens_unique ) ) + 1 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n",
      "(209783, 51)\n",
      "[1472  301   93  185   11   40  101    1    3  144    2   40  193    1    3\n",
      " 1691    2   52    9   53  168    1  110    3  216    2   56    9  641  312\n",
      "  215   52    3  216    2   56   14    1    3   61    2   66   35  641    1\n",
      "   18    7   12    6  123   11]\n",
      "<class 'numpy.ndarray'>\n",
      "X.shape (209783, 50)\n",
      "y.shape (209783,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print( type( sequences_np ) )\n",
    "print( sequences_np.shape )\n",
    "\n",
    "print( sequences_np[ 0 ] )\n",
    "print( type( sequences_np[ 0 ] ) )\n",
    "\n",
    "# separate into input and output: for now it's 50 words input and 1 word output\n",
    "#sequences = np.array( sequences )\n",
    "X = sequences_np[ :,:-1 ] # all rows, from word 0 up to, but not including, the last word\n",
    "y = sequences_np[ :,-1 ]  # all rows, last word only\n",
    "\n",
    "# Throws MemoryError\n",
    "# https://stackoverflow.com/questions/46293734/memoryerror-in-keras-utils-np-utils-to-categorical\n",
    "#y = to_categorical( y, num_classes=vocab_size )\n",
    "print( \"X.shape\", X.shape )\n",
    "print( \"y.shape\", y.shape )\n",
    "\n",
    "seq_length = len( X[ 0 ] )\n",
    "seq_length"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and Filter GloVe Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.06.27 11:23\n",
      "¿ 7668 == 7668 ?\n",
      "[3] entries found for [the]\n",
      "[3] entries found for [of]\n",
      "[3] entries found for [to]\n",
      "[3] entries found for [and]\n",
      "[3] entries found for [in]\n",
      "[2] entries found for [a]\n",
      "[3] entries found for [for]\n",
      "[3] entries found for [that]\n",
      "[3] entries found for [on]\n",
      "[3] entries found for [is]\n",
      "[3] entries found for [was]\n",
      "[3] entries found for [said]\n",
      "[3] entries found for [with]\n",
      "[3] entries found for [he]\n",
      "[3] entries found for [as]\n",
      "[3] entries found for [it]\n",
      "[3] entries found for [by]\n",
      "[3] entries found for [at]\n",
      "[3] entries found for [from]\n",
      "[3] entries found for [his]\n",
      "[3] entries found for [an]\n",
      "[3] entries found for [be]\n",
      "[3] entries found for [has]\n",
      "[3] entries found for [are]\n",
      "[3] entries found for [have]\n",
      "[3] entries found for [but]\n",
      "[3] entries found for [were]\n",
      "[3] entries found for [not]\n",
      "[3] entries found for [this]\n",
      "[3] entries found for [who]\n",
      "[3] entries found for [they]\n",
      "[2] entries found for [had]\n",
      "[3] entries found for [which]\n",
      "[3] entries found for [will]\n",
      "[3] entries found for [their]\n",
      "[3] entries found for [or]\n",
      "[3] entries found for [its]\n",
      "[3] entries found for [one]\n",
      "[3] entries found for [after]\n",
      "[3] entries found for [new]\n",
      "[2] entries found for [been]\n",
      "[3] entries found for [also]\n",
      "[3] entries found for [we]\n",
      "[3] entries found for [would]\n",
      "[3] entries found for [two]\n",
      "[3] entries found for [more]\n",
      "[3] entries found for [first]\n",
      "[3] entries found for [about]\n",
      "[2] entries found for [up]\n",
      "[3] entries found for [when]\n",
      "[3] entries found for [year]\n",
      "[3] entries found for [there]\n",
      "[3] entries found for [all]\n",
      "[3] entries found for [out]\n",
      "[3] entries found for [she]\n",
      "[3] entries found for [other]\n",
      "[3] entries found for [people]\n",
      "[3] entries found for [her]\n",
      "[3] entries found for [than]\n",
      "[3] entries found for [over]\n",
      "[2] entries found for [into]\n",
      "[3] entries found for [last]\n",
      "[3] entries found for [some]\n",
      "[3] entries found for [government]\n",
      "[3] entries found for [time]\n",
      "[3] entries found for [you]\n",
      "[2] entries found for [years]\n",
      "[3] entries found for [if]\n",
      "[3] entries found for [no]\n",
      "[3] entries found for [world]\n",
      "[3] entries found for [can]\n",
      "[2] entries found for [three]\n",
      "[3] entries found for [do]\n",
      "[3] entries found for [president]\n",
      "[3] entries found for [only]\n",
      "[3] entries found for [state]\n",
      "[2] entries found for [million]\n",
      "[3] entries found for [could]\n",
      "[2] entries found for [us]\n",
      "[3] entries found for [most]\n",
      "[2] entries found for [against]\n",
      "[3] entries found for [so]\n",
      "[3] entries found for [them]\n",
      "[3] entries found for [what]\n",
      "[3] entries found for [him]\n",
      "[3] entries found for [united]\n",
      "[3] entries found for [during]\n",
      "[3] entries found for [before]\n",
      "[2] entries found for [may]\n",
      "[3] entries found for [since]\n",
      "[3] entries found for [many]\n",
      "[2] entries found for [while]\n",
      "[3] entries found for [where]\n",
      "[3] entries found for [states]\n",
      "[3] entries found for [because]\n",
      "[3] entries found for [now]\n",
      "[2] entries found for [city]\n",
      "[3] entries found for [made]\n",
      "[3] entries found for [like]\n",
      "[3] entries found for [did]\n",
      "[3] entries found for [just]\n",
      "[3] entries found for [national]\n",
      "[3] entries found for [day]\n",
      "[2] entries found for [country]\n",
      "[3] entries found for [under]\n",
      "[3] entries found for [such]\n",
      "[3] entries found for [second]\n",
      "[3] entries found for [then]\n",
      "[3] entries found for [company]\n",
      "[2] entries found for [group]\n",
      "[3] entries found for [any]\n",
      "[3] entries found for [through]\n",
      "[2] entries found for [china]\n",
      "[3] entries found for [four]\n",
      "[2] entries found for [being]\n",
      "[3] entries found for [down]\n",
      "[3] entries found for [war]\n",
      "[3] entries found for [back]\n",
      "[2] entries found for [off]\n",
      "[2] entries found for [south]\n",
      "[2] entries found for [american]\n",
      "[2] entries found for [police]\n",
      "[3] entries found for [well]\n",
      "[3] entries found for [including]\n",
      "[2] entries found for [team]\n",
      "[2] entries found for [international]\n",
      "[2] entries found for [week]\n",
      "[2] entries found for [officials]\n",
      "[2] entries found for [still]\n",
      "[2] entries found for [both]\n",
      "[3] entries found for [even]\n",
      "[3] entries found for [high]\n",
      "[2] entries found for [part]\n",
      "[2] entries found for [told]\n",
      "[3] entries found for [those]\n",
      "[3] entries found for [end]\n",
      "[3] entries found for [these]\n",
      "[3] entries found for [make]\n",
      "[2] entries found for [billion]\n",
      "[2] entries found for [work]\n",
      "[3] entries found for [our]\n",
      "[2] entries found for [home]\n",
      "[3] entries found for [school]\n",
      "[3] entries found for [party]\n",
      "[3] entries found for [house]\n",
      "[3] entries found for [old]\n",
      "[3] entries found for [get]\n",
      "[3] entries found for [another]\n",
      "[3] entries found for [news]\n",
      "[2] entries found for [long]\n",
      "[3] entries found for [five]\n",
      "[3] entries found for [called]\n",
      "[3] entries found for [military]\n",
      "[3] entries found for [way]\n",
      "[2] entries found for [used]\n",
      "[3] entries found for [much]\n",
      "[2] entries found for [next]\n",
      "[3] entries found for [here]\n",
      "[3] entries found for [should]\n",
      "[3] entries found for [take]\n",
      "[3] entries found for [very]\n",
      "[3] entries found for [my]\n",
      "[3] entries found for [security]\n",
      "[2] entries found for [season]\n",
      "[2] entries found for [york]\n",
      "[3] entries found for [how]\n",
      "[2] entries found for [public]\n",
      "[2] entries found for [according]\n",
      "[2] entries found for [court]\n",
      "[3] entries found for [say]\n",
      "[2] entries found for [around]\n",
      "[3] entries found for [until]\n",
      "[2] entries found for [political]\n",
      "[3] entries found for [says]\n",
      "[2] entries found for [however]\n",
      "[2] entries found for [family]\n",
      "[2] entries found for [life]\n",
      "[2] entries found for [same]\n",
      "[3] entries found for [general]\n",
      "[2] entries found for [left]\n",
      "[3] entries found for [good]\n",
      "[3] entries found for [top]\n",
      "[3] entries found for [going]\n",
      "[3] entries found for [number]\n",
      "[2] entries found for [major]\n",
      "[2] entries found for [known]\n",
      "[3] entries found for [won]\n",
      "[3] entries found for [six]\n",
      "[2] entries found for [month]\n",
      "[2] entries found for [dollars]\n",
      "[3] entries found for [bank]\n",
      "[2] entries found for [iraq]\n",
      "[3] entries found for [use]\n",
      "[2] entries found for [members]\n",
      "[2] entries found for [each]\n",
      "[2] entries found for [place]\n",
      "[3] entries found for [go]\n",
      "[2] entries found for [among]\n",
      "[3] entries found for [third]\n",
      "[3] entries found for [times]\n",
      "[3] entries found for [took]\n",
      "[3] entries found for [right]\n",
      "[2] entries found for [days]\n",
      "[2] entries found for [local]\n",
      "[2] entries found for [economic]\n",
      "[2] entries found for [countries]\n",
      "[3] entries found for [see]\n",
      "[2] entries found for [best]\n",
      "[2] entries found for [report]\n",
      "[3] entries found for [killed]\n",
      "[2] entries found for [held]\n",
      "[2] entries found for [business]\n",
      "[2] entries found for [west]\n",
      "[2] entries found for [does]\n",
      "[2] entries found for [came]\n",
      "[3] entries found for [law]\n",
      "[2] entries found for [women]\n",
      "[2] entries found for [power]\n",
      "[3] entries found for [think]\n",
      "[2] entries found for [service]\n",
      "[2] entries found for [children]\n",
      "[2] entries found for [bush]\n",
      "[3] entries found for [show]\n",
      "[2] entries found for [help]\n",
      "[3] entries found for [system]\n",
      "[2] entries found for [support]\n",
      "[2] entries found for [office]\n",
      "[3] entries found for [me]\n",
      "[2] entries found for [meeting]\n",
      "[2] entries found for [expected]\n",
      "[2] entries found for [league]\n",
      "[2] entries found for [reported]\n",
      "[2] entries found for [without]\n",
      "[2] entries found for [white]\n",
      "[3] entries found for [history]\n",
      "[3] entries found for [man]\n",
      "[2] entries found for [became]\n",
      "[2] entries found for [want]\n",
      "[2] entries found for [few]\n",
      "[2] entries found for [run]\n",
      "[2] entries found for [money]\n",
      "[3] entries found for [open]\n",
      "[3] entries found for [trade]\n",
      "[2] entries found for [center]\n",
      "[2] entries found for [israel]\n",
      "[3] entries found for [oil]\n",
      "[3] entries found for [too]\n",
      "[3] entries found for [win]\n",
      "[2] entries found for [east]\n",
      "[2] entries found for [central]\n",
      "[3] entries found for [air]\n",
      "[3] entries found for [come]\n",
      "[2] entries found for [leader]\n",
      "[2] entries found for [army]\n",
      "[3] entries found for [never]\n",
      "[3] entries found for [little]\n",
      "[3] entries found for [companies]\n",
      "[2] entries found for [least]\n",
      "[3] entries found for [put]\n",
      "[2] entries found for [half]\n",
      "[2] entries found for [saying]\n",
      "[2] entries found for [know]\n",
      "[2] entries found for [force]\n",
      "[3] entries found for [great]\n",
      "[3] entries found for [small]\n",
      "[2] entries found for [department]\n",
      "[3] entries found for [every]\n",
      "[2] entries found for [japan]\n",
      "[2] entries found for [head]\n",
      "[2] entries found for [ago]\n",
      "[3] entries found for [night]\n",
      "[3] entries found for [big]\n",
      "[2] entries found for [election]\n",
      "[2] entries found for [director]\n",
      "[2] entries found for [talks]\n",
      "[2] entries found for [far]\n",
      "[3] entries found for [today]\n",
      "[2] entries found for [although]\n",
      "[3] entries found for [again]\n",
      "[2] entries found for [close]\n",
      "[2] entries found for [record]\n",
      "[2] entries found for [along]\n",
      "[2] entries found for [france]\n",
      "[2] entries found for [went]\n",
      "[3] entries found for [point]\n",
      "[2] entries found for [must]\n",
      "[3] entries found for [your]\n",
      "[2] entries found for [financial]\n",
      "[2] entries found for [recent]\n",
      "[2] entries found for [campaign]\n",
      "[2] entries found for [troops]\n",
      "[3] entries found for [whether]\n",
      "[2] entries found for [lost]\n",
      "[2] entries found for [music]\n",
      "[3] entries found for [got]\n",
      "[2] entries found for [need]\n",
      "[2] entries found for [already]\n",
      "[2] entries found for [might]\n",
      "[3] entries found for [free]\n",
      "[2] entries found for [away]\n",
      "[3] entries found for [others]\n",
      "[3] entries found for [within]\n",
      "[3] entries found for [large]\n",
      "[3] entries found for [press]\n",
      "[2] entries found for [water]\n",
      "[2] entries found for [making]\n",
      "[3] entries found for [deal]\n",
      "[2] entries found for [side]\n",
      "[2] entries found for [seven]\n",
      "[3] entries found for [better]\n",
      "[3] entries found for [once]\n",
      "[2] entries found for [clinton]\n",
      "[3] entries found for [committee]\n",
      "[3] entries found for [building]\n",
      "[2] entries found for [conference]\n",
      "[2] entries found for [club]\n",
      "[2] entries found for [america]\n",
      "[2] entries found for [given]\n",
      "[3] entries found for [give]\n",
      "[2] entries found for [often]\n",
      "[2] entries found for [television]\n",
      "[3] entries found for [order]\n",
      "[3] entries found for [young]\n",
      "[3] entries found for [start]\n",
      "[2] entries found for [administration]\n",
      "[3] entries found for [nations]\n",
      "[2] entries found for [human]\n",
      "[2] entries found for [defense]\n",
      "[2] entries found for [asked]\n",
      "[2] entries found for [total]\n",
      "[3] entries found for [bill]\n",
      "[2] entries found for [important]\n",
      "[2] entries found for [southern]\n",
      "[2] entries found for [move]\n",
      "[2] entries found for [fire]\n",
      "[2] entries found for [rose]\n",
      "[3] entries found for [nuclear]\n",
      "[2] entries found for [street]\n",
      "[3] entries found for [taken]\n",
      "[2] entries found for [media]\n",
      "[3] entries found for [different]\n",
      "[3] entries found for [secretary]\n",
      "[3] entries found for [college]\n",
      "[2] entries found for [working]\n",
      "[3] entries found for [community]\n",
      "[3] entries found for [level]\n",
      "[2] entries found for [largest]\n",
      "[2] entries found for [change]\n",
      "[2] entries found for [nation]\n",
      "[2] entries found for [weeks]\n",
      "[2] entries found for [having]\n",
      "[2] entries found for [research]\n",
      "[2] entries found for [black]\n",
      "[2] entries found for [services]\n",
      "[2] entries found for [story]\n",
      "[3] entries found for [policy]\n",
      "[2] entries found for [lot]\n",
      "[2] entries found for [across]\n",
      "[2] entries found for [football]\n",
      "[2] entries found for [vote]\n",
      "[2] entries found for [fell]\n",
      "[2] entries found for [seen]\n",
      "[2] entries found for [iran]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2] entries found for [agreement]\n",
      "[3] entries found for [started]\n",
      "[2] entries found for [growth]\n",
      "[3] entries found for [yet]\n",
      "[2] entries found for [western]\n",
      "[3] entries found for [special]\n",
      "[3] entries found for [interest]\n",
      "[3] entries found for [strong]\n",
      "[2] entries found for [england]\n",
      "[2] entries found for [named]\n",
      "[3] entries found for [real]\n",
      "[2] entries found for [rate]\n",
      "[2] entries found for [race]\n",
      "[2] entries found for [nearly]\n",
      "[2] entries found for [enough]\n",
      "[3] entries found for [keep]\n",
      "[3] entries found for [call]\n",
      "[3] entries found for [taking]\n",
      "[3] entries found for [outside]\n",
      "[3] entries found for [really]\n",
      "[3] entries found for [almost]\n",
      "[2] entries found for [single]\n",
      "[3] entries found for [leading]\n",
      "[2] entries found for [trying]\n",
      "[2] entries found for [find]\n",
      "[2] entries found for [minutes]\n",
      "[3] entries found for [together]\n",
      "[3] entries found for [hard]\n",
      "[2] entries found for [hours]\n",
      "[2] entries found for [san]\n",
      "[2] entries found for [executive]\n",
      "[2] entries found for [areas]\n",
      "[2] entries found for [face]\n",
      "[2] entries found for [park]\n",
      "[2] entries found for [father]\n",
      "[2] entries found for [son]\n",
      "[3] entries found for [education]\n",
      "[2] entries found for [average]\n",
      "[2] entries found for [pay]\n",
      "[3] entries found for [something]\n",
      "[2] entries found for [gave]\n",
      "[2] entries found for [victory]\n",
      "[3] entries found for [low]\n",
      "[3] entries found for [things]\n",
      "[2] entries found for [post]\n",
      "[2] entries found for [social]\n",
      "[2] entries found for [continue]\n",
      "[3] entries found for [ever]\n",
      "[3] entries found for [look]\n",
      "[2] entries found for [chairman]\n",
      "[2] entries found for [job]\n",
      "[2] entries found for [able]\n",
      "[2] entries found for [front]\n",
      "[2] entries found for [problems]\n",
      "[2] entries found for [private]\n",
      "[2] entries found for [list]\n",
      "[2] entries found for [built]\n",
      "[2] entries found for [live]\n",
      "[2] entries found for [form]\n",
      "[2] entries found for [increase]\n",
      "[2] entries found for [fourth]\n",
      "[3] entries found for [always]\n",
      "[3] entries found for [king]\n",
      "[2] entries found for [tax]\n",
      "[3] entries found for [middle]\n",
      "[2] entries found for [meet]\n",
      "[2] entries found for [wife]\n",
      "[2] entries found for [position]\n",
      "[3] entries found for [border]\n",
      "[2] entries found for [soon]\n",
      "[3] entries found for [believe]\n",
      "[2] entries found for [organization]\n",
      "[2] entries found for [weapons]\n",
      "[3] entries found for [why]\n",
      "[3] entries found for [nine]\n",
      "[2] entries found for [summer]\n",
      "[2] entries found for [wanted]\n",
      "[2] entries found for [republican]\n",
      "[2] entries found for [act]\n",
      "[2] entries found for [recently]\n",
      "[2] entries found for [course]\n",
      "[2] entries found for [problem]\n",
      "[2] entries found for [senate]\n",
      "[3] entries found for [done]\n",
      "[2] entries found for [living]\n",
      "[2] entries found for [care]\n",
      "[2] entries found for [signed]\n",
      "[2] entries found for [art]\n",
      "[3] entries found for [presidential]\n",
      "[2] entries found for [obama]\n",
      "[3] entries found for [morning]\n",
      "[2] entries found for [dead]\n",
      "[2] entries found for [instead]\n",
      "[2] entries found for [coming]\n",
      "[2] entries found for [civil]\n",
      "[2] entries found for [woman]\n",
      "[2] entries found for [similar]\n",
      "[2] entries found for [los]\n",
      "[2] entries found for [running]\n",
      "[2] entries found for [fighting]\n",
      "[2] entries found for [mark]\n",
      "[2] entries found for [thought]\n",
      "[2] entries found for [saw]\n",
      "[3] entries found for [hope]\n",
      "[2] entries found for [love]\n",
      "[2] entries found for [wrote]\n",
      "[3] entries found for [stop]\n",
      "[3] entries found for [fight]\n",
      "[2] entries found for [turned]\n",
      "[2] entries found for [fact]\n",
      "[3] entries found for [vice]\n",
      "[2] entries found for [mexico]\n",
      "[3] entries found for [common]\n",
      "[3] entries found for [looking]\n",
      "[2] entries found for [rates]\n",
      "[2] entries found for [loss]\n",
      "[2] entries found for [justice]\n",
      "[3] entries found for [thousands]\n",
      "[2] entries found for [rather]\n",
      "[2] entries found for [fund]\n",
      "[2] entries found for [thing]\n",
      "[2] entries found for [winning]\n",
      "[2] entries found for [getting]\n",
      "[3] entries found for [biggest]\n",
      "[3] entries found for [let]\n",
      "[2] entries found for [means]\n",
      "[2] entries found for [turn]\n",
      "[2] entries found for [leave]\n",
      "[3] entries found for [person]\n",
      "[2] entries found for [either]\n",
      "[2] entries found for [majority]\n",
      "[3] entries found for [society]\n",
      "[2] entries found for [stage]\n",
      "[3] entries found for [am]\n",
      "[2] entries found for [doing]\n",
      "[2] entries found for [families]\n",
      "[2] entries found for [construction]\n",
      "[2] entries found for [various]\n",
      "[2] entries found for [kind]\n",
      "[2] entries found for [airport]\n",
      "[2] entries found for [room]\n",
      "[2] entries found for [angeles]\n",
      "[3] entries found for [nothing]\n",
      "[2] entries found for [sea]\n",
      "[3] entries found for [bring]\n",
      "[2] entries found for [remain]\n",
      "[2] entries found for [allow]\n",
      "[2] entries found for [florida]\n",
      "[2] entries found for [created]\n",
      "[2] entries found for [events]\n",
      "[2] entries found for [beat]\n",
      "[3] entries found for [probably]\n",
      "[3] entries found for [lives]\n",
      "[2] entries found for [yen]\n",
      "[3] entries found for [bad]\n",
      "[2] entries found for [toward]\n",
      "[2] entries found for [buy]\n",
      "[2] entries found for [green]\n",
      "[2] entries found for [poor]\n",
      "[2] entries found for [question]\n",
      "[2] entries found for [prison]\n",
      "[3] entries found for [feel]\n",
      "[3] entries found for [governor]\n",
      "[3] entries found for [wall]\n",
      "[2] entries found for [chance]\n",
      "[2] entries found for [elected]\n",
      "[2] entries found for [wants]\n",
      "[2] entries found for [serious]\n",
      "[2] entries found for [immediately]\n",
      "[2] entries found for [parts]\n",
      "[2] entries found for [la]\n",
      "[3] entries found for [jobs]\n",
      "[2] entries found for [heart]\n",
      "[2] entries found for [planned]\n",
      "[2] entries found for [grand]\n",
      "[2] entries found for [institute]\n",
      "[2] entries found for [labor]\n",
      "[2] entries found for [sometimes]\n",
      "[2] entries found for [raised]\n",
      "[3] entries found for [talk]\n",
      "[2] entries found for [rules]\n",
      "[2] entries found for [beginning]\n",
      "[3] entries found for [stay]\n",
      "[2] entries found for [worth]\n",
      "[3] entries found for [friends]\n",
      "[3] entries found for [anything]\n",
      "[2] entries found for [sign]\n",
      "[2] entries found for [spending]\n",
      "[2] entries found for [e]\n",
      "[2] entries found for [whole]\n",
      "[3] entries found for [cannot]\n",
      "[2] entries found for [matter]\n",
      "[2] entries found for [access]\n",
      "[2] entries found for [compared]\n",
      "[2] entries found for [break]\n",
      "[2] entries found for [turkey]\n",
      "[2] entries found for [usually]\n",
      "[2] entries found for [books]\n",
      "[2] entries found for [chicago]\n",
      "[3] entries found for [hundreds]\n",
      "[2] entries found for [cause]\n",
      "[2] entries found for [idea]\n",
      "[2] entries found for [domestic]\n",
      "[3] entries found for [everything]\n",
      "[2] entries found for [quickly]\n",
      "[2] entries found for [finance]\n",
      "[2] entries found for [cities]\n",
      "[2] entries found for [blue]\n",
      "[2] entries found for [richard]\n",
      "[3] entries found for [ready]\n",
      "[2] entries found for [build]\n",
      "[2] entries found for [designed]\n",
      "[2] entries found for [certain]\n",
      "[2] entries found for [negotiations]\n",
      "[2] entries found for [bomb]\n",
      "[2] entries found for [warned]\n",
      "[2] entries found for [attorney]\n",
      "[2] entries found for [intelligence]\n",
      "[2] entries found for [hotel]\n",
      "[2] entries found for [finally]\n",
      "[3] entries found for [magazine]\n",
      "[2] entries found for [whom]\n",
      "[2] entries found for [moving]\n",
      "[2] entries found for [carried]\n",
      "[2] entries found for [candidate]\n",
      "[2] entries found for [fifth]\n",
      "[2] entries found for [highest]\n",
      "[2] entries found for [employees]\n",
      "[2] entries found for [sell]\n",
      "[2] entries found for [mostly]\n",
      "[2] entries found for [reason]\n",
      "[2] entries found for [focus]\n",
      "[2] entries found for [wide]\n",
      "[2] entries found for [figure]\n",
      "[3] entries found for [cars]\n",
      "[2] entries found for [double]\n",
      "[2] entries found for [kept]\n",
      "[2] entries found for [terrorism]\n",
      "[2] entries found for [speech]\n",
      "[2] entries found for [plane]\n",
      "[2] entries found for [someone]\n",
      "[2] entries found for [comment]\n",
      "[2] entries found for [huge]\n",
      "[2] entries found for [mayor]\n",
      "[2] entries found for [cover]\n",
      "[2] entries found for [couple]\n",
      "[2] entries found for [positive]\n",
      "[2] entries found for [mail]\n",
      "[3] entries found for [crime]\n",
      "[3] entries found for [stand]\n",
      "[2] entries found for [seems]\n",
      "[2] entries found for [felt]\n",
      "[2] entries found for [leadership]\n",
      "[3] entries found for [tell]\n",
      "[2] entries found for [straight]\n",
      "[2] entries found for [progress]\n",
      "[2] entries found for [words]\n",
      "[2] entries found for [politics]\n",
      "[2] entries found for [word]\n",
      "[3] entries found for [wounded]\n",
      "[2] entries found for [pro]\n",
      "[2] entries found for [broke]\n",
      "[3] entries found for [everyone]\n",
      "[3] entries found for [actually]\n",
      "[2] entries found for [equipment]\n",
      "[3] entries found for [greater]\n",
      "[2] entries found for [primary]\n",
      "[2] entries found for [ways]\n",
      "[2] entries found for [debate]\n",
      "[2] entries found for [places]\n",
      "[2] entries found for [illegal]\n",
      "[2] entries found for [message]\n",
      "[2] entries found for [originally]\n",
      "[2] entries found for [entire]\n",
      "[2] entries found for [product]\n",
      "[2] entries found for [conservative]\n",
      "[2] entries found for [protection]\n",
      "[2] entries found for [read]\n",
      "[2] entries found for [foundation]\n",
      "[2] entries found for [names]\n",
      "[2] entries found for [train]\n",
      "[2] entries found for [discuss]\n",
      "[3] entries found for [perhaps]\n",
      "[2] entries found for [laws]\n",
      "[2] entries found for [buildings]\n",
      "[2] entries found for [quality]\n",
      "[2] entries found for [missing]\n",
      "[2] entries found for [beyond]\n",
      "[2] entries found for [section]\n",
      "[2] entries found for [knew]\n",
      "[2] entries found for [calling]\n",
      "[2] entries found for [gone]\n",
      "[2] entries found for [powerful]\n",
      "[2] entries found for [losing]\n",
      "[2] entries found for [awards]\n",
      "[3] entries found for [send]\n",
      "[2] entries found for [protest]\n",
      "[2] entries found for [consider]\n",
      "[2] entries found for [citizens]\n",
      "[2] entries found for [happened]\n",
      "[2] entries found for [mike]\n",
      "[3] entries found for [businesses]\n",
      "[2] entries found for [bit]\n",
      "[3] entries found for [tough]\n",
      "[2] entries found for [moment]\n",
      "[2] entries found for [drugs]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2] entries found for [fast]\n",
      "[2] entries found for [boy]\n",
      "[3] entries found for [worst]\n",
      "[2] entries found for [weather]\n",
      "[2] entries found for [bridge]\n",
      "[2] entries found for [cancer]\n",
      "[2] entries found for [crimes]\n",
      "[2] entries found for [sun]\n",
      "[2] entries found for [becoming]\n",
      "[2] entries found for [gulf]\n",
      "[3] entries found for [gets]\n",
      "[2] entries found for [corporate]\n",
      "[2] entries found for [easy]\n",
      "[2] entries found for [choice]\n",
      "[2] entries found for [add]\n",
      "[2] entries found for [reduce]\n",
      "[2] entries found for [crowd]\n",
      "[2] entries found for [fine]\n",
      "[2] entries found for [environmental]\n",
      "[2] entries found for [subject]\n",
      "[2] entries found for [mean]\n",
      "[2] entries found for [ten]\n",
      "[3] entries found for [ask]\n",
      "[3] entries found for [watch]\n",
      "[2] entries found for [else]\n",
      "[3] entries found for [rich]\n",
      "[3] entries found for [happy]\n",
      "[3] entries found for [save]\n",
      "[2] entries found for [article]\n",
      "[2] entries found for [standing]\n",
      "[2] entries found for [doctors]\n",
      "[2] entries found for [sanctions]\n",
      "[2] entries found for [bought]\n",
      "[2] entries found for [secret]\n",
      "[2] entries found for [highly]\n",
      "[2] entries found for [fair]\n",
      "[2] entries found for [spoke]\n",
      "[2] entries found for [documents]\n",
      "[3] entries found for [super]\n",
      "[2] entries found for [directly]\n",
      "[3] entries found for [earth]\n",
      "[2] entries found for [islands]\n",
      "[2] entries found for [swiss]\n",
      "[2] entries found for [trust]\n",
      "[2] entries found for [guy]\n",
      "[2] entries found for [fully]\n",
      "[2] entries found for [c]\n",
      "[2] entries found for [cold]\n",
      "[3] entries found for [safe]\n",
      "[3] entries found for [maybe]\n",
      "[3] entries found for [millions]\n",
      "[3] entries found for [poll]\n",
      "[2] entries found for [lose]\n",
      "[2] entries found for [talking]\n",
      "[2] entries found for [specific]\n",
      "[3] entries found for [kill]\n",
      "[2] entries found for [broadcast]\n",
      "[3] entries found for [pretty]\n",
      "[2] entries found for [happen]\n",
      "[2] entries found for [fed]\n",
      "[2] entries found for [ending]\n",
      "[2] entries found for [agent]\n",
      "[3] entries found for [massive]\n",
      "[2] entries found for [advantage]\n",
      "[2] entries found for [respect]\n",
      "[2] entries found for [joe]\n",
      "[2] entries found for [heads]\n",
      "[2] entries found for [pennsylvania]\n",
      "[2] entries found for [ohio]\n",
      "[2] entries found for [m]\n",
      "[3] entries found for [ones]\n",
      "[2] entries found for [voted]\n",
      "[2] entries found for [slightly]\n",
      "[2] entries found for [allowing]\n",
      "[2] entries found for [spend]\n",
      "[2] entries found for [loans]\n",
      "[2] entries found for [certainly]\n",
      "[2] entries found for [syria]\n",
      "[2] entries found for [effective]\n",
      "[2] entries found for [representative]\n",
      "[2] entries found for [location]\n",
      "[3] entries found for [audience]\n",
      "[2] entries found for [uses]\n",
      "[2] entries found for [clearly]\n",
      "[2] entries found for [combat]\n",
      "[3] entries found for [except]\n",
      "[2] entries found for [golf]\n",
      "[2] entries found for [deficit]\n",
      "[2] entries found for [agents]\n",
      "[2] entries found for [meant]\n",
      "[2] entries found for [fox]\n",
      "[2] entries found for [waiting]\n",
      "[2] entries found for [none]\n",
      "[2] entries found for [prisoners]\n",
      "[2] entries found for [unless]\n",
      "[2] entries found for [strip]\n",
      "[2] entries found for [voting]\n",
      "[2] entries found for [attacked]\n",
      "[2] entries found for [simple]\n",
      "[2] entries found for [reduced]\n",
      "[2] entries found for [raising]\n",
      "[2] entries found for [t]\n",
      "[3] entries found for [steel]\n",
      "[2] entries found for [opinion]\n",
      "[2] entries found for [answer]\n",
      "[2] entries found for [guilty]\n",
      "[2] entries found for [strength]\n",
      "[2] entries found for [page]\n",
      "[2] entries found for [buying]\n",
      "[2] entries found for [knows]\n",
      "[2] entries found for [speak]\n",
      "[2] entries found for [leaves]\n",
      "[2] entries found for [arizona]\n",
      "[2] entries found for [dozens]\n",
      "[2] entries found for [senator]\n",
      "[2] entries found for [miss]\n",
      "[2] entries found for [putting]\n",
      "[2] entries found for [singer]\n",
      "[2] entries found for [communities]\n",
      "[2] entries found for [immigration]\n",
      "[2] entries found for [plus]\n",
      "[2] entries found for [hear]\n",
      "[2] entries found for [learned]\n",
      "[3] entries found for [guys]\n",
      "[2] entries found for [meaning]\n",
      "[2] entries found for [commerce]\n",
      "[2] entries found for [sets]\n",
      "[2] entries found for [row]\n",
      "[2] entries found for [closing]\n",
      "[2] entries found for [enter]\n",
      "[3] entries found for [ships]\n",
      "[2] entries found for [flying]\n",
      "[3] entries found for [lies]\n",
      "[2] entries found for [fly]\n",
      "[2] entries found for [politicians]\n",
      "[2] entries found for [piece]\n",
      "[2] entries found for [honor]\n",
      "[3] entries found for [walk]\n",
      "[2] entries found for [bottom]\n",
      "[2] entries found for [disaster]\n",
      "[2] entries found for [mine]\n",
      "[2] entries found for [thinking]\n",
      "[2] entries found for [sentence]\n",
      "[2] entries found for [write]\n",
      "[2] entries found for [replace]\n",
      "[2] entries found for [speaker]\n",
      "[2] entries found for [becomes]\n",
      "[2] entries found for [wait]\n",
      "[2] entries found for [unemployment]\n",
      "[3] entries found for [enforcement]\n",
      "[2] entries found for [balance]\n",
      "[2] entries found for [seriously]\n",
      "[2] entries found for [feeling]\n",
      "[2] entries found for [protesters]\n",
      "[2] entries found for [presidency]\n",
      "[2] entries found for [favor]\n",
      "[2] entries found for [negative]\n",
      "[2] entries found for [pop]\n",
      "[2] entries found for [quick]\n",
      "[3] entries found for [veteran]\n",
      "[3] entries found for [everybody]\n",
      "[3] entries found for [sort]\n",
      "[2] entries found for [greatest]\n",
      "[2] entries found for [perfect]\n",
      "[3] entries found for [core]\n",
      "[3] entries found for [manufacturing]\n",
      "[2] entries found for [watching]\n",
      "[3] entries found for [whatever]\n",
      "[2] entries found for [dallas]\n",
      "[3] entries found for [sir]\n",
      "[2] entries found for [paying]\n",
      "[2] entries found for [treated]\n",
      "[3] entries found for [remember]\n",
      "[2] entries found for [apart]\n",
      "[2] entries found for [otherwise]\n",
      "[2] entries found for [delivered]\n",
      "[2] entries found for [worse]\n",
      "[3] entries found for [yes]\n",
      "[2] entries found for [planes]\n",
      "[2] entries found for [minnesota]\n",
      "[3] entries found for [exactly]\n",
      "[2] entries found for [changing]\n",
      "[2] entries found for [networks]\n",
      "[2] entries found for [telling]\n",
      "[2] entries found for [motor]\n",
      "[3] entries found for [beating]\n",
      "[2] entries found for [grown]\n",
      "[3] entries found for [yesterday]\n",
      "[2] entries found for [horse]\n",
      "[2] entries found for [represent]\n",
      "[2] entries found for [partnership]\n",
      "[2] entries found for [dream]\n",
      "[2] entries found for [values]\n",
      "[2] entries found for [nationwide]\n",
      "[2] entries found for [cards]\n",
      "[2] entries found for [enemy]\n",
      "[2] entries found for [kerry]\n",
      "[2] entries found for [signing]\n",
      "[3] entries found for [deals]\n",
      "[2] entries found for [competitive]\n",
      "[2] entries found for [worried]\n",
      "[2] entries found for [borders]\n",
      "[2] entries found for [stronger]\n",
      "[2] entries found for [equal]\n",
      "[2] entries found for [shape]\n",
      "[2] entries found for [reporter]\n",
      "[2] entries found for [mentioned]\n",
      "[2] entries found for [wake]\n",
      "[2] entries found for [ah]\n",
      "[3] entries found for [nice]\n",
      "[2] entries found for [regulations]\n",
      "[2] entries found for [representing]\n",
      "[2] entries found for [bigger]\n",
      "[2] entries found for [ill]\n",
      "[3] entries found for [oh]\n",
      "[2] entries found for [rating]\n",
      "[2] entries found for [jets]\n",
      "[2] entries found for [papers]\n",
      "[3] entries found for [trillion]\n",
      "[2] entries found for [threw]\n",
      "[2] entries found for [gross]\n",
      "[3] entries found for [nobody]\n",
      "[2] entries found for [tiny]\n",
      "[2] entries found for [trains]\n",
      "[2] entries found for [promise]\n",
      "[2] entries found for [virtually]\n",
      "[2] entries found for [kick]\n",
      "[3] entries found for [beautiful]\n",
      "[2] entries found for [happens]\n",
      "[2] entries found for [throw]\n",
      "[3] entries found for [explain]\n",
      "[3] entries found for [amendment]\n",
      "[3] entries found for [walls]\n",
      "[2] entries found for [radical]\n",
      "[3] entries found for [obviously]\n",
      "[2] entries found for [handle]\n",
      "[2] entries found for [dealing]\n",
      "[2] entries found for [fixed]\n",
      "[2] entries found for [wish]\n",
      "[3] entries found for [patrol]\n",
      "[2] entries found for [carrier]\n",
      "[2] entries found for [stuff]\n",
      "[2] entries found for [arena]\n",
      "[2] entries found for [fighter]\n",
      "[2] entries found for [normally]\n",
      "[2] entries found for [delegates]\n",
      "[2] entries found for [worry]\n",
      "[2] entries found for [upset]\n",
      "[2] entries found for [waste]\n",
      "[2] entries found for [indiana]\n",
      "[2] entries found for [hero]\n",
      "[2] entries found for [pan]\n",
      "[3] entries found for [veterans]\n",
      "[2] entries found for [tens]\n",
      "[2] entries found for [pilots]\n",
      "[2] entries found for [enjoy]\n",
      "[2] entries found for [pages]\n",
      "[2] entries found for [ratings]\n",
      "[2] entries found for [fat]\n",
      "[2] entries found for [fill]\n",
      "[3] entries found for [please]\n",
      "[2] entries found for [louisiana]\n",
      "[2] entries found for [mention]\n",
      "[2] entries found for [politically]\n",
      "[2] entries found for [donald]\n",
      "[2] entries found for [senators]\n",
      "[2] entries found for [mosque]\n",
      "[2] entries found for [badly]\n",
      "[2] entries found for [shell]\n",
      "[2] entries found for [chapter]\n",
      "[2] entries found for [homeland]\n",
      "[2] entries found for [anger]\n",
      "[3] entries found for [absolutely]\n",
      "[2] entries found for [rush]\n",
      "[2] entries found for [notice]\n",
      "[2] entries found for [consequences]\n",
      "[3] entries found for [tomorrow]\n",
      "[2] entries found for [boats]\n",
      "[2] entries found for [hostages]\n",
      "[3] entries found for [zero]\n",
      "[2] entries found for [extent]\n",
      "[2] entries found for [guarantee]\n",
      "[2] entries found for [saved]\n",
      "[2] entries found for [vital]\n",
      "[2] entries found for [sounds]\n",
      "[2] entries found for [plenty]\n",
      "[2] entries found for [loved]\n",
      "[2] entries found for [potentially]\n",
      "[2] entries found for [negotiating]\n",
      "[2] entries found for [conservatives]\n",
      "[2] entries found for [thinks]\n",
      "[2] entries found for [realize]\n",
      "[2] entries found for [lots]\n",
      "[2] entries found for [victories]\n",
      "[2] entries found for [headlines]\n",
      "[2] entries found for [regulation]\n",
      "[2] entries found for [proud]\n",
      "[2] entries found for [colonel]\n",
      "[3] entries found for [treat]\n",
      "[2] entries found for [carl]\n",
      "[3] entries found for [beef]\n",
      "[3] entries found for [anybody]\n",
      "[2] entries found for [totally]\n",
      "[3] entries found for [somebody]\n",
      "[3] entries found for [tonight]\n",
      "[2] entries found for [educated]\n",
      "[2] entries found for [puts]\n",
      "[2] entries found for [stick]\n",
      "[2] entries found for [businessman]\n",
      "[2] entries found for [smart]\n",
      "[2] entries found for [tone]\n",
      "[3] entries found for [forget]\n",
      "[2] entries found for [burning]\n",
      "[2] entries found for [gate]\n",
      "[2] entries found for [inner]\n",
      "[2] entries found for [hillary]\n",
      "[2] entries found for [wealthy]\n",
      "[2] entries found for [fairly]\n",
      "[2] entries found for [anywhere]\n",
      "[2] entries found for [unrest]\n",
      "[2] entries found for [personally]\n",
      "[3] entries found for [truly]\n",
      "[2] entries found for [boom]\n",
      "[2] entries found for [unfortunately]\n",
      "[2] entries found for [cameras]\n",
      "[2] entries found for [subjects]\n",
      "[2] entries found for [expression]\n",
      "[2] entries found for [imagine]\n",
      "[2] entries found for [parade]\n",
      "[2] entries found for [bobby]\n",
      "[2] entries found for [shed]\n",
      "[2] entries found for [happening]\n",
      "[2] entries found for [knight]\n",
      "[2] entries found for [knowing]\n",
      "[2] entries found for [routine]\n",
      "[2] entries found for [killer]\n",
      "[2] entries found for [likes]\n",
      "[2] entries found for [mistakes]\n",
      "[2] entries found for [pledge]\n",
      "[2] entries found for [listen]\n",
      "[2] entries found for [billions]\n",
      "[2] entries found for [tool]\n",
      "[2] entries found for [airports]\n",
      "[2] entries found for [yourself]\n",
      "[2] entries found for [rough]\n",
      "[2] entries found for [christians]\n",
      "[2] entries found for [factories]\n",
      "[2] entries found for [situations]\n",
      "[2] entries found for [achievement]\n",
      "[3] entries found for [guess]\n",
      "[2] entries found for [gotten]\n",
      "[2] entries found for [bombers]\n",
      "[2] entries found for [fix]\n",
      "[2] entries found for [sudden]\n",
      "[2] entries found for [enemies]\n",
      "[2] entries found for [fortune]\n",
      "[2] entries found for [chaos]\n",
      "[2] entries found for [siege]\n",
      "[2] entries found for [universe]\n",
      "[2] entries found for [donations]\n",
      "[3] entries found for [wonderful]\n",
      "[2] entries found for [qatar]\n",
      "[2] entries found for [nominee]\n",
      "[2] entries found for [silence]\n",
      "[2] entries found for [orlando]\n",
      "[2] entries found for [hate]\n",
      "[2] entries found for [respected]\n",
      "[2] entries found for [silent]\n",
      "[2] entries found for [everywhere]\n",
      "[2] entries found for [lifetime]\n",
      "[2] entries found for [sheriff]\n",
      "[2] entries found for [yemen]\n",
      "[2] entries found for [anyway]\n",
      "[2] entries found for [forever]\n",
      "[2] entries found for [favored]\n",
      "[2] entries found for [comparison]\n",
      "[2] entries found for [governors]\n",
      "[2] entries found for [professionals]\n",
      "[2] entries found for [zones]\n",
      "[2] entries found for [terrible]\n",
      "[2] entries found for [bridges]\n",
      "[3] entries found for [crowds]\n",
      "[3] entries found for [thank]\n",
      "[2] entries found for [luck]\n",
      "[2] entries found for [loaded]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2] entries found for [crazy]\n",
      "[2] entries found for [discipline]\n",
      "[3] entries found for [hell]\n",
      "[2] entries found for [shocked]\n",
      "[2] entries found for [afterwards]\n",
      "[2] entries found for [rely]\n",
      "[2] entries found for [endorsed]\n",
      "[2] entries found for [jihad]\n",
      "[2] entries found for [anymore]\n",
      "[2] entries found for [legally]\n",
      "[3] entries found for [amazing]\n",
      "[2] entries found for [surprising]\n",
      "[2] entries found for [cheese]\n",
      "[2] entries found for [hire]\n",
      "[2] entries found for [negotiated]\n",
      "[2] entries found for [negotiators]\n",
      "[2] entries found for [literally]\n",
      "[2] entries found for [weakness]\n",
      "[3] entries found for [tremendous]\n",
      "[2] entries found for [rebuild]\n",
      "[2] entries found for [businessmen]\n",
      "[2] entries found for [somehow]\n",
      "[2] entries found for [stopping]\n",
      "[2] entries found for [hopefully]\n",
      "[3] entries found for [folks]\n",
      "[2] entries found for [honored]\n",
      "[2] entries found for [whenever]\n",
      "[2] entries found for [taxpayers]\n",
      "[3] entries found for [sorry]\n",
      "[2] entries found for [mr]\n",
      "[2] entries found for [indianapolis]\n",
      "[2] entries found for [phenomenon]\n",
      "[2] entries found for [thirty]\n",
      "[2] entries found for [speeches]\n",
      "[2] entries found for [rid]\n",
      "[2] entries found for [warriors]\n",
      "[2] entries found for [trader]\n",
      "[2] entries found for [talented]\n",
      "[2] entries found for [highways]\n",
      "[2] entries found for [migration]\n",
      "[2] entries found for [ladies]\n",
      "[2] entries found for [joke]\n",
      "[2] entries found for [heroin]\n",
      "[2] entries found for [appreciate]\n",
      "[2] entries found for [debates]\n",
      "[2] entries found for [rebuilt]\n",
      "[3] entries found for [incredible]\n",
      "[2] entries found for [pence]\n",
      "[2] entries found for [rolls]\n",
      "[2] entries found for [floating]\n",
      "[2] entries found for [endorsement]\n",
      "[2] entries found for [fifty]\n",
      "[2] entries found for [portions]\n",
      "[2] entries found for [hey]\n",
      "[2] entries found for [deficits]\n",
      "[2] entries found for [landslide]\n",
      "[2] entries found for [sanders]\n",
      "[2] entries found for [economically]\n",
      "[3] entries found for [lets]\n",
      "[2] entries found for [sergeant]\n",
      "[2] entries found for [fixing]\n",
      "[2] entries found for [negotiation]\n",
      "[3] entries found for [fathers]\n",
      "[2] entries found for [consequence]\n",
      "[2] entries found for [subcommittee]\n",
      "[2] entries found for [yeah]\n",
      "[2] entries found for [battlefield]\n",
      "[2] entries found for [killers]\n",
      "[2] entries found for [practically]\n",
      "[2] entries found for [hatred]\n",
      "[3] entries found for [stupid]\n",
      "[2] entries found for [chopped]\n",
      "[2] entries found for [likewise]\n",
      "[2] entries found for [screaming]\n",
      "[2] entries found for [wherever]\n",
      "[2] entries found for [potatoes]\n",
      "[2] entries found for [snake]\n",
      "[2] entries found for [primaries]\n",
      "[2] entries found for [whoever]\n",
      "[2] entries found for [raped]\n",
      "[3] entries found for [applause]\n",
      "[2] entries found for [mails]\n",
      "[2] entries found for [globalization]\n",
      "[2] entries found for [honey]\n",
      "[2] entries found for [prosperous]\n",
      "[2] entries found for [passports]\n",
      "[3] entries found for [nuts]\n",
      "[2] entries found for [consultants]\n",
      "[2] entries found for [emmy]\n",
      "[2] entries found for [tore]\n",
      "[2] entries found for [premiums]\n",
      "[3] entries found for [horrible]\n",
      "[3] entries found for [conditioning]\n",
      "[3] entries found for [frankly]\n",
      "[2] entries found for [mosques]\n",
      "[2] entries found for [skip]\n",
      "[2] entries found for [nightclub]\n",
      "[3] entries found for [trump]\n",
      "[2] entries found for [admitting]\n",
      "[2] entries found for [slate]\n",
      "[2] entries found for [nasty]\n",
      "[2] entries found for [presbyterian]\n",
      "[2] entries found for [deserted]\n",
      "[2] entries found for [hostility]\n",
      "[2] entries found for [terrific]\n",
      "[2] entries found for [independents]\n",
      "[2] entries found for [vicious]\n",
      "[2] entries found for [pageant]\n",
      "[2] entries found for [pouring]\n",
      "[2] entries found for [unbelievable]\n",
      "[2] entries found for [gonna]\n",
      "[2] entries found for [fairness]\n",
      "[2] entries found for [iconic]\n",
      "[2] entries found for [darling]\n",
      "[2] entries found for [owe]\n",
      "[2] entries found for [rasmussen]\n",
      "[2] entries found for [unpredictable]\n",
      "[2] entries found for [petraeus]\n",
      "[2] entries found for [crippled]\n",
      "[2] entries found for [bye]\n",
      "[2] entries found for [debating]\n",
      "[2] entries found for [nonsense]\n",
      "[2] entries found for [laughter]\n",
      "[2] entries found for [hated]\n",
      "[2] entries found for [hello]\n",
      "[2] entries found for [landslides]\n",
      "[2] entries found for [fences]\n",
      "[2] entries found for [clue]\n",
      "[2] entries found for [honestly]\n",
      "[2] entries found for [rigged]\n",
      "[2] entries found for [bernie]\n",
      "[2] entries found for [paperwork]\n",
      "[2] entries found for [depleted]\n",
      "[2] entries found for [messenger]\n",
      "[2] entries found for [disadvantage]\n",
      "[2] entries found for [mankind]\n",
      "[3] entries found for [wow]\n",
      "[2] entries found for [tractor]\n",
      "[2] entries found for [fabulous]\n",
      "[2] entries found for [im]\n",
      "[2] entries found for [sadly]\n",
      "[2] entries found for [nicely]\n",
      "[2] entries found for [arenas]\n",
      "[2] entries found for [gentlemen]\n",
      "[2] entries found for [pretend]\n",
      "[2] entries found for [destabilize]\n",
      "[3] entries found for [okay]\n",
      "[2] entries found for [damn]\n",
      "[2] entries found for [imbalance]\n",
      "[2] entries found for [endorsing]\n",
      "[3] entries found for [congratulations]\n",
      "[2] entries found for [wed]\n",
      "[2] entries found for [swear]\n",
      "[2] entries found for [complicate]\n",
      "[2] entries found for [sharia]\n",
      "[2] entries found for [phenomenal]\n",
      "[2] entries found for [autism]\n",
      "[2] entries found for [militarily]\n",
      "[2] entries found for [dislike]\n",
      "[2] entries found for [disgrace]\n",
      "[2] entries found for [peanuts]\n",
      "[2] entries found for [anchors]\n",
      "[2] entries found for [teller]\n",
      "[2] entries found for [protester]\n",
      "[2] entries found for [bing]\n",
      "[2] entries found for [utter]\n",
      "[2] entries found for [huh]\n",
      "[2] entries found for [sharper]\n",
      "[2] entries found for [bernardino]\n",
      "[2] entries found for [phony]\n",
      "[2] entries found for [believer]\n",
      "[2] entries found for [ceilings]\n",
      "[2] entries found for [incompetence]\n",
      "[2] entries found for [incompetent]\n",
      "[2] entries found for [taxed]\n",
      "[2] entries found for [temperament]\n",
      "[2] entries found for [icahn]\n",
      "[2] entries found for [fools]\n",
      "[2] entries found for [crooked]\n",
      "[2] entries found for [interestingly]\n",
      "[2] entries found for [cages]\n",
      "[2] entries found for [amazingly]\n",
      "[2] entries found for [forgetting]\n",
      "[3] entries found for [disgusting]\n",
      "[2] entries found for [heavens]\n",
      "[2] entries found for [renegotiate]\n",
      "[2] entries found for [vets]\n",
      "[3] entries found for [conditioners]\n",
      "[2] entries found for [dishonest]\n",
      "[2] entries found for [cherish]\n",
      "[2] entries found for [chopping]\n",
      "[2] entries found for [omnibus]\n",
      "[2] entries found for [persians]\n",
      "[2] entries found for [horribly]\n",
      "[2] entries found for [stupidity]\n",
      "[2] entries found for [disgraceful]\n",
      "[2] entries found for [conditioner]\n",
      "[2] entries found for [bibi]\n",
      "[2] entries found for [straighten]\n",
      "[3] entries found for [trillions]\n",
      "[2] entries found for [pacs]\n",
      "[2] entries found for [trumps]\n",
      "[2] entries found for [simplification]\n",
      "[2] entries found for [scum]\n",
      "[2] entries found for [whoa]\n",
      "[2] entries found for [arpaio]\n",
      "[2] entries found for [maniac]\n",
      "[3] entries found for [dont]\n",
      "[2] entries found for [dreamers]\n",
      "[2] entries found for [deductibles]\n",
      "[2] entries found for [hacks]\n",
      "[3] entries found for [cant]\n",
      "[2] entries found for [womens]\n",
      "[2] entries found for [congratulation]\n",
      "[3] entries found for [wont]\n",
      "[2] entries found for [gitmo]\n",
      "[2] entries found for [hed]\n",
      "[3] entries found for [fellas]\n",
      "[3] entries found for [hes]\n",
      "[2] entries found for [dumbest]\n",
      "[2] entries found for [ive]\n",
      "[2] entries found for [heros]\n",
      "[3] entries found for [thats]\n",
      "[2] entries found for [favorability]\n",
      "[3] entries found for [didnt]\n",
      "[2] entries found for [bergdahl]\n",
      "[2] entries found for [oreos]\n",
      "[2] entries found for [obamacare]\n",
      "[2] entries found for [transpacific]\n",
      "[2] entries found for [theres]\n",
      "[2] entries found for [whos]\n",
      "[2] entries found for [hows]\n",
      "[3] entries found for [whats]\n",
      "[2] entries found for [anyways]\n",
      "[3] entries found for [doesnt]\n",
      "[3] entries found for [weve]\n",
      "[2] entries found for [arent]\n",
      "[2] entries found for [itd]\n",
      "[3] entries found for [shes]\n",
      "[2] entries found for [lyin]\n",
      "[2] entries found for [phenomenons]\n",
      "[2] entries found for [couldnt]\n",
      "[2] entries found for [havent]\n",
      "[3] entries found for [theyre]\n",
      "[3] entries found for [youre]\n",
      "[2] entries found for [isnt]\n",
      "[3] entries found for [wouldnt]\n",
      "[2] entries found for [priebus]\n",
      "[2] entries found for [heres]\n",
      "[2] entries found for [youd]\n",
      "[3] entries found for [youll]\n",
      "\n",
      "Loaded 7537 word vectors.\n",
      "2018.06.27 11:24\n",
      "Time to process: [30.420278787612915] seconds\n"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "\n",
    "start_time = get_time()\n",
    "\n",
    "# verify that mixed and lower case lists are the same length\n",
    "print( \"¿\", len( tokens_unique ), \"==\", len( tokens_unique_lowercase ), \"?\" )\n",
    "\n",
    "# load the whole embedding into memory\n",
    "embeddings_index = dict()\n",
    "\n",
    "# moved up!\n",
    "#embeddings_dimension = 300 #must be 50, 100, 200, 300\n",
    "glove = open( \"../glove/glove.6B.\" + str( embeddings_dimension ) + \"d.txt\" )\n",
    "\n",
    "# ASSUME: that 1st item in list is lowercase word, vectors are 2nd item\n",
    "for line in glove:\n",
    "    \n",
    "    values = line.split()\n",
    "    # 1st string is word...\n",
    "    word = values[ 0 ]\n",
    "    \n",
    "    # we're now searching w/in lowercase version to allow both mixed and lower case words (\"WorD\" and \"word\") to \n",
    "    # inherit same vectors\n",
    "    if word in tokens_unique_lowercase:\n",
    "        \n",
    "        # ...the rest are coefficients\n",
    "        coefs = np.asarray( values[ 1: ], dtype='float32' )\n",
    "        \n",
    "        # get indices for all occurences in lower_case list\n",
    "        indices = [ i for i, word_lower in enumerate( tokens_unique_lowercase ) if word_lower == word ]\n",
    "        \n",
    "        if len( indices ) > 1:\n",
    "            \n",
    "            print( \"[%d] entries found for [%s]\" % ( len( indices ), word ) )\n",
    "        \n",
    "            # iterate indices for this word\n",
    "            for i in indices:\n",
    "                word_temp = tokens_unique[ i ]\n",
    "                embeddings_index[ word_temp ] = coefs\n",
    "        else:\n",
    "            \n",
    "            embeddings_index[ word ] = coefs \n",
    "            \n",
    "    \n",
    "glove.close()\n",
    "print( '\\nLoaded %s word vectors.' % len( embeddings_index ) )\n",
    "#print( '\\nWords not found %d.' % ( len( tokenizer.word_index ) - len( embeddings_index ) ) )\n",
    "print_time( start_time, get_time(), interval=\"minute\" )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "index [2185] word [PRESIDENT] frequencey [7]\n",
      "index [3220] word [President] frequencey [80]\n",
      "index [3686] word [president] frequencey [117]\n"
     ]
    }
   ],
   "source": [
    "indices = [ i for i, word_lower in enumerate( tokens_unique_lowercase ) if word_lower == \"president\" ]\n",
    "\n",
    "for i in indices:\n",
    "    \n",
    "    print( \"index [%d] word [%s] frequencey [%d]\" % ( i, tokens_unique[ i ], word_counts[ tokens_unique[ i ] ] ) )\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len( words_uppercase ) 1157\n"
     ]
    }
   ],
   "source": [
    "# what words appear in upper case?\n",
    "words_uppercase = []\n",
    "\n",
    "for word in tokens_unique:\n",
    "    \n",
    "    if word.isupper():\n",
    "        words_uppercase.append( word )\n",
    "        \n",
    "print( \"len( words_uppercase )\", len( words_uppercase ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "word [TO], count[265]\n",
      "word [CONSEQUENCE], count[1]\n",
      "word [DOUBLE], count[1]\n",
      "word [ECONOMICALLY], count[1]\n",
      "word [FACE], count[2]\n",
      "word [LIFETIME], count[2]\n",
      "word [RAISING], count[1]\n",
      "word [EAST], count[3]\n",
      "word [HES], count[13]\n",
      "word [LEGALLY], count[2]\n",
      "word [AAA], count[4]\n",
      "word [CHARACTERISTICS], count[1]\n",
      "word [JOBS], count[7]\n",
      "word [WHEN], count[26]\n",
      "word [DEAD], count[1]\n",
      "word [TINY], count[2]\n",
      "word [VITAL], count[3]\n",
      "word [RULES], count[1]\n",
      "word [LACE], count[1]\n",
      "word [TEN], count[1]\n",
      "word [HEAR], count[6]\n",
      "word [YEARS], count[17]\n",
      "word [GSA], count[4]\n",
      "word [LAUGHTER], count[1]\n",
      "word [AGENTS], count[3]\n",
      "word [LOST], count[3]\n",
      "word [KILL], count[1]\n",
      "word [FOREVER], count[1]\n",
      "word [CHEERS], count[2]\n",
      "word [STATES], count[3]\n",
      "word [SMALL], count[3]\n",
      "word [PENNSYLVANIA], count[2]\n",
      "word [PPP], count[5]\n",
      "word [ISOLATIONIST], count[3]\n",
      "word [CRIME], count[1]\n",
      "word [DEBATE], count[1]\n",
      "word [AGAIN], count[6]\n",
      "word [IMBALANCE], count[2]\n",
      "word [IVE], count[4]\n",
      "word [COULDNT], count[1]\n",
      "word [THIS], count[43]\n",
      "word [RESPECT], count[2]\n",
      "word [COMBAT], count[1]\n",
      "word [CHOKING], count[1]\n",
      "word [YET], count[1]\n",
      "word [SITUATIONS], count[1]\n",
      "word [VISUALLY], count[1]\n",
      "word [ATTACKED], count[2]\n",
      "word [JETS], count[3]\n",
      "word [COMPARED], count[2]\n",
      "word [CHANCE], count[1]\n",
      "word [DEBATES], count[1]\n",
      "word [POSITION], count[1]\n",
      "word [BEFORE], count[13]\n",
      "word [PROBLEMS], count[5]\n",
      "word [SUCH], count[5]\n",
      "word [THAT], count[116]\n",
      "word [SHE], count[11]\n",
      "word [SENATE], count[1]\n",
      "word [COULD], count[8]\n",
      "word [RESPECTED], count[1]\n",
      "word [ANYWHERE], count[1]\n",
      "word [BEGINNING], count[3]\n",
      "word [BEING], count[2]\n",
      "word [WORRIED], count[1]\n",
      "word [NEGOTIATION], count[2]\n",
      "word [EXPRESSION], count[1]\n",
      "word [WISH], count[2]\n",
      "word [COMMUNITIES], count[1]\n",
      "word [FRANKLY], count[2]\n",
      "word [EMMY], count[1]\n",
      "word [VOTING], count[3]\n",
      "word [CONGRATULATIONS], count[1]\n",
      "word [WEVE], count[11]\n",
      "word [LEARNED], count[4]\n",
      "word [LOVE], count[14]\n",
      "word [FRONT], count[1]\n",
      "word [DAMN], count[1]\n",
      "word [UPSET], count[1]\n",
      "word [MOVE], count[5]\n",
      "word [LIVE], count[1]\n",
      "word [B], count[1]\n",
      "word [DOWN], count[3]\n",
      "word [BANK], count[3]\n",
      "word [CITIES], count[3]\n",
      "word [NAMES], count[1]\n",
      "word [FLORIDA], count[1]\n",
      "word [JOKE], count[1]\n",
      "word [D], count[5]\n",
      "word [ALMOST], count[1]\n",
      "word [STAGE], count[2]\n",
      "word [ASKED], count[2]\n",
      "word [NAFTA], count[20]\n",
      "word [PARTY], count[2]\n",
      "word [PLEDGES], count[1]\n",
      "word [NEGOTIATED], count[1]\n",
      "word [BELIEVE], count[5]\n",
      "word [RECOVERING], count[1]\n",
      "word [TOUGH], count[12]\n",
      "word [NOW], count[24]\n",
      "word [IPO], count[1]\n",
      "word [NEGOTIATIONS], count[1]\n",
      "word [OUR], count[74]\n",
      "word [DESERTER], count[1]\n",
      "word [F], count[5]\n",
      "word [THEYLL], count[5]\n",
      "word [STRONGER], count[1]\n",
      "word [EARTH], count[3]\n",
      "word [FROM], count[14]\n",
      "word [PERSIANS], count[1]\n",
      "word [RASMUSSEN], count[1]\n",
      "word [OKAY], count[6]\n",
      "word [DONATIONS], count[1]\n",
      "word [NEXT], count[3]\n",
      "word [BUY], count[1]\n",
      "word [FRIENDS], count[4]\n",
      "word [LOUISIANA], count[1]\n",
      "word [NO], count[13]\n",
      "word [DRIVERS], count[1]\n",
      "word [REGULATION], count[1]\n",
      "word [YOUR], count[17]\n",
      "word [PERSON], count[4]\n",
      "word [STOPPING], count[1]\n",
      "word [REDUCED], count[1]\n",
      "word [MIT], count[1]\n",
      "word [GUYS], count[1]\n",
      "word [REMEMBER], count[17]\n",
      "word [GRAVEYARDS], count[1]\n",
      "word [JEOPARDY], count[2]\n",
      "word [PRESIDENTIAL], count[1]\n",
      "word [INCREDIBLE], count[2]\n",
      "word [GIVEN], count[1]\n",
      "word [OFFICIALS], count[1]\n",
      "word [CHILDREN], count[2]\n",
      "word [DISADVANTAGE], count[1]\n",
      "word [CONSIDER], count[1]\n",
      "word [CEILINGS], count[1]\n",
      "word [EVERYONE], count[2]\n",
      "word [NATION], count[2]\n",
      "word [FIND], count[1]\n",
      "word [LAWS], count[3]\n",
      "word [LET], count[9]\n",
      "word [ROUTINE], count[1]\n",
      "word [LETS], count[1]\n",
      "word [ZONES], count[3]\n",
      "word [MONEY], count[15]\n",
      "word [TALKS], count[2]\n",
      "word [CHOPPED], count[1]\n",
      "word [IQ], count[1]\n",
      "word [OPEN], count[3]\n",
      "word [YESTERDAY], count[1]\n",
      "word [IT], count[130]\n",
      "word [TRAINING], count[1]\n",
      "word [NOMINEE], count[1]\n",
      "word [JAPAN], count[3]\n",
      "word [ANYBODY], count[5]\n",
      "word [HILLARY], count[10]\n",
      "word [DIDNT], count[12]\n",
      "word [PRETZEL], count[1]\n",
      "word [ACCESS], count[2]\n",
      "word [WE], count[185]\n",
      "word [AMAZING], count[5]\n",
      "word [EXTENT], count[1]\n",
      "word [PUT], count[5]\n",
      "word [MINNESOTA], count[1]\n",
      "word [INCENTIVIZE], count[1]\n",
      "word [DEBATING], count[1]\n",
      "word [AREAS], count[1]\n",
      "word [VERY], count[15]\n",
      "word [DOZENS], count[1]\n",
      "word [SIX], count[1]\n",
      "word [CONTINUE], count[1]\n",
      "word [CAN], count[14]\n",
      "word [ELECTED], count[2]\n",
      "word [YOUVE], count[5]\n",
      "word [PRISON], count[1]\n",
      "word [ENTIRE], count[3]\n",
      "word [BROADCAST], count[1]\n",
      "word [NETWORKS], count[1]\n",
      "word [CALL], count[6]\n",
      "word [TEAM], count[1]\n",
      "word [VETERAN], count[1]\n",
      "word [RECOGNIZES], count[1]\n",
      "word [ASK], count[1]\n",
      "word [BUILDINGS], count[1]\n",
      "word [I], count[4873]\n",
      "word [ILL], count[8]\n",
      "word [RICH], count[2]\n",
      "word [SERIOUS], count[2]\n",
      "word [PROGRESS], count[1]\n",
      "word [NUTS], count[1]\n",
      "word [BUILD], count[10]\n",
      "word [NATIONS], count[1]\n",
      "word [TWO], count[7]\n",
      "word [CLOSING], count[3]\n",
      "word [TEMPERAMENT], count[2]\n",
      "word [GPS], count[1]\n",
      "word [SOON], count[4]\n",
      "word [GOTTEN], count[2]\n",
      "word [POTENTIALLY], count[1]\n",
      "word [FEEL], count[1]\n",
      "word [MIDDLE], count[4]\n",
      "word [FAST], count[1]\n",
      "word [FOX], count[1]\n",
      "word [ANY], count[3]\n",
      "word [EDUCATED], count[1]\n",
      "word [OUTSIDE], count[1]\n",
      "word [ENEMIES], count[2]\n",
      "word [ALL], count[38]\n",
      "word [VETERANS], count[2]\n",
      "word [HER], count[6]\n",
      "word [3D], count[1]\n",
      "word [GENE], count[1]\n",
      "word [CAME], count[8]\n",
      "word [RATING], count[1]\n",
      "word [PROTEST], count[1]\n",
      "word [GETS], count[3]\n",
      "word [FILL], count[1]\n",
      "word [KNEW], count[6]\n",
      "word [ANOTHER], count[1]\n",
      "word [EDITION], count[1]\n",
      "word [KILLERS], count[1]\n",
      "word [TRILLION], count[8]\n",
      "word [TAKE], count[34]\n",
      "word [SINCE], count[3]\n",
      "word [REPRESENT], count[1]\n",
      "word [PEANUTS], count[1]\n",
      "word [LAST], count[1]\n",
      "word [DID], count[7]\n",
      "word [PROBLEM], count[6]\n",
      "word [PP], count[2]\n",
      "word [OVER], count[20]\n",
      "word [HATED], count[2]\n",
      "word [PAC], count[17]\n",
      "word [ALWAYS], count[6]\n",
      "word [WHOM], count[1]\n",
      "word [WEAPONS], count[3]\n",
      "word [THING], count[9]\n",
      "word [IRAN], count[2]\n",
      "word [QUALITY], count[2]\n",
      "word [FATHER], count[6]\n",
      "word [MAEPGESSENGER], count[1]\n",
      "word [HAVENT], count[1]\n",
      "word [AVERAGE], count[1]\n",
      "word [FAIRLY], count[1]\n",
      "word [AGAINST], count[3]\n",
      "word [MOVING], count[2]\n",
      "word [SANDERS], count[4]\n",
      "word [PERHAPS], count[1]\n",
      "word [LOADED], count[1]\n",
      "word [OBAMACARE], count[1]\n",
      "word [LIES], count[4]\n",
      "word [PAYING], count[1]\n",
      "word [FORM], count[1]\n",
      "word [COVER], count[6]\n",
      "word [STANDING], count[1]\n",
      "word [STRIP], count[1]\n",
      "word [PLACE], count[8]\n",
      "word [WAS], count[43]\n",
      "word [INDIANAPOLIS], count[1]\n",
      "word [SPEND], count[1]\n",
      "word [SPOKE], count[1]\n",
      "word [BERGDAHL], count[3]\n",
      "word [MILLION], count[2]\n",
      "word [SEE], count[15]\n",
      "word [WORLD], count[9]\n",
      "word [DALLAS], count[1]\n",
      "word [PRESS], count[4]\n",
      "word [STUPID], count[3]\n",
      "word [AND], count[264]\n",
      "word [MAN], count[3]\n",
      "word [SAID], count[45]\n",
      "word [DEBUT], count[1]\n",
      "word [CONSERVATIVE], count[1]\n",
      "word [WRITE], count[1]\n",
      "word [CARL], count[1]\n",
      "word [DUMBEST], count[1]\n",
      "word [GREATEST], count[10]\n",
      "word [ORDER], count[2]\n",
      "word [TOGETHER], count[1]\n",
      "word [THATS], count[8]\n",
      "word [INTO], count[6]\n",
      "word [HIGH], count[3]\n",
      "word [GOING], count[127]\n",
      "word [EQUIPMENT], count[1]\n",
      "word [ASPECTS], count[1]\n",
      "word [AGO], count[8]\n",
      "word [L], count[1]\n",
      "word [OTHER], count[9]\n",
      "word [SHOUTING], count[2]\n",
      "word [CHEERING], count[3]\n",
      "word [NINE], count[1]\n",
      "word [NEED], count[8]\n",
      "word [REASON], count[2]\n",
      "word [HE], count[35]\n",
      "word [WALL], count[25]\n",
      "word [MOMENT], count[1]\n",
      "word [IF], count[18]\n",
      "word [WHILE], count[2]\n",
      "word [OH], count[6]\n",
      "word [IS], count[95]\n",
      "word [DEFICIT], count[1]\n",
      "word [INDIRECTLY], count[1]\n",
      "word [PAPERS], count[1]\n",
      "word [ADD], count[1]\n",
      "word [SWEAR], count[1]\n",
      "word [CAMERAS], count[9]\n",
      "word [DIGGER], count[1]\n",
      "word [MEMBERS], count[1]\n",
      "word [DOESNT], count[6]\n",
      "word [NBC], count[17]\n",
      "word [WAYS], count[3]\n",
      "word [SUDDEN], count[1]\n",
      "word [MISSING], count[1]\n",
      "word [KNOWN], count[1]\n",
      "word [PAY], count[8]\n",
      "word [REAL], count[2]\n",
      "word [WON], count[1]\n",
      "word [LIVES], count[1]\n",
      "word [WATCH], count[3]\n",
      "word [UTTER], count[1]\n",
      "word [VICE], count[2]\n",
      "word [HEART], count[1]\n",
      "word [SAVED], count[2]\n",
      "word [CHOICE], count[2]\n",
      "word [ON], count[34]\n",
      "word [PLIBLG], count[1]\n",
      "word [DAYS], count[7]\n",
      "word [TREAT], count[5]\n",
      "word [PRESIDENT], count[7]\n",
      "word [LOOKING], count[4]\n",
      "word [COME], count[17]\n",
      "word [GUILTY], count[1]\n",
      "word [COMMITTEE], count[1]\n",
      "word [DO], count[28]\n",
      "word [BEEF], count[1]\n",
      "word [PAPERWORK], count[1]\n",
      "word [MATERIAL], count[1]\n",
      "word [DAY], count[3]\n",
      "word [DEALING], count[1]\n",
      "word [AN], count[9]\n",
      "word [NASCAR], count[6]\n",
      "word [COMPANIES], count[3]\n",
      "word [TRADER], count[3]\n",
      "word [RACE], count[1]\n",
      "word [WEATHER], count[1]\n",
      "word [MUCH], count[13]\n",
      "word [REGULATIONS], count[2]\n",
      "word [MAGNIFIED], count[1]\n",
      "word [CONSEQUENCES], count[1]\n",
      "word [ABC], count[2]\n",
      "word [THEIR], count[16]\n",
      "word [DELIVERED], count[1]\n",
      "word [RUN], count[2]\n",
      "word [AMATEURS], count[1]\n",
      "word [LEAVES], count[1]\n",
      "word [NOT], count[36]\n",
      "word [PHENOMENON], count[2]\n",
      "word [BE], count[54]\n",
      "word [LAX], count[2]\n",
      "word [OMNIBUS], count[1]\n",
      "word [PRETTY], count[1]\n",
      "word [YOUNG], count[3]\n",
      "word [LG], count[3]\n",
      "word [VETS], count[2]\n",
      "word [BECAME], count[1]\n",
      "word [WHOLE], count[6]\n",
      "word [ZERO], count[1]\n",
      "word [OHIO], count[1]\n",
      "word [ISRAEL], count[1]\n",
      "word [SHERIFF], count[2]\n",
      "word [UNTIL], count[1]\n",
      "word [HOSTILITY], count[1]\n",
      "word [MY], count[14]\n",
      "word [LOCAL], count[2]\n",
      "word [RIGHT], count[24]\n",
      "word [TRADE], count[22]\n",
      "word [ANCHORS], count[1]\n",
      "word [KILLINGS], count[1]\n",
      "word [RICHARD], count[2]\n",
      "word [III], count[2]\n",
      "word [IDEALS], count[1]\n",
      "word [SAY], count[41]\n",
      "word [WEE], count[1]\n",
      "word [FIGURE], count[2]\n",
      "word [THINKING], count[2]\n",
      "word [RINAS], count[1]\n",
      "word [CHRISTIANS], count[1]\n",
      "word [OF], count[141]\n",
      "word [MILITARY], count[9]\n",
      "word [HAPPENING], count[3]\n",
      "word [LIFE], count[2]\n",
      "word [BERNARDINO], count[1]\n",
      "word [DISGRACEFUL], count[1]\n",
      "word [THIRD], count[1]\n",
      "word [FOLKS], count[19]\n",
      "word [ITS], count[13]\n",
      "word [WAR], count[1]\n",
      "word [ARE], count[67]\n",
      "word [PROUD], count[1]\n",
      "word [PORTIONS], count[1]\n",
      "word [GETTING], count[3]\n",
      "word [IDEA], count[3]\n",
      "word [OREILLY], count[2]\n",
      "word [EXPLODING], count[1]\n",
      "word [ENDORSING], count[2]\n",
      "word [CLEARLY], count[1]\n",
      "word [NASTY], count[1]\n",
      "word [NICE], count[5]\n",
      "word [THINGS], count[16]\n",
      "word [GOOD], count[8]\n",
      "word [SAYING], count[3]\n",
      "word [SUN], count[1]\n",
      "word [MEXICO], count[4]\n",
      "word [THESE], count[17]\n",
      "word [WAY], count[19]\n",
      "word [THANK], count[10]\n",
      "word [GOT], count[10]\n",
      "word [UNDER], count[1]\n",
      "word [INTEREST], count[2]\n",
      "word [CHINA], count[1]\n",
      "word [QUICKLY], count[1]\n",
      "word [HAVE], count[141]\n",
      "word [POINT], count[1]\n",
      "word [FATHERS], count[1]\n",
      "word [MAGAZINE], count[4]\n",
      "word [RATHER], count[3]\n",
      "word [RECORDED], count[2]\n",
      "word [WITH], count[42]\n",
      "word [STILL], count[2]\n",
      "word [SELL], count[2]\n",
      "word [WORKING], count[3]\n",
      "word [LUCK], count[1]\n",
      "word [KEEP], count[11]\n",
      "word [RNC], count[5]\n",
      "word [TELEVISION], count[1]\n",
      "word [PETRAEUS], count[1]\n",
      "word [POLITICS], count[1]\n",
      "word [SHES], count[1]\n",
      "word [WIFE], count[1]\n",
      "word [WORRY], count[2]\n",
      "word [ENGLAND], count[1]\n",
      "word [CROWD], count[3]\n",
      "word [BROKE], count[2]\n",
      "word [IN], count[84]\n",
      "word [WHACKED], count[1]\n",
      "word [FELT], count[1]\n",
      "word [ROW], count[2]\n",
      "word [GROUP], count[2]\n",
      "word [AIRLINERS], count[1]\n",
      "word [KING], count[1]\n",
      "word [ANSWER], count[1]\n",
      "word [SILENCE], count[2]\n",
      "word [HARD], count[3]\n",
      "word [SYSTEM], count[11]\n",
      "word [PACS], count[1]\n",
      "word [FAVOR], count[3]\n",
      "word [PRIDE], count[1]\n",
      "word [WATER], count[1]\n",
      "word [PRIEBUS], count[1]\n",
      "word [STAND], count[4]\n",
      "word [POTATOES], count[1]\n",
      "word [SHAPE], count[1]\n",
      "word [LOSING], count[4]\n",
      "word [COLLEGE], count[1]\n",
      "word [SURPRISING], count[1]\n",
      "word [PEOPLE], count[74]\n",
      "word [GUARANTEE], count[1]\n",
      "word [INNER], count[1]\n",
      "word [NEVER], count[28]\n",
      "word [REPUBLICAN], count[4]\n",
      "word [BLOWUP], count[1]\n",
      "word [FIRE], count[1]\n",
      "word [FIRST], count[2]\n",
      "word [T], count[1]\n",
      "word [HONOR], count[3]\n",
      "word [CHANGE], count[1]\n",
      "word [YOURE], count[21]\n",
      "word [REBUILT], count[1]\n",
      "word [PLANE], count[1]\n",
      "word [OIL], count[7]\n",
      "word [SOMETHING], count[7]\n",
      "word [ENOUGH], count[5]\n",
      "word [PROTESTERS], count[2]\n",
      "word [SPEECH], count[11]\n",
      "word [NAMED], count[1]\n",
      "word [STAY], count[1]\n",
      "word [SEASON], count[1]\n",
      "word [MSNBC], count[1]\n",
      "word [WARNED], count[1]\n",
      "word [THINK], count[15]\n",
      "word [GITMO], count[4]\n",
      "word [NATIONAL], count[1]\n",
      "word [THOUSANDS], count[3]\n",
      "word [WAITING], count[2]\n",
      "word [WOW], count[2]\n",
      "word [STRAIGHTEN], count[1]\n",
      "word [WORDS], count[2]\n",
      "word [VENGEANCE], count[1]\n",
      "word [WHERE], count[10]\n",
      "word [NOBLE], count[1]\n",
      "word [WASTE], count[1]\n",
      "word [WANTED], count[2]\n",
      "word [WENT], count[4]\n",
      "word [MEANS], count[2]\n",
      "word [MONTH], count[3]\n",
      "word [SECRETARY], count[1]\n",
      "word [POURING], count[3]\n",
      "word [ENDORSEMENT], count[2]\n",
      "word [PRIMARY], count[1]\n",
      "word [BECOMES], count[1]\n",
      "word [LITANY], count[1]\n",
      "word [OR], count[12]\n",
      "word [OFF], count[7]\n",
      "word [LIKES], count[1]\n",
      "word [COMMON], count[3]\n",
      "word [DURING], count[2]\n",
      "word [OPINION], count[1]\n",
      "word [SUBJECT], count[3]\n",
      "word [AMT], count[1]\n",
      "word [FREE], count[5]\n",
      "word [WHEAT], count[1]\n",
      "word [SHOCKED], count[1]\n",
      "word [PROTESTER], count[1]\n",
      "word [PERSONALLY], count[1]\n",
      "word [C], count[3]\n",
      "word [BEAT], count[2]\n",
      "word [FUND], count[1]\n",
      "word [SWISS], count[1]\n",
      "word [STARTED], count[3]\n",
      "word [BECOMING], count[1]\n",
      "word [TOOL], count[2]\n",
      "word [WALK], count[1]\n",
      "word [WHETHER], count[1]\n",
      "word [MAYBE], count[5]\n",
      "word [SEEN], count[10]\n",
      "word [CARE], count[12]\n",
      "word [THEYVE], count[7]\n",
      "word [SUPPORT], count[4]\n",
      "word [YEAR], count[2]\n",
      "word [PRIMARIES], count[1]\n",
      "word [WATCHING], count[1]\n",
      "word [KIND], count[1]\n",
      "word [NASHLTIONALLY], count[1]\n",
      "word [HORRIBLE], count[2]\n",
      "word [TRULY], count[1]\n",
      "word [CONDITIONING], count[1]\n",
      "word [BANG], count[2]\n",
      "word [X], count[1]\n",
      "word [MAKING], count[3]\n",
      "word [TZ], count[1]\n",
      "word [UN], count[1]\n",
      "word [NICELY], count[1]\n",
      "word [SECURITY], count[1]\n",
      "word [DEPLETED], count[2]\n",
      "word [VICENTE], count[1]\n",
      "word [FLYING], count[1]\n",
      "word [OTHERS], count[2]\n",
      "word [BORE], count[1]\n",
      "word [CROWDS], count[2]\n",
      "word [TONE], count[4]\n",
      "word [WIN], count[18]\n",
      "word [UP], count[23]\n",
      "word [DESERTS], count[1]\n",
      "word [MARCHES], count[2]\n",
      "word [SCHOOL], count[1]\n",
      "word [CARS], count[2]\n",
      "word [SANCTIONS], count[2]\n",
      "word [CONSTRUCTION], count[2]\n",
      "word [SO], count[78]\n",
      "word [BELIEVER], count[1]\n",
      "word [THROUGH], count[5]\n",
      "word [WHAT], count[60]\n",
      "word [SUPER], count[1]\n",
      "word [LOU], count[2]\n",
      "word [POSITIVE], count[1]\n",
      "word [BECAUSE], count[28]\n",
      "word [EMPLOYEES], count[2]\n",
      "word [ISIS], count[86]\n",
      "word [BADLY], count[6]\n",
      "word [EXPLAIN], count[1]\n",
      "word [SAVE], count[2]\n",
      "word [KEPT], count[2]\n",
      "word [WHO], count[13]\n",
      "word [SINGLE], count[5]\n",
      "word [RGA], count[1]\n",
      "word [K], count[3]\n",
      "word [KILLED], count[4]\n",
      "word [HEADLINES], count[1]\n",
      "word [SOME], count[13]\n",
      "word [THE], count[287]\n",
      "word [TIME], count[13]\n",
      "word [WEAKNESS], count[1]\n",
      "word [UPBEAT], count[1]\n",
      "word [THOSE], count[7]\n",
      "word [BILL], count[4]\n",
      "word [OFTEN], count[1]\n",
      "word [BY], count[15]\n",
      "word [NEW], count[2]\n",
      "word [IRS], count[4]\n",
      "word [MATTER], count[2]\n",
      "word [JUST], count[19]\n",
      "word [KNOWING], count[1]\n",
      "word [AM], count[3]\n",
      "word [GREATER], count[2]\n",
      "word [SIEGE], count[1]\n",
      "word [MENTION], count[1]\n",
      "word [AIR], count[5]\n",
      "word [MESSENGER], count[3]\n",
      "word [WAKE], count[1]\n",
      "word [EVERYWHERE], count[1]\n",
      "word [LIST], count[1]\n",
      "word [SCREAMING], count[1]\n",
      "word [EVER], count[12]\n",
      "word [REMAIN], count[1]\n",
      "word [BOOING], count[1]\n",
      "word [HAPPY], count[3]\n",
      "word [CHICAGO], count[2]\n",
      "word [MOTIVES], count[1]\n",
      "word [VOTE], count[1]\n",
      "word [FAR], count[5]\n",
      "word [BATHROOMS], count[3]\n",
      "word [SIMPLE], count[1]\n",
      "word [FAMILIES], count[2]\n",
      "word [HOUSE], count[1]\n",
      "word [IMPORTANT], count[12]\n",
      "word [V], count[3]\n",
      "word [ALONG], count[3]\n",
      "word [IM], count[21]\n",
      "word [STORY], count[7]\n",
      "word [DISGUSTING], count[2]\n",
      "word [MANKIND], count[1]\n",
      "word [WHY], count[3]\n",
      "word [CANT], count[14]\n",
      "word [ALLOW], count[2]\n",
      "word [CHERISH], count[1]\n",
      "word [TELL], count[19]\n",
      "word [BUYING], count[3]\n",
      "word [WIDE], count[1]\n",
      "word [PROBABLY], count[4]\n",
      "word [CLOSE], count[1]\n",
      "word [CHOPPING], count[2]\n",
      "word [CONDITIONERS], count[4]\n",
      "word [TORE], count[2]\n",
      "word [SORT], count[1]\n",
      "word [ONE], count[23]\n",
      "word [GIVE], count[3]\n",
      "word [DROWNING], count[1]\n",
      "word [CLINTON], count[10]\n",
      "word [MEETING], count[3]\n",
      "word [JOB], count[8]\n",
      "word [PASSPORTS], count[1]\n",
      "word [BUSINESSES], count[1]\n",
      "word [HOSTAGES], count[4]\n",
      "word [WONT], count[4]\n",
      "word [TODAY], count[3]\n",
      "word [BEEN], count[13]\n",
      "word [GUY], count[8]\n",
      "word [MIKE], count[6]\n",
      "word [GDP], count[4]\n",
      "word [ANGER], count[1]\n",
      "word [FIX], count[1]\n",
      "word [A], count[245]\n",
      "word [TALENTED], count[1]\n",
      "word [THEY], count[132]\n",
      "word [LA], count[4]\n",
      "word [TOOK], count[3]\n",
      "word [SHED], count[1]\n",
      "word [TSA], count[1]\n",
      "word [SPENDING], count[1]\n",
      "word [KICK], count[1]\n",
      "word [CANNOT], count[1]\n",
      "word [PROFESSIONALS], count[1]\n",
      "word [RAISED], count[1]\n",
      "word [AP], count[1]\n",
      "word [CORE], count[1]\n",
      "word [ELECTRIC], count[2]\n",
      "word [STUFF], count[3]\n",
      "word [SAW], count[4]\n",
      "word [HATRED], count[2]\n",
      "word [WORTH], count[1]\n",
      "word [AMERICA], count[2]\n",
      "word [MANY], count[12]\n",
      "word [TOO], count[6]\n",
      "word [SHIPS], count[2]\n",
      "word [CREATED], count[4]\n",
      "word [HOSTAGE], count[1]\n",
      "word [THEM], count[33]\n",
      "word [QUESTION], count[1]\n",
      "word [POWERFUL], count[1]\n",
      "word [HOLTS], count[1]\n",
      "word [CANDIDATE], count[3]\n",
      "word [SOCIETY], count[2]\n",
      "word [GOP], count[2]\n",
      "word [HOURS], count[1]\n",
      "word [DISLIKE], count[1]\n",
      "word [E], count[6]\n",
      "word [OPEC], count[1]\n",
      "word [PHENOMENONS], count[1]\n",
      "word [GAVE], count[2]\n",
      "word [DONE], count[13]\n",
      "word [MILITARILY], count[1]\n",
      "word [NOTHING], count[2]\n",
      "word [TROOPS], count[2]\n",
      "word [US], count[34]\n",
      "word [OWE], count[3]\n",
      "word [ONLY], count[7]\n",
      "word [MANUFACTURING], count[2]\n",
      "word [ADVANTAGE], count[4]\n",
      "word [FORGET], count[3]\n",
      "word [SPEAK], count[1]\n",
      "word [PLACES], count[4]\n",
      "word [BILLIONS], count[3]\n",
      "word [SECOND], count[4]\n",
      "word [MESSAGE], count[2]\n",
      "word [NASTIEST], count[2]\n",
      "word [SHOW], count[1]\n",
      "word [POLICY], count[2]\n",
      "word [BRIAN], count[1]\n",
      "word [TOP], count[2]\n",
      "word [PRAY], count[4]\n",
      "word [HONORED], count[1]\n",
      "word [DIFFERENT], count[6]\n",
      "word [MEDIA], count[1]\n",
      "word [TAKING], count[1]\n",
      "word [FOR], count[54]\n",
      "word [PBS], count[1]\n",
      "word [BORDER], count[9]\n",
      "word [ONCE], count[4]\n",
      "word [POLITICAL], count[5]\n",
      "word [FLOWN], count[1]\n",
      "word [MR], count[10]\n",
      "word [ANKLES], count[1]\n",
      "word [ENTER], count[1]\n",
      "word [EASY], count[7]\n",
      "word [USES], count[2]\n",
      "word [ONES], count[2]\n",
      "word [BUT], count[69]\n",
      "word [MENTIONED], count[2]\n",
      "word [HUNDREDS], count[1]\n",
      "word [SON], count[2]\n",
      "word [MADE], count[5]\n",
      "word [BOOKS], count[2]\n",
      "word [CNBC], count[3]\n",
      "word [STOP], count[5]\n",
      "word [LEGS], count[1]\n",
      "word [POOR], count[1]\n",
      "word [RATE], count[1]\n",
      "word [COLONEL], count[2]\n",
      "word [PART], count[1]\n",
      "word [OLD], count[5]\n",
      "word [PIECE], count[1]\n",
      "word [PIGGY], count[3]\n",
      "word [CONSULTANTS], count[1]\n",
      "word [WITHOUT], count[2]\n",
      "word [STRANGE], count[1]\n",
      "word [TPP], count[14]\n",
      "word [CBS], count[5]\n",
      "word [EXCEPT], count[2]\n",
      "word [ANYTHING], count[5]\n",
      "word [SHOULD], count[15]\n",
      "word [LITTLE], count[4]\n",
      "word [MILLIONS], count[3]\n",
      "word [TAX], count[2]\n",
      "word [USED], count[10]\n",
      "word [COACHES], count[5]\n",
      "word [LONG], count[5]\n",
      "word [EVERY], count[6]\n",
      "word [PLEASE], count[5]\n",
      "word [TOWARD], count[2]\n",
      "word [FOOLS], count[2]\n",
      "word [THINKS], count[1]\n",
      "word [NATO], count[18]\n",
      "word [JV], count[3]\n",
      "word [CERTAIN], count[1]\n",
      "word [MUSIC], count[1]\n",
      "word [SYRIA], count[1]\n",
      "word [GATE], count[2]\n",
      "word [WONDERFUL], count[2]\n",
      "word [FEC], count[1]\n",
      "word [BUILT], count[2]\n",
      "word [DELEGATION], count[1]\n",
      "word [READ], count[5]\n",
      "word [SPECIAL], count[3]\n",
      "word [POP], count[1]\n",
      "word [MOURNS], count[1]\n",
      "word [SEEMS], count[1]\n",
      "word [AMENDMENT], count[3]\n",
      "word [EXACTLY], count[1]\n",
      "word [OUTRAGEOUS], count[1]\n",
      "word [WORK], count[1]\n",
      "word [HAD], count[23]\n",
      "word [BATTLEFIELD], count[1]\n",
      "word [YES], count[1]\n",
      "word [LOS], count[1]\n",
      "word [MORE], count[11]\n",
      "word [BOMB], count[4]\n",
      "word [SUBJECTS], count[1]\n",
      "word [TRUMP], count[34]\n",
      "word [UNREST], count[1]\n",
      "word [FLOATING], count[1]\n",
      "word [PERFECT], count[4]\n",
      "word [WROTE], count[2]\n",
      "word [LARGE], count[2]\n",
      "word [OBAMA], count[6]\n",
      "word [M], count[1]\n",
      "word [LEAVE], count[6]\n",
      "word [HUMAN], count[3]\n",
      "word [PUTTING], count[1]\n",
      "word [CHANGING], count[1]\n",
      "word [INCOMPETENT], count[1]\n",
      "word [PILOTS], count[1]\n",
      "word [AMERICAN], count[2]\n",
      "word [LUMPS], count[6]\n",
      "word [PLENTY], count[1]\n",
      "word [GULF], count[1]\n",
      "word [WITHIN], count[1]\n",
      "word [DESERTED], count[1]\n",
      "word [REPLACE], count[1]\n",
      "word [DELEGATES], count[1]\n",
      "word [RID], count[4]\n",
      "word [SIMILAR], count[1]\n",
      "word [J], count[2]\n",
      "word [LAW], count[5]\n",
      "word [HACKS], count[2]\n",
      "word [COMPASSION], count[3]\n",
      "word [THAN], count[13]\n",
      "word [WEEKS], count[3]\n",
      "word [EVERYBODY], count[12]\n",
      "word [END], count[7]\n",
      "word [STRONG], count[5]\n",
      "word [CHAIR], count[2]\n",
      "word [PRO], count[1]\n",
      "word [FIGHT], count[1]\n",
      "word [COMMUNITY], count[1]\n",
      "word [ANGELES], count[1]\n",
      "word [WALLS], count[1]\n",
      "word [CAGES], count[1]\n",
      "word [DONALD], count[7]\n",
      "word [KERRY], count[3]\n",
      "word [ELSE], count[4]\n",
      "word [GENERAL], count[3]\n",
      "word [WORD], count[3]\n",
      "word [THEN], count[5]\n",
      "word [WHATS], count[2]\n",
      "word [CIVIL], count[1]\n",
      "word [FRANCE], count[1]\n",
      "word [LOANS], count[2]\n",
      "word [AT], count[44]\n",
      "word [INDIANA], count[7]\n",
      "word [MIGRATION], count[1]\n",
      "word [FELL], count[1]\n",
      "word [VICIOUS], count[1]\n",
      "word [WOUNDED], count[1]\n",
      "word [ABLE], count[1]\n",
      "word [BEINGS], count[3]\n",
      "word [ICAHN], count[1]\n",
      "word [BEATING], count[1]\n",
      "word [TONIGHT], count[2]\n",
      "word [DRUGS], count[1]\n",
      "word [HIS], count[7]\n",
      "word [TAXED], count[1]\n",
      "word [BICYCLE], count[2]\n",
      "word [FAIRNESS], count[1]\n",
      "word [COURSE], count[1]\n",
      "word [HATE], count[2]\n",
      "word [DISHONEST], count[9]\n",
      "word [CNN], count[29]\n",
      "word [OUT], count[34]\n",
      "word [SENTENCE], count[1]\n",
      "word [TIMES], count[3]\n",
      "word [CHEESE], count[1]\n",
      "word [HERE], count[16]\n",
      "word [MECHANICALLY], count[1]\n",
      "word [INCOMPETENCE], count[1]\n",
      "word [MASSIVE], count[3]\n",
      "word [LEVEL], count[1]\n",
      "word [USA], count[2]\n",
      "word [LEADING], count[5]\n",
      "word [PARTS], count[2]\n",
      "word [YOURSELF], count[1]\n",
      "word [ORLANDO], count[1]\n",
      "word [STUPIDITY], count[1]\n",
      "word [BADEN], count[1]\n",
      "word [MAIL], count[2]\n",
      "word [WELL], count[13]\n",
      "word [MAJOR], count[1]\n",
      "word [DONT], count[54]\n",
      "word [ARIZONA], count[1]\n",
      "word [PROTESTS], count[1]\n",
      "word [AROUND], count[5]\n",
      "word [VA], count[7]\n",
      "word [NIGHT], count[2]\n",
      "word [KNIGHT], count[4]\n",
      "word [DOING], count[6]\n",
      "word [FAIR], count[1]\n",
      "word [MEDIEVAL], count[1]\n",
      "word [CARDS], count[1]\n",
      "word [LEAST], count[2]\n",
      "word [MEAN], count[12]\n",
      "word [HIGHEST], count[2]\n",
      "word [SIDE], count[2]\n",
      "word [IRAQ], count[3]\n",
      "word [GO], count[22]\n",
      "word [YOU], count[132]\n",
      "word [WHOA], count[1]\n",
      "word [Q], count[1]\n",
      "word [FLAT], count[1]\n",
      "word [GOVERNOR], count[4]\n",
      "word [COMPLICATE], count[2]\n",
      "word [FACT], count[3]\n",
      "word [SERGEANT], count[2]\n",
      "word [PROMISE], count[3]\n",
      "word [MUST], count[1]\n",
      "word [ACHIEVEMENT], count[2]\n",
      "word [TRAINS], count[1]\n",
      "word [PLEDGE], count[10]\n",
      "word [FIGHTING], count[3]\n",
      "word [CAMERAMANS], count[1]\n",
      "word [SAYS], count[3]\n",
      "word [TOMORROW], count[1]\n",
      "word [NOBODY], count[5]\n",
      "word [DOLLARS], count[4]\n",
      "word [BIGGEST], count[2]\n",
      "word [THETORY], count[1]\n",
      "word [FABULOUS], count[1]\n",
      "word [BEST], count[2]\n",
      "word [BILLION], count[5]\n",
      "word [BIT], count[2]\n",
      "word [ABSOLUTELY], count[1]\n",
      "word [EVEN], count[11]\n",
      "word [MISTAKES], count[2]\n",
      "word [DREAM], count[3]\n",
      "word [BORDERS], count[6]\n",
      "word [LOVED], count[1]\n",
      "word [BRING], count[9]\n",
      "word [RATES], count[1]\n",
      "word [HELL], count[6]\n",
      "word [BOTTOM], count[1]\n",
      "word [THOUGHT], count[1]\n",
      "word [KNOW], count[37]\n",
      "word [WORSE], count[9]\n",
      "word [WANT], count[41]\n",
      "word [LIKE], count[28]\n",
      "word [UNITED], count[2]\n",
      "word [BREAK], count[1]\n",
      "word [CRIMES], count[1]\n",
      "word [CALLED], count[3]\n",
      "word [WHICH], count[6]\n",
      "word [WEEK], count[2]\n",
      "word [HEROS], count[1]\n",
      "word [BUSINESSMAN], count[1]\n",
      "word [POLL], count[1]\n",
      "word [AIDS], count[1]\n",
      "word [IMAGINE], count[2]\n",
      "word [LANDSLIDES], count[1]\n",
      "word [BUILDING], count[1]\n",
      "word [START], count[4]\n",
      "word [INCLUDING], count[4]\n",
      "word [NEWS], count[1]\n",
      "word [JOE], count[2]\n",
      "word [KILLER], count[2]\n",
      "word [HEADS], count[4]\n",
      "word [DISGRACE], count[1]\n",
      "word [ALSO], count[1]\n",
      "word [DESTABILIZE], count[1]\n",
      "word [CHAOS], count[1]\n",
      "word [TRILLIONS], count[2]\n",
      "word [MEANT], count[1]\n",
      "word [CLUE], count[3]\n",
      "word [SIGN], count[4]\n",
      "word [COMPANY], count[3]\n",
      "word [BIG], count[9]\n",
      "word [REALLY], count[18]\n",
      "word [MIDST], count[1]\n",
      "word [RIGGED], count[8]\n",
      "word [PETTY], count[1]\n",
      "word [PAGE], count[6]\n",
      "word [MUSEUMS], count[1]\n",
      "word [GONE], count[3]\n",
      "word [BACK], count[28]\n",
      "word [ANYMORE], count[2]\n",
      "word [MAILS], count[3]\n",
      "word [WILL], count[26]\n",
      "word [TURN], count[5]\n",
      "word [TAKEN], count[1]\n",
      "word [REALIZE], count[1]\n",
      "word [TRYING], count[1]\n",
      "word [AMVETS], count[1]\n",
      "word [HISTORY], count[6]\n",
      "word [HOPE], count[1]\n",
      "word [GOVERNMENT], count[1]\n",
      "word [FAMILY], count[2]\n",
      "word [DSHTIDNT], count[1]\n",
      "word [HOW], count[11]\n",
      "word [ARENA], count[1]\n",
      "word [WHATEVER], count[1]\n",
      "word [STEEL], count[1]\n",
      "word [WINNING], count[14]\n",
      "word [GREAT], count[39]\n",
      "word [MAJORITY], count[1]\n",
      "word [COUNTRY], count[31]\n",
      "word [TERRIFIC], count[1]\n",
      "word [AIRPORTS], count[1]\n",
      "word [PENCE], count[1]\n",
      "word [SIR], count[4]\n",
      "word [AUDIENCE], count[2]\n",
      "word [DIRECTLY], count[1]\n",
      "word [FRACTION], count[2]\n",
      "word [TRUMPS], count[1]\n",
      "word [PLANES], count[1]\n",
      "word [REINCE], count[1]\n",
      "word [STRENGTH], count[3]\n",
      "word [STRAIGHT], count[1]\n",
      "word [LEFT], count[13]\n",
      "word [VALUES], count[1]\n",
      "word [EITHER], count[2]\n",
      "word [LOT], count[17]\n",
      "word [TALKING], count[6]\n",
      "word [BEAUTIFUL], count[5]\n",
      "word [FBI], count[3]\n",
      "word [LOW], count[2]\n",
      "word [ME], count[22]\n",
      "word [NEGOTIATING], count[4]\n",
      "word [DOCUMENT], count[1]\n",
      "word [BUSINESS], count[5]\n",
      "word [ARENT], count[1]\n",
      "word [ABOUT], count[28]\n",
      "word [ENDORSED], count[7]\n",
      "word [COUNTRIES], count[4]\n",
      "word [SHARPER], count[1]\n",
      "word [HIGHWAYS], count[1]\n",
      "word [SURVIVORS], count[1]\n",
      "word [APPLAUSE], count[1]\n",
      "word [CARRIER], count[5]\n",
      "word [FIXED], count[1]\n",
      "word [WOULD], count[34]\n",
      "word [DEALS], count[12]\n",
      "word [U], count[1]\n",
      "word [PROOF], count[1]\n",
      "word [HAPPENED], count[13]\n",
      "word [YOULL], count[2]\n",
      "word [WANTS], count[1]\n",
      "word [FEELING], count[1]\n",
      "word [LANDSLIDE], count[1]\n",
      "word [VIRTUALLY], count[1]\n",
      "word [DOCUMENTS], count[1]\n",
      "word [THREW], count[1]\n",
      "word [TOLD], count[3]\n",
      "word [SORRY], count[2]\n",
      "word [SNL], count[1]\n",
      "word [EVERYTHING], count[3]\n",
      "word [CALLING], count[1]\n",
      "word [LOSE], count[5]\n",
      "word [DISASTER], count[2]\n",
      "word [EDUCATION], count[2]\n",
      "word [AWAY], count[2]\n",
      "word [SEC], count[2]\n",
      "word [THEYRE], count[18]\n",
      "word [PATROL], count[3]\n",
      "word [HAPPEN], count[5]\n",
      "word [GRIEVES], count[1]\n",
      "word [ROUGH], count[4]\n",
      "word [FOOTBALL], count[1]\n",
      "word [YEMEN], count[3]\n",
      "word [OK], count[60]\n",
      "word [SOMEBODY], count[2]\n",
      "word [GUESS], count[1]\n",
      "word [FIVE], count[7]\n",
      "word [SAN], count[1]\n",
      "word [FLY], count[1]\n",
      "word [S], count[1]\n",
      "word [FLOAT], count[1]\n",
      "word [USE], count[4]\n",
      "word [TREMENDOUS], count[1]\n",
      "word [VICTORY], count[1]\n",
      "word [SIGNED], count[6]\n",
      "word [BOATS], count[1]\n",
      "word [REPRESENTING], count[1]\n",
      "word [SLOWLY], count[1]\n",
      "word [ELECTION], count[1]\n",
      "word [BUSINESSMEN], count[1]\n",
      "word [ATE], count[1]\n",
      "word [MOST], count[5]\n",
      "word [SMART], count[8]\n",
      "word [ACROSS], count[2]\n",
      "word [TREATED], count[1]\n",
      "word [READY], count[1]\n",
      "word [HOUDINI], count[1]\n",
      "word [PRISONERS], count[2]\n",
      "word [FOLLOWED], count[2]\n",
      "word [MORNING], count[1]\n",
      "word [BETTER], count[8]\n",
      "word [MANIAC], count[1]\n",
      "word [SEND], count[3]\n",
      "word [THERE], count[17]\n",
      "word [LOOK], count[35]\n",
      "word [BERNIE], count[4]\n",
      "word [HOME], count[2]\n",
      "word [DISCUSS], count[1]\n",
      "word [DEFINITION], count[3]\n",
      "word [GET], count[36]\n",
      "word [DEFICITS], count[2]\n",
      "word [FOUR], count[7]\n",
      "word [FIGHTER], count[3]\n",
      "word [HAS], count[12]\n",
      "word [APART], count[1]\n",
      "word [TALK], count[2]\n",
      "word [LOSS], count[1]\n",
      "word [YORK], count[1]\n",
      "word [ENFORCEMENT], count[3]\n",
      "word [HAPPENS], count[1]\n",
      "word [AS], count[15]\n",
      "word [BAD], count[9]\n",
      "word [KPZCOMPANIES], count[1]\n",
      "word [COMING], count[3]\n",
      "word [FAT], count[1]\n",
      "word [LEG], count[1]\n",
      "word [TELLING], count[2]\n",
      "word [POWER], count[3]\n",
      "word [ENEMY], count[3]\n",
      "word [FRIENDLIER], count[2]\n",
      "word [VARIOUS], count[1]\n",
      "word [HELP], count[3]\n",
      "word [COUPLE], count[1]\n",
      "word [ROOM], count[4]\n",
      "word [WTO], count[3]\n",
      "word [VICTORIES], count[1]\n",
      "word [FELLAS], count[1]\n",
      "word [NEGOTIATORS], count[4]\n",
      "word [WORST], count[5]\n",
      "word [NUMBER], count[4]\n",
      "word [WERE], count[113]\n",
      "word [BOBBY], count[7]\n",
      "word [KNOWS], count[3]\n",
      "word [POLICE], count[8]\n",
      "word [HIM], count[10]\n",
      "word [FEW], count[2]\n",
      "word [SAFE], count[4]\n",
      "word [STATE], count[4]\n",
      "word [COMPARISON], count[1]\n",
      "word [MAKE], count[19]\n",
      "word [CROOKED], count[6]\n",
      "word [DEAL], count[11]\n",
      "word [ACTUALLY], count[4]\n",
      "word [BLUE], count[1]\n",
      "word [ARENAS], count[2]\n",
      "word [CERTAINLY], count[1]\n",
      "word [WOULDNT], count[3]\n",
      "word [LGBT], count[5]\n",
      "word [AFTER], count[4]\n",
      "word [NRA], count[4]\n",
      "word [NUCLEAR], count[5]\n",
      "word [ARPAIO], count[1]\n",
      "word [SCC], count[1]\n",
      "word [ADMITTING], count[1]\n",
      "word [OBVIOUSLY], count[1]\n"
     ]
    }
   ],
   "source": [
    "for word in words_uppercase:\n",
    "    \n",
    "    print( \"word [%s], count[%d]\" % ( word, word_counts[ word ] ) ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len( tokens_unique ) == len( tokens_unique_lowercase ) True\n",
      "Words not in embeddings_index: 131\n",
      "len( tokenizer.word_index ) 7668\n",
      "len( embeddings_index ) 7537\n",
      "vocab_size 7669\n"
     ]
    }
   ],
   "source": [
    "print( \"len( tokens_unique ) == len( tokens_unique_lowercase )\", len( tokens_unique ) == len( tokens_unique_lowercase ) )\n",
    "print( \"Words not in embeddings_index:\", len( tokens_unique_lowercase ) - len( embeddings_index ) )\n",
    "#print( embeddings_index[ \"the\" ] )\n",
    "print( \"len( tokenizer.word_index )\", len( tokenizer.word_index ) )\n",
    "print( \"len( embeddings_index )\", len( embeddings_index ) )\n",
    "print( \"vocab_size\", vocab_size )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transform into Matrix That Maps Coefs by Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Missing words: 131\n"
     ]
    }
   ],
   "source": [
    "# create a weight matrix for words in training docs\n",
    "embedding_matrix = np.zeros( ( vocab_size, embeddings_dimension ) )\n",
    "missing_words = []\n",
    "\n",
    "# we need this to create empty coefficients array\n",
    "dummy_shape = embeddings_index[ \"the\" ].shape\n",
    "\n",
    "# use the mixed case list: tokens_unique\n",
    "#for i, word in enumerate( tokens_unique ):\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    \n",
    "    embedding_vector = embeddings_index.get( word )\n",
    "    \n",
    "    # not all words in our token list are in the wikipedia 400K set!\n",
    "    if embedding_vector is None:\n",
    "        \n",
    "        # 1st time, get the lowercase vector\n",
    "        embedding_vector = embeddings_index.get( word.lower() )\n",
    "        \n",
    "    # 2nd test: If not found as original or lower case, then assign it an empty vector\n",
    "    if embedding_vector is None:  \n",
    "        \n",
    "        # report and create empty coefficients array\n",
    "        missing_words.append( word )\n",
    "        embedding_vector = np.zeros( dummy_shape )\n",
    "     \n",
    "    #print( \"i\", i, \"word\", word )\n",
    "    embedding_matrix[ i ] = embedding_vector\n",
    "    \n",
    "print( \"Missing words:\", len( missing_words ) )\n",
    "\n",
    "# before hashtag segmentation Missing words: 5640\n",
    "# after hashtag segmentation: 5060"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['1stand',\n",
       " '22Kill',\n",
       " 'Alumisource',\n",
       " 'Ayeyeye',\n",
       " 'Beada',\n",
       " 'Brexits',\n",
       " 'CAMERAMANS',\n",
       " 'CNBCs',\n",
       " 'Cruzs',\n",
       " 'DSHTIDNT',\n",
       " 'DonaldJTrump',\n",
       " 'Everybodys',\n",
       " 'Everyones',\n",
       " 'Everythings',\n",
       " 'ISISs',\n",
       " 'Indianas',\n",
       " 'Iowas',\n",
       " 'Itll',\n",
       " 'Ivankas',\n",
       " 'Jebs',\n",
       " 'KPZCOMPANIES',\n",
       " 'Kasichs',\n",
       " 'Komatsus',\n",
       " 'Kristols',\n",
       " 'MAEPGESSENGER',\n",
       " 'NASHLTIONALLY',\n",
       " 'Nabiscos',\n",
       " 'Nabsico',\n",
       " 'Nobodys',\n",
       " 'OMalley',\n",
       " 'OREILLY',\n",
       " 'OReilly',\n",
       " 'Obamacares',\n",
       " 'Ordierno',\n",
       " 'Orlandos',\n",
       " 'PLIBLG',\n",
       " 'Pences',\n",
       " 'Pfizers',\n",
       " 'REINCE',\n",
       " 'RealDonaldTrump',\n",
       " 'Reince',\n",
       " 'Romneycare',\n",
       " 'Rubios',\n",
       " 'Sanderss',\n",
       " 'Sarahs',\n",
       " 'Schusters',\n",
       " 'Sharons',\n",
       " 'Shouldnt',\n",
       " 'Swenden',\n",
       " 'THETORY',\n",
       " 'THEYLL',\n",
       " 'THEYVE',\n",
       " 'Therell',\n",
       " 'Theyd',\n",
       " 'Theyll',\n",
       " 'Theyve',\n",
       " 'Universitys',\n",
       " 'Vetdogs',\n",
       " 'Wisconsins',\n",
       " 'YOUVE',\n",
       " 'Youve',\n",
       " 'anybodys',\n",
       " 'ayeyayay',\n",
       " 'ayeyeye',\n",
       " 'baby’',\n",
       " 'bigly',\n",
       " 'bleh',\n",
       " 'braggadocious',\n",
       " 'braggingly',\n",
       " 'cetain',\n",
       " 'clearsighted',\n",
       " 'contractionclose',\n",
       " 'contractionopen',\n",
       " 'corruptness',\n",
       " 'couldve',\n",
       " 'deductability',\n",
       " 'embargoing',\n",
       " 'endexclamation',\n",
       " 'endperiod',\n",
       " 'endquestion',\n",
       " 'escavators',\n",
       " 'eschelons',\n",
       " 'everybodys',\n",
       " 'everyones',\n",
       " 'everythings',\n",
       " 'familys',\n",
       " 'fricking',\n",
       " 'hadnt',\n",
       " 'hijackists',\n",
       " 'incomptent',\n",
       " 'itll',\n",
       " 'mackeral',\n",
       " 'manufactur',\n",
       " 'mortagage',\n",
       " 'nobodys',\n",
       " 'obselete',\n",
       " 'oclock',\n",
       " 'pausecolon',\n",
       " 'pausecomma',\n",
       " 'pausedash',\n",
       " 'pauseemdash',\n",
       " 'pausesemicolon',\n",
       " 'phenomenol',\n",
       " 'quoteclose',\n",
       " 'quoteopen',\n",
       " 'resect',\n",
       " 'resonants',\n",
       " 'retweeted',\n",
       " 'scums',\n",
       " 'shouldnt',\n",
       " 'shouldve',\n",
       " 'smartquoteclose',\n",
       " 'smartquoteopen',\n",
       " 'somebodys',\n",
       " 'thatd',\n",
       " 'thatll',\n",
       " 'thereve',\n",
       " 'theyd',\n",
       " 'theyll',\n",
       " 'theyve',\n",
       " 'tweetelipsis',\n",
       " 'waitlists',\n",
       " 'werent',\n",
       " 'wholl',\n",
       " 'whove',\n",
       " 'wouldve',\n",
       " 'youve',\n",
       " '’04',\n",
       " '’16',\n",
       " '’17',\n",
       " '’tis']"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missing_words.sort()\n",
    "missing_words#[ :20 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(7669, 300)\n",
      "(209783, 50)\n",
      "(209783,)\n"
     ]
    }
   ],
   "source": [
    "print( embedding_matrix.shape )\n",
    "print( X.shape )\n",
    "print( y.shape )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write missing words to file\n",
    "with open( \"output/missing-words-speeches.txt\", \"w\" ) as out_file:\n",
    "    \n",
    "    for word in missing_words:\n",
    "\n",
    "        out_file.write( \"%s\\n\" % word )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300\n",
      "0.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "132"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# confirm visually that \n",
    "print( len( embedding_matrix[ 0 ] ) )\n",
    "print( sum( embedding_matrix[ 0 ] ) )\n",
    "empty_coefficients_count = 0\n",
    "\n",
    "for i in range( len( embedding_matrix ) ):\n",
    "    if sum( embedding_matrix[ i ] ) == 0:\n",
    "        empty_coefficients_count += 1\n",
    "        \n",
    "empty_coefficients_count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.3\n",
      "1.4.1\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "print( keras.__version__ )\n",
    "\n",
    "import tensorflow as tf\n",
    "print( tf.__version__ )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 50, 300)           2300700   \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, 50, 200)           320800    \n",
      "_________________________________________________________________\n",
      "bidirectional_2 (Bidirection (None, 200)               240800    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 100)               20100     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 7669)              774569    \n",
      "=================================================================\n",
      "Total params: 3,656,969\n",
      "Trainable params: 3,656,969\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# define model\n",
    "model = Sequential()\n",
    "\n",
    "# now using a pre-trained, non-trainable embedding from glove's wiki analysis\n",
    "model.add( Embedding( vocab_size, embeddings_dimension, weights=[embedding_matrix], input_length=seq_length, trainable=True ) )\n",
    "model.add( Bidirectional( LSTM( seq_length * 2, return_sequences=True ) ) )\n",
    "#model.add( Dropout( 0.90 ) )\n",
    "model.add( Bidirectional( LSTM( seq_length * 2 ) ) )\n",
    "#model.add( Dropout( 0.90 ) )\n",
    "model.add( Dense( seq_length * 2, activation='relu' ) )\n",
    "\n",
    "# fixed TypeError below, downgraded keras from 2.1.5 to 2.1.3: https://github.com/keras-team/keras/issues/9621\n",
    "# TypeError: softmax() got an unexpected keyword argument 'axis'\n",
    "model.add( Dense( vocab_size, activation='softmax' ) )\n",
    "\n",
    "print( model.summary() )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1638.9296875\n",
      "204.06906614785993\n"
     ]
    }
   ],
   "source": [
    "# calc batch size\n",
    "print( len( sequences ) / 128 )\n",
    "print( len( sequences ) / 1028 )\n",
    "# Was:\n",
    "#batch_size = 124\n",
    "batch_size = 1024\n",
    "\n",
    "# can't remember where I read that batch sizes larger than 512 cause erratic convergence patterns.\n",
    "# TODO: find that article!\n",
    "#batch_size = 512\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Model?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "load = input( \"Load model? [y/n]\" )\n",
    "\n",
    "if load == \"y\":\n",
    "    \n",
    "    model_name = \"models/trump-speeches-take-III.h5\"\n",
    "    print( \"Loading model %s\" % model_name )\n",
    "    model = load_model( model_name )\n",
    "    \n",
    "else:\n",
    "    \n",
    "    print( \"NOT loading model, using default untrained model\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.06.27 11:29\n",
      "Epoch 1/100\n",
      "209783/209783 [==============================] - 53s 254us/step - loss: 6.1914 - acc: 0.0685\n",
      "Epoch 2/100\n",
      "209783/209783 [==============================] - 53s 253us/step - loss: 5.2401 - acc: 0.1445\n",
      "Epoch 3/100\n",
      "209783/209783 [==============================] - 53s 254us/step - loss: 4.6947 - acc: 0.1959\n",
      "Epoch 4/100\n",
      "209783/209783 [==============================] - 53s 255us/step - loss: 4.3975 - acc: 0.2213\n",
      "Epoch 5/100\n",
      "209783/209783 [==============================] - 53s 255us/step - loss: 4.2015 - acc: 0.2375\n",
      "Epoch 6/100\n",
      "209783/209783 [==============================] - 53s 255us/step - loss: 4.0606 - acc: 0.2481\n",
      "Epoch 7/100\n",
      "209783/209783 [==============================] - 53s 255us/step - loss: 3.9436 - acc: 0.2573\n",
      "Epoch 8/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 3.8405 - acc: 0.2660\n",
      "Epoch 9/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 3.7450 - acc: 0.2752\n",
      "Epoch 10/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 3.6578 - acc: 0.2842\n",
      "Epoch 11/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 3.5729 - acc: 0.2932\n",
      "Epoch 12/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 3.4913 - acc: 0.3012\n",
      "Epoch 13/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 3.4120 - acc: 0.3099\n",
      "Epoch 14/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 3.3328 - acc: 0.3182\n",
      "Epoch 15/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 3.2557 - acc: 0.3274\n",
      "Epoch 16/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 3.1809 - acc: 0.3362\n",
      "Epoch 17/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 3.1034 - acc: 0.3460\n",
      "Epoch 18/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 3.0297 - acc: 0.3549\n",
      "Epoch 19/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 2.9573 - acc: 0.3652\n",
      "Epoch 20/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 2.8834 - acc: 0.3761\n",
      "Epoch 21/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 2.8133 - acc: 0.3857\n",
      "Epoch 22/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 2.7427 - acc: 0.3968\n",
      "Epoch 23/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 2.6762 - acc: 0.4072\n",
      "Epoch 24/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 2.6082 - acc: 0.4187\n",
      "Epoch 25/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 2.5468 - acc: 0.4295\n",
      "Epoch 26/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 2.4831 - acc: 0.4410\n",
      "Epoch 27/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 2.4195 - acc: 0.4526\n",
      "Epoch 28/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 2.3615 - acc: 0.4632\n",
      "Epoch 29/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 2.2999 - acc: 0.4746\n",
      "Epoch 30/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 2.2449 - acc: 0.4850\n",
      "Epoch 31/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 2.1887 - acc: 0.4953\n",
      "Epoch 32/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 2.1346 - acc: 0.5067\n",
      "Epoch 33/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 2.0823 - acc: 0.5175\n",
      "Epoch 34/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 2.0265 - acc: 0.5283\n",
      "Epoch 35/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.9797 - acc: 0.5381\n",
      "Epoch 36/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.9281 - acc: 0.5489\n",
      "Epoch 37/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.8807 - acc: 0.5581\n",
      "Epoch 38/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.8348 - acc: 0.5688\n",
      "Epoch 39/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.7885 - acc: 0.5769\n",
      "Epoch 40/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.7460 - acc: 0.5865\n",
      "Epoch 41/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.6997 - acc: 0.5971\n",
      "Epoch 42/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.6574 - acc: 0.6058\n",
      "Epoch 43/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.6164 - acc: 0.6145\n",
      "Epoch 44/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.5790 - acc: 0.6223\n",
      "Epoch 45/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.5376 - acc: 0.6315\n",
      "Epoch 46/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.4949 - acc: 0.6410\n",
      "Epoch 47/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.4618 - acc: 0.6486\n",
      "Epoch 48/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.4248 - acc: 0.6571\n",
      "Epoch 49/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.3991 - acc: 0.6619\n",
      "Epoch 50/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.3563 - acc: 0.6714\n",
      "Epoch 51/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.3214 - acc: 0.6795\n",
      "Epoch 52/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.2898 - acc: 0.6867\n",
      "Epoch 53/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.2492 - acc: 0.6967\n",
      "Epoch 54/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.2285 - acc: 0.7011\n",
      "Epoch 55/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.1911 - acc: 0.7097\n",
      "Epoch 56/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.1663 - acc: 0.7150\n",
      "Epoch 57/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.1376 - acc: 0.7212\n",
      "Epoch 58/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.1031 - acc: 0.7298\n",
      "Epoch 59/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.0722 - acc: 0.7380\n",
      "Epoch 60/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.0485 - acc: 0.7424\n",
      "Epoch 61/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.0231 - acc: 0.7497\n",
      "Epoch 62/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 1.0041 - acc: 0.7533\n",
      "Epoch 63/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.9774 - acc: 0.7579\n",
      "Epoch 64/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.9496 - acc: 0.7664\n",
      "Epoch 65/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.9332 - acc: 0.7690\n",
      "Epoch 66/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.9077 - acc: 0.7754\n",
      "Epoch 67/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.8788 - acc: 0.7827\n",
      "Epoch 68/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.8625 - acc: 0.7862\n",
      "Epoch 69/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.8366 - acc: 0.7934\n",
      "Epoch 70/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.8145 - acc: 0.7980\n",
      "Epoch 71/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.7875 - acc: 0.8046\n",
      "Epoch 72/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.7760 - acc: 0.8065\n",
      "Epoch 73/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.7571 - acc: 0.8109\n",
      "Epoch 74/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.7431 - acc: 0.8146\n",
      "Epoch 75/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.7234 - acc: 0.8202\n",
      "Epoch 76/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.7030 - acc: 0.8242\n",
      "Epoch 77/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.6850 - acc: 0.8289\n",
      "Epoch 78/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.6682 - acc: 0.8327\n",
      "Epoch 79/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.6525 - acc: 0.8363\n",
      "Epoch 80/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.6306 - acc: 0.8432\n",
      "Epoch 81/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.6208 - acc: 0.8437\n",
      "Epoch 82/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.6209 - acc: 0.8425\n",
      "Epoch 83/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.6047 - acc: 0.8471\n",
      "Epoch 84/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.5804 - acc: 0.8545\n",
      "Epoch 85/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.5642 - acc: 0.8580\n",
      "Epoch 86/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.5469 - acc: 0.8630\n",
      "Epoch 87/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.5337 - acc: 0.8663\n",
      "Epoch 88/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.5239 - acc: 0.8682\n",
      "Epoch 89/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.5176 - acc: 0.8688\n",
      "Epoch 90/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.5126 - acc: 0.8701\n",
      "Epoch 91/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.4992 - acc: 0.8730\n",
      "Epoch 92/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.4883 - acc: 0.8757\n",
      "Epoch 93/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.4823 - acc: 0.8772\n",
      "Epoch 94/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.4507 - acc: 0.8876\n",
      "Epoch 95/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.4354 - acc: 0.8906\n",
      "Epoch 96/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.4478 - acc: 0.8852\n",
      "Epoch 97/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.4376 - acc: 0.8879\n",
      "Epoch 98/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.4148 - acc: 0.8950\n",
      "Epoch 99/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.4035 - acc: 0.8977\n",
      "Epoch 100/100\n",
      "209783/209783 [==============================] - 54s 256us/step - loss: 0.3977 - acc: 0.8988\n",
      "2018.06.27 12:59\n",
      "Time to process: [1.4905232154660755] hours\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "\n",
    "# Per comment here: https://stackoverflow.com/questions/46293734/memoryerror-in-keras-utils-np-utils-to-categoricalhttps://stackoverflow.com/questions/46293734/memoryerror-in-keras-utils-np-utils-to-categorical\n",
    "model.compile( loss='sparse_categorical_crossentropy', optimizer='adam', metrics=[ 'accuracy' ] )\n",
    "# model.compile( loss='categorical_crossentropy', optimizer='adam', metrics=[ 'accuracy' ] )\n",
    "\n",
    "model.fit( X, y, batch_size=batch_size, epochs=100 )\n",
    "end_time = get_time()\n",
    "print_time( start_time, end_time, interval=\"hours\" )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the whole model to file\n",
    "model.save( \"models/trump-speeches-take-III.h5\" )\n",
    "\n",
    "# save the tokenizer\n",
    "dump( tokenizer, open( \"tokenizers/trump-speeches-take-III.dump\", 'wb' ) )\n",
    "\n",
    "# save embedding_matrix based on wiki embeddings, complete w/ missing coefficients array dummies\n",
    "dump( embedding_matrix, open( \"embeddings/trump-speeches-take-III.glove\", 'wb' ) )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use The Model to Generate Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seq_length = len( lines[ 0 ].split() ) - 1\n",
    "seq_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'”'"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "punctuation_dict.get( \"smartquoteclose\", \"bar\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_yhats( preds, temperature=1.0 ):\n",
    "    \n",
    "    # helper function to sample an index from a probability array\n",
    "    preds = np.asarray( preds ).astype( 'float64' )\n",
    "    preds = np.log( preds ) / temperature\n",
    "    exp_preds = np.exp( preds )\n",
    "    preds = exp_preds / np.sum( exp_preds )\n",
    "    probas = np.random.multinomial( 1, preds, 1 )\n",
    "    return np.argmax( probas )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_seq( model, tokenizer, seq_length, seed_text, n_words, temperature=1.0 ):\n",
    "    \n",
    "    result = list()\n",
    "    result_literal = list()\n",
    "    in_text = seed_text\n",
    "    yhat = [ 0.1 ]\n",
    "    \n",
    "    # generate a fixed number of words\n",
    "    for _ in range( n_words ):\n",
    "        \n",
    "        # encode the text as integer\n",
    "        encoded = tokenizer.texts_to_sequences( [ in_text ] )[ 0 ] \n",
    "        \n",
    "        # truncate sequences to a fixed length\n",
    "        encoded = pad_sequences( [ encoded ], maxlen=seq_length, truncating='pre' ) \n",
    "        \n",
    "        # predict probabilities for each word\n",
    "        yhat = model.predict_classes( encoded, verbose=0 )\n",
    "        # this returns list of predictions\n",
    "        yhats = model.predict( encoded, verbose=0 )[ 0 ]\n",
    "        \n",
    "        # map predicted word index to word\n",
    "        #print( \"yhat\", yhat, words_by_id[ yhat[ 0 ] ] )\n",
    "        \n",
    "        out_word_id = sample_yhats( yhats, temperature )\n",
    "        out_word = words_by_id[ out_word_id ]\n",
    "        # out_word = words_by_id[ yhat[ 0 ] ]\n",
    "                \n",
    "        # append to input\n",
    "        in_text += ' ' + out_word\n",
    "        \n",
    "        #result.append( out_word )\n",
    "        # substitute punctuation tags for actual punctuation\n",
    "        result.append( punctuation_dict.get( out_word, out_word ) )\n",
    "        \n",
    "#         if out_word == \"closetweetclose\":\n",
    "#             #print( \"Tweet end detected\" )\n",
    "#             break\n",
    "            \n",
    "    return ' '.join( result )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reformat_punctuation( doc ):\n",
    "    \n",
    "    doc = doc.replace( ' . ', '. ' )\n",
    "    doc = doc.replace( ' ! ', '! ' )\n",
    "    doc = doc.replace( ' ? ', '? ' )\n",
    "    doc = doc.replace( ' , ', ', ' )\n",
    "    doc = doc.replace( ' : ', ': ' )\n",
    "    doc = doc.replace( ' ; ', '; ' )\n",
    "    \n",
    "    doc = doc.replace( '“ ', '“' )\n",
    "    doc = doc.replace( ' ”', '”' )\n",
    "    doc = doc.replace( \"attweetat\", '@' )\n",
    "    #doc = doc.replace( \"hashtweethash\", '#' )\n",
    "    doc = doc.replace( \" amp; \", '&' )\n",
    "    \n",
    "    return doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "to be pretty good because I see the response I mean pausecomma even this with all the people still trying to come into this building endperiod Somebody said contractionopen its contractionclose a barn endperiod This is a very nice barn actually endperiod But with all of the people still trying to \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rruiz/anaconda3/envs/py36/lib/python3.6/site-packages/ipykernel_launcher.py:5: RuntimeWarning: divide by zero encountered in log\n",
      "  \"\"\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "... come back with that and contractionopen its contractionclose going to be Youre of an okay friend and you know , you have a couple of dollars to into the first desk in Brooklyn and gave me an NBC show and I disappeared . I built the honor of his name\n"
     ]
    }
   ],
   "source": [
    "# select a seed text\n",
    "seed_text = lines[ randint( 0, len( lines ) ) ]\n",
    "print( seed_text, \"\\n\" )\n",
    "\n",
    "# # substitute the seed words\n",
    "# raw_text = seed_text.split( \" \" )\n",
    "\n",
    "# clean_text = [ punctuation_dict.get( word, word ) for word in raw_text ]\n",
    "# clean_text = ' '.join( clean_text )\n",
    "\n",
    "# print( reformat_punctuation( clean_text ) + '... \\n' )\n",
    "# #print( len( seed_text.split( \" \" ) ) )\n",
    "\n",
    "# generate new text\n",
    "generated = generate_seq( model, tokenizer, seq_length, seed_text, 50, 1.0 )\n",
    "\n",
    "print( \"... \" + generated )\n",
    "#print()\n",
    "#print( \"\\n\\n... \" + reformat_punctuation( generated ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1  my_input = input()  2  #\"opentweetopen \" + my_input  3  generated = generate_seq( model, tokenizer, seq_length, my_input, 50, 1.5 )  4  print( \"... \" + reformat_punctuation( generated ) )  Nobody has better respect for intelligence than Donald Trump.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rruiz/anaconda3/envs/py36/lib/python3.6/site-packages/ipykernel_launcher.py:5: RuntimeWarning: divide by zero encountered in log\n",
      "  \"\"\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "... disaster The Obama campaign until a change. And contractionopen thats contractionclose the way it contractionopen wont contractionclose be; contractionopen shes contractionclose so stupid enough to bring us back and maybe really shocking contractionopen Ive contractionclose heard that right, right You know if you contractionopen dont contractionclose want\n"
     ]
    }
   ],
   "source": [
    "my_input = input()\n",
    "#\"opentweetopen \" + my_input\n",
    "generated = generate_seq( model, tokenizer, seq_length, my_input, 50, 1.5 )\n",
    "print( \"... \" + reformat_punctuation( generated ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating text with the following seed: \"a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a Nobody has better respect for intelligence than Donald Trump .\"\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#initiate sentences\n",
    "seed_sentences = \"Nobody has better respect for intelligence than Donald Trump .\"\n",
    "generated = ''\n",
    "sentence = []\n",
    "\n",
    "for i in range( seq_length ):\n",
    "    sentence.append( \"a\" )\n",
    "\n",
    "seed = seed_sentences.split()\n",
    "\n",
    "for i in range( len( seed ) ):\n",
    "    sentence[ seq_length - i - 1 ]= seed[ len( seed ) - i - 1 ]\n",
    "\n",
    "generated += ' '.join(sentence)\n",
    "print('Generating text with the following seed: \"' + ' '.join(sentence) + '\"')\n",
    "\n",
    "print ()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_number = 100\n",
    "#generate the text\n",
    "for i in range(words_number):\n",
    "    #create the vector\n",
    "    x = np.zeros((1, seq_length, vocab_size))\n",
    "    for t, word in enumerate(sentence):\n",
    "        x[0, t, vocab[word]] = 1.\n",
    "    #print(x.shape)\n",
    "\n",
    "    #calculate next word\n",
    "    preds = model.predict(x, verbose=0)[0]\n",
    "    next_index = sample(preds, 0.34)\n",
    "    next_word = vocabulary_inv[next_index]\n",
    "\n",
    "    #add the next word to the text\n",
    "    generated += \" \" + next_word\n",
    "    # shift the sentence by one, and and the next word at its end\n",
    "    sentence = sentence[1:] + [next_word]\n",
    "\n",
    "print(generated)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}