word-level-rnn-w-word2vec-tweets-w-boundaries-take-I.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Based on: https://machinelearningmastery.com/use-word-embedding-layers-deep-learning-keras/ and https://machinelearningmastery.com/how-to-develop-a-word-level-neural-language-model-in-keras/ and https://github.com/stanfordnlp/GloVe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.05.23 15:42\n",
      "Time to process: [1] seconds\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import time\n",
    "import datetime\n",
    "import string\n",
    "from pickle import dump\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils import to_categorical\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import LSTM\n",
    "from keras.layers import Embedding\n",
    "from keras.models import load_model\n",
    "import collections\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from random import randint\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "def get_time( output=True ):\n",
    "    \n",
    "    temp = time.time()\n",
    "    if output:\n",
    "        now = datetime.datetime.now()\n",
    "        print( now.strftime( \"%Y.%m.%d %H:%M\" ) )\n",
    "        \n",
    "    return temp\n",
    "\n",
    "foo = get_time()\n",
    "\n",
    "def print_time( start_time, end_time, interval=\"seconds\" ):\n",
    "    \n",
    "    if interval == \"hours\":\n",
    "        print ( \"Time to process: [%s] hours\" % ( str( ( end_time - start_time ) / 60 / 60 ) ) )\n",
    "    else:\n",
    "        print ( \"Time to process: [%s] seconds\" % ( str( end_time - start_time ) ) )\n",
    "\n",
    "print_time( 0, 1 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Just met with UN Secretary-General António Guterres who is working hard to “Make the United Nations Great Again.” When the UN does more to solve conflicts around the world it means the U.S. has less t\n"
     ]
    }
   ],
   "source": [
    "# load doc into memory\n",
    "def load_doc( filename ):\n",
    "    \n",
    "    # open the file as read only\n",
    "    file = open( filename, 'r' )\n",
    "    # read all text\n",
    "    text = file.read()\n",
    "    # close the file\n",
    "    file.close()\n",
    "    \n",
    "    return text\n",
    "\n",
    "# load document\n",
    "#in_filename = \"../texts/alice-in-wonderland.txt\"\n",
    "#in_filename = \"../texts/dr-zeuss-compilation.txt\"\n",
    "in_filename = \"../texts/trump-tweets.txt\"\n",
    "doc = load_doc( in_filename )\n",
    "print( doc[ :200 ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Just met with UN Secretary-General António Guterres who is working hard to “Make the United Nations Great Again.” When the UN does more to solve conflicts around the world it means the U.S. has less to do and we save money. @NikkiHaley is doing a fantastic job! https://t.co/pqUv6cyH2z\n",
      "America is a Nation that believes in the power of redemption. America is a Nation that believes in second chances \n"
     ]
    }
   ],
   "source": [
    "print( doc[ :400 ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\"#$%&\\'()*+,-/:;<=>@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# my_punctuation = string.punctuation\n",
    "# print( type( my_punctuation ) )\n",
    "# print( my_punctuation )\n",
    "my_punctuation = '\"#$%&\\'()*+,-/:;<=>@[\\\\]^_`{|}~'\n",
    "my_punctuation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Encoded Punctuation to Punctuation Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "punctuation_dict = {}\n",
    "punctuation_dict[ \"endperiod\" ] = \".\"\n",
    "punctuation_dict[ \"endquestion\" ] = \"?\"\n",
    "punctuation_dict[ \"endexclamation\" ] = \"!\"\n",
    "punctuation_dict[ \"pausecomma\" ] = \",\"\n",
    "punctuation_dict[ \"pausecolon\" ] = \":\"\n",
    "punctuation_dict[ \"pausesemicolon\" ] = \";\"\n",
    "punctuation_dict[ \"smartquoteopen\" ] = '“'\n",
    "punctuation_dict[ \"smartquoteclose\" ] = '”'\n",
    "punctuation_dict[ \"attweetat\" ] = '@'\n",
    "punctuation_dict[ \"tweetlink\" ] = \"[link]\"\n",
    "punctuation_dict[ \"hashtweethash\" ] = '#'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# turn a doc into clean tokens\n",
    "def clean_doc( doc, to_lower=True ):\n",
    "    \n",
    "    # replace '--' with a space ' '\n",
    "    doc = doc.replace( '--', ' ' )\n",
    "    # replace sentence simple sentence boundaries w/ unique token/markers\n",
    "    doc = doc.replace( '. ', ' endperiod ' )\n",
    "    doc = doc.replace( '! ', ' endexclamation ' )\n",
    "    doc = doc.replace( '? ', ' endquestion ' )\n",
    "    doc = doc.replace( ', ', ' pausecomma ' )\n",
    "    doc = doc.replace( ': ', ' pausecolon ' )\n",
    "    doc = doc.replace( '; ', ' pausesemicolon ' )\n",
    "    doc = doc.replace( '“', 'smartquoteopen ' )\n",
    "    doc = doc.replace( '”', ' smartquoteclose' )\n",
    "    doc = doc.replace( \"@ \", \" \" ) # remove trailing @'s first...\n",
    "    doc = doc.replace( \" @\", \" attweetat\" ) # ...then encode 1st char @'s\n",
    "    doc = doc.replace( \"# \", \" \" ) # remove trailing #'s first...\n",
    "    doc = doc.replace( \" #\", \" hashtweethash\" ) # ...then encode 1st char #'s\n",
    "    \n",
    "    # replace links w/ \"tweetlink\"\n",
    "    # basic regex here: https://bytes.com/topic/python/answers/741677-find-replace-hyperlinks-string\n",
    "    http_pattern = r'http[^\\s\\n\\r]+'\n",
    "    doc = re.sub( http_pattern , \"tweetlink\", doc )\n",
    "    \n",
    "    # split into tokens by white space\n",
    "    tokens = doc.split()\n",
    "    \n",
    "    # remove punctuation from each token\n",
    "    table = str.maketrans( '', '', string.punctuation ) # will strip all .?!,:; that don't fit replace expr above.\n",
    "    #table = str.maketrans( '', '', my_punctuation )\n",
    "    tokens = [ w.translate( table ) for w in tokens ]\n",
    "    \n",
    "    # remove remaining tokens that are not alphabetic\n",
    "    if to_lower:\n",
    "        tokens = [ word for word in tokens if word.isalpha() ]\n",
    "    \n",
    "    # make lower case\n",
    "    tokens = [ word.lower() for word in tokens ] \n",
    "    \n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and Clean Doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.05.23 15:45\n",
      "['just', 'met', 'with', 'un', 'secretarygeneral', 'antónio', 'guterres', 'who', 'is', 'working', 'hard', 'to', 'smartquoteopen', 'make', 'the', 'united', 'nations', 'great', 'again', 'smartquoteclose', 'when', 'the', 'un', 'does', 'more', 'to', 'solve', 'conflicts', 'around', 'the', 'world', 'it', 'means', 'the', 'us', 'endperiod', 'has', 'less', 'to', 'do', 'and', 'we', 'save', 'money', 'endperiod', 'attweetatnikkihaley', 'is', 'doing', 'a', 'fantastic', 'job', 'endexclamation', 'tweetlink', 'america', 'is', 'a', 'nation', 'that', 'believes', 'in', 'the', 'power', 'of', 'redemption', 'endperiod', 'america', 'is', 'a', 'nation', 'that', 'believes', 'in', 'second', 'chances', 'and', 'america', 'is', 'a', 'nation', 'that', 'believes', 'that', 'the', 'best', 'is', 'always', 'yet', 'to', 'come', 'endexclamation', 'hashtweethashprisonreform', 'tweetlink', 'we', 'grieve', 'for', 'the', 'terrible', 'loss', 'of', 'life']\n",
      "Total Tokens: 399137\n",
      "Unique Tokens: 21148\n",
      "2018.05.23 15:45\n",
      "Time to process: [0.3152651786804199] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "\n",
    "# clean document\n",
    "tokens = clean_doc( doc )\n",
    "tokens_unique = list( set( tokens ) )\n",
    "print( tokens[ :100 ] )\n",
    "print( 'Total Tokens: %d' % len( tokens ) )\n",
    "print( 'Unique Tokens: %d' % len( tokens_unique ) )\n",
    "\n",
    "print_time( start_time, get_time() )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'just': 1334,\n",
       "         'met': 51,\n",
       "         'with': 2401,\n",
       "         'un': 37,\n",
       "         'secretarygeneral': 1,\n",
       "         'antónio': 1,\n",
       "         'guterres': 1,\n",
       "         'who': 1028,\n",
       "         'is': 6207,\n",
       "         'working': 202,\n",
       "         'hard': 324,\n",
       "         'to': 9223,\n",
       "         'smartquoteopen': 1300,\n",
       "         'make': 788,\n",
       "         'the': 15056,\n",
       "         'united': 196,\n",
       "         'nations': 66,\n",
       "         'great': 3051,\n",
       "         'again': 635,\n",
       "         'smartquoteclose': 1283,\n",
       "         'when': 625,\n",
       "         'does': 236,\n",
       "         'more': 812,\n",
       "         'solve': 34,\n",
       "         'conflicts': 5,\n",
       "         'around': 93,\n",
       "         'world': 376,\n",
       "         'it': 2446,\n",
       "         'means': 47,\n",
       "         'us': 1091,\n",
       "         'endperiod': 14523,\n",
       "         'has': 1370,\n",
       "         'less': 86,\n",
       "         'do': 936,\n",
       "         'and': 6876,\n",
       "         'we': 1986,\n",
       "         'save': 60,\n",
       "         'money': 337,\n",
       "         'attweetatnikkihaley': 3,\n",
       "         'doing': 458,\n",
       "         'a': 6911,\n",
       "         'fantastic': 218,\n",
       "         'job': 437,\n",
       "         'endexclamation': 3153,\n",
       "         'tweetlink': 6742,\n",
       "         'america': 807,\n",
       "         'nation': 97,\n",
       "         'that': 2463,\n",
       "         'believes': 15,\n",
       "         'in': 5728,\n",
       "         'power': 96,\n",
       "         'of': 6100,\n",
       "         'redemption': 1,\n",
       "         'second': 61,\n",
       "         'chances': 7,\n",
       "         'best': 409,\n",
       "         'always': 306,\n",
       "         'yet': 136,\n",
       "         'come': 197,\n",
       "         'hashtweethashprisonreform': 1,\n",
       "         'grieve': 5,\n",
       "         'for': 4062,\n",
       "         'terrible': 212,\n",
       "         'loss': 35,\n",
       "         'life': 149,\n",
       "         'send': 70,\n",
       "         'our': 1955,\n",
       "         'support': 212,\n",
       "         'love': 309,\n",
       "         'everyone': 230,\n",
       "         'affected': 10,\n",
       "         'by': 1651,\n",
       "         'this': 1339,\n",
       "         'horrible': 72,\n",
       "         'attack': 156,\n",
       "         'texas': 75,\n",
       "         'students': 12,\n",
       "         'families': 73,\n",
       "         'teachers': 15,\n",
       "         'personnel': 3,\n",
       "         'at': 2413,\n",
       "         'santa': 3,\n",
       "         'fe': 1,\n",
       "         'high': 172,\n",
       "         'school': 62,\n",
       "         'are': 2397,\n",
       "         'you': 3687,\n",
       "         'tragic': 14,\n",
       "         'hour': 37,\n",
       "         'will': 3460,\n",
       "         'be': 3470,\n",
       "         'forever': 19,\n",
       "         'shooting': 33,\n",
       "         'early': 48,\n",
       "         'reports': 34,\n",
       "         'not': 1648,\n",
       "         'looking': 279,\n",
       "         'good': 758,\n",
       "         'god': 59,\n",
       "         'bless': 28,\n",
       "         'all': 1448,\n",
       "         'there': 544,\n",
       "         'was': 1489,\n",
       "         'indeed': 9,\n",
       "         'least': 69,\n",
       "         'one': 670,\n",
       "         'fbi': 97,\n",
       "         'representative': 12,\n",
       "         'implanted': 1,\n",
       "         'political': 123,\n",
       "         'purposes': 5,\n",
       "         'into': 338,\n",
       "         'my': 1816,\n",
       "         'campaign': 273,\n",
       "         'president': 663,\n",
       "         'took': 69,\n",
       "         'place': 194,\n",
       "         'very': 1062,\n",
       "         'on': 4075,\n",
       "         'long': 221,\n",
       "         'before': 222,\n",
       "         'phony': 91,\n",
       "         'russia': 185,\n",
       "         'hoax': 24,\n",
       "         'became': 25,\n",
       "         'hot': 15,\n",
       "         'fake': 242,\n",
       "         'news': 475,\n",
       "         'story': 183,\n",
       "         'if': 842,\n",
       "         'true': 314,\n",
       "         'time': 783,\n",
       "         'biggest': 104,\n",
       "         'scandal': 32,\n",
       "         'why': 555,\n",
       "         'disgraced': 3,\n",
       "         'official': 31,\n",
       "         'andrew': 37,\n",
       "         'mccabe': 17,\n",
       "         'being': 472,\n",
       "         'investigated': 7,\n",
       "         'crooked': 280,\n",
       "         'hillary': 578,\n",
       "         'democrats': 215,\n",
       "         'virginia': 83,\n",
       "         'led': 20,\n",
       "         'clinton': 373,\n",
       "         'friend': 129,\n",
       "         'terry': 6,\n",
       "         'm': 19,\n",
       "         'under': 194,\n",
       "         'investigation': 46,\n",
       "         'they': 1330,\n",
       "         'killed': 75,\n",
       "         'gave': 122,\n",
       "         'wife': 65,\n",
       "         'her': 414,\n",
       "         'run': 200,\n",
       "         'office': 164,\n",
       "         'endquestion': 761,\n",
       "         'then': 228,\n",
       "         'dropped': 47,\n",
       "         'case': 82,\n",
       "         'media': 300,\n",
       "         'had': 408,\n",
       "         'me': 1173,\n",
       "         'calling': 36,\n",
       "         'immigrants': 35,\n",
       "         'or': 627,\n",
       "         'illegal': 154,\n",
       "         'animals': 8,\n",
       "         'wrong': 162,\n",
       "         'were': 369,\n",
       "         'begrudgingly': 1,\n",
       "         'forced': 36,\n",
       "         'withdraw': 1,\n",
       "         'their': 598,\n",
       "         'stories': 73,\n",
       "         'i': 4061,\n",
       "         'referred': 11,\n",
       "         'ms': 13,\n",
       "         'gang': 21,\n",
       "         'members': 48,\n",
       "         'as': 1074,\n",
       "         'big': 856,\n",
       "         'difference': 27,\n",
       "         'so': 1162,\n",
       "         'got': 263,\n",
       "         'purposely': 13,\n",
       "         'usual': 13,\n",
       "         'apparently': 3,\n",
       "         'doj': 8,\n",
       "         'put': 181,\n",
       "         'spy': 21,\n",
       "         'trump': 1777,\n",
       "         'never': 665,\n",
       "         'been': 527,\n",
       "         'done': 237,\n",
       "         'any': 216,\n",
       "         'necessary': 29,\n",
       "         'out': 935,\n",
       "         'frame': 3,\n",
       "         'donald': 600,\n",
       "         'crimes': 18,\n",
       "         'he': 1684,\n",
       "         'commit': 3,\n",
       "         'david': 75,\n",
       "         'asman': 1,\n",
       "         'attweetatloudobbs': 8,\n",
       "         'attweetatgreggjarrett': 1,\n",
       "         'really': 498,\n",
       "         'bad': 477,\n",
       "         'stuff': 20,\n",
       "         'honor': 235,\n",
       "         'visit': 72,\n",
       "         'heroes': 40,\n",
       "         'last': 536,\n",
       "         'night': 418,\n",
       "         'walter': 9,\n",
       "         'reed': 7,\n",
       "         'medical': 15,\n",
       "         'center': 67,\n",
       "         'nobody': 83,\n",
       "         'like': 734,\n",
       "         'them': 476,\n",
       "         'tomorrow': 285,\n",
       "         'house': 245,\n",
       "         'vote': 317,\n",
       "         'strong': 184,\n",
       "         'farm': 13,\n",
       "         'bill': 200,\n",
       "         'which': 304,\n",
       "         'includes': 11,\n",
       "         'work': 446,\n",
       "         'requirements': 2,\n",
       "         'must': 572,\n",
       "         'farmers': 12,\n",
       "         'talk': 164,\n",
       "         'mauricio': 1,\n",
       "         'macri': 2,\n",
       "         'argentina': 7,\n",
       "         'week': 149,\n",
       "         'such': 179,\n",
       "         'his': 1015,\n",
       "         'vision': 45,\n",
       "         'transforming': 1,\n",
       "         'economy': 185,\n",
       "         'unleashing': 1,\n",
       "         'its': 525,\n",
       "         'potential': 32,\n",
       "         'talking': 124,\n",
       "         'trade': 195,\n",
       "         'vice': 14,\n",
       "         'premier': 5,\n",
       "         'republic': 10,\n",
       "         'china': 481,\n",
       "         'liu': 1,\n",
       "         'congratulations': 264,\n",
       "         'new': 1074,\n",
       "         'cia': 28,\n",
       "         'director': 30,\n",
       "         'gina': 7,\n",
       "         'haspel': 6,\n",
       "         'congrats': 101,\n",
       "         'passing': 25,\n",
       "         'va': 48,\n",
       "         'mission': 18,\n",
       "         'act': 81,\n",
       "         'yesterday': 212,\n",
       "         'without': 122,\n",
       "         'funding': 29,\n",
       "         'veterans': 77,\n",
       "         'stand': 65,\n",
       "         'ending': 16,\n",
       "         'lines': 24,\n",
       "         'order': 145,\n",
       "         'receive': 20,\n",
       "         'care': 134,\n",
       "         'putting': 30,\n",
       "         'politics': 77,\n",
       "         'over': 579,\n",
       "         'unacceptable': 6,\n",
       "         'senate': 122,\n",
       "         'yes': 79,\n",
       "         'memorial': 28,\n",
       "         'day': 404,\n",
       "         'despite': 87,\n",
       "         'disgusting': 36,\n",
       "         'unwarranted': 2,\n",
       "         'witch': 48,\n",
       "         'hunt': 45,\n",
       "         'have': 2009,\n",
       "         'most': 280,\n",
       "         'successful': 94,\n",
       "         'first': 328,\n",
       "         'month': 40,\n",
       "         'administration': 112,\n",
       "         'history': 150,\n",
       "         'far': 192,\n",
       "         'sorry': 63,\n",
       "         'haters': 63,\n",
       "         'but': 912,\n",
       "         'way': 361,\n",
       "         'wow': 232,\n",
       "         'word': 59,\n",
       "         'seems': 31,\n",
       "         'coming': 149,\n",
       "         'obama': 1092,\n",
       "         'spied': 4,\n",
       "         'an': 758,\n",
       "         'embedded': 1,\n",
       "         'informant': 4,\n",
       "         'mccarthy': 7,\n",
       "         'says': 141,\n",
       "         'probably': 45,\n",
       "         'no': 919,\n",
       "         'doubt': 34,\n",
       "         'confidential': 5,\n",
       "         'bigger': 61,\n",
       "         'than': 568,\n",
       "         'watergate': 9,\n",
       "         'imbedded': 1,\n",
       "         'now': 977,\n",
       "         'year': 300,\n",
       "         'greatest': 100,\n",
       "         'american': 365,\n",
       "         'historyand': 1,\n",
       "         'still': 151,\n",
       "         'collusion': 63,\n",
       "         'obstruction': 20,\n",
       "         'only': 455,\n",
       "         'unable': 24,\n",
       "         'win': 381,\n",
       "         'election': 288,\n",
       "         'spending': 132,\n",
       "         'step': 29,\n",
       "         'closer': 15,\n",
       "         'leading': 50,\n",
       "         'brave': 50,\n",
       "         'men': 71,\n",
       "         'women': 127,\n",
       "         'she': 447,\n",
       "         'exceptionally': 2,\n",
       "         'qualified': 9,\n",
       "         'should': 867,\n",
       "         'confirm': 4,\n",
       "         'immediately': 82,\n",
       "         'need': 313,\n",
       "         'keep': 325,\n",
       "         'country': 661,\n",
       "         'safe': 93,\n",
       "         'hashtweethashconfirmgina': 1,\n",
       "         'voted': 50,\n",
       "         'against': 280,\n",
       "         'massive': 190,\n",
       "         'tax': 306,\n",
       "         'cut': 110,\n",
       "         'also': 213,\n",
       "         'weak': 124,\n",
       "         'borders': 68,\n",
       "         'crime': 96,\n",
       "         'sadly': 46,\n",
       "         'military': 212,\n",
       "         'vets': 53,\n",
       "         'mean': 30,\n",
       "         'nothing': 242,\n",
       "         'bobby': 17,\n",
       "         'jr': 10,\n",
       "         'lou': 9,\n",
       "         'barletta': 4,\n",
       "         'hashtweethashmaga': 97,\n",
       "         'senator': 112,\n",
       "         'pennsylvania': 82,\n",
       "         'opponent': 9,\n",
       "         'bob': 42,\n",
       "         'casey': 2,\n",
       "         'donothing': 1,\n",
       "         'shows': 96,\n",
       "         'up': 653,\n",
       "         'votes': 69,\n",
       "         'along': 54,\n",
       "         'nancy': 11,\n",
       "         'pelosi': 12,\n",
       "         'elizabeth': 31,\n",
       "         'warren': 32,\n",
       "         'loves': 51,\n",
       "         'sanctuary': 26,\n",
       "         'cities': 25,\n",
       "         'expensive': 28,\n",
       "         'healthcare': 127,\n",
       "         'today': 611,\n",
       "         'welcome': 84,\n",
       "         'mirziyoyev': 1,\n",
       "         'uzbekistan': 1,\n",
       "         'attweetatwhitehouse': 61,\n",
       "         'choicemission': 1,\n",
       "         'caregivers': 1,\n",
       "         'service': 66,\n",
       "         'organizations': 6,\n",
       "         'get': 896,\n",
       "         'choice': 61,\n",
       "         'passed': 47,\n",
       "         'seen': 70,\n",
       "         'demands': 13,\n",
       "         'few': 43,\n",
       "         'previous': 13,\n",
       "         'administrations': 10,\n",
       "         'poorly': 29,\n",
       "         'negotiating': 28,\n",
       "         'folding': 3,\n",
       "         'would': 650,\n",
       "         'people': 1207,\n",
       "         'believe': 248,\n",
       "         'meetings': 39,\n",
       "         'even': 423,\n",
       "         'started': 60,\n",
       "         'little': 125,\n",
       "         'give': 249,\n",
       "         'because': 312,\n",
       "         'given': 91,\n",
       "         'much': 540,\n",
       "         'years': 390,\n",
       "         'washington': 192,\n",
       "         'post': 109,\n",
       "         'cnn': 118,\n",
       "         'typically': 3,\n",
       "         'written': 28,\n",
       "         'false': 74,\n",
       "         'about': 910,\n",
       "         'negotiations': 32,\n",
       "         'happened': 56,\n",
       "         'zte': 3,\n",
       "         'except': 35,\n",
       "         'pertains': 4,\n",
       "         'larger': 20,\n",
       "         'deal': 413,\n",
       "         'losing': 82,\n",
       "         'hundreds': 27,\n",
       "         'billions': 66,\n",
       "         'dollars': 136,\n",
       "         'deb': 2,\n",
       "         'fischer': 2,\n",
       "         'nebraska': 8,\n",
       "         'what': 870,\n",
       "         'showed': 26,\n",
       "         'ballot': 11,\n",
       "         'box': 10,\n",
       "         'represent': 19,\n",
       "         'well': 272,\n",
       "         'represented': 9,\n",
       "         'many': 665,\n",
       "         'mine': 27,\n",
       "         'special': 156,\n",
       "         'guy': 211,\n",
       "         'help': 178,\n",
       "         'thank': 1341,\n",
       "         'examiner': 2,\n",
       "         'attweetatcortessteve': 1,\n",
       "         'article': 82,\n",
       "         'winning': 134,\n",
       "         'peaceofficersmemorialday': 2,\n",
       "         'important': 135,\n",
       "         'solemn': 3,\n",
       "         'occasions': 5,\n",
       "         'pay': 138,\n",
       "         'tribute': 10,\n",
       "         'law': 117,\n",
       "         'enforcement': 51,\n",
       "         'lives': 72,\n",
       "         'line': 51,\n",
       "         'duty': 16,\n",
       "         'made': 349,\n",
       "         'ultimate': 22,\n",
       "         'sacrifice': 12,\n",
       "         'could': 206,\n",
       "         'live': 224,\n",
       "         'safety': 51,\n",
       "         'peace': 31,\n",
       "         'police': 66,\n",
       "         'sure': 151,\n",
       "         'polls': 151,\n",
       "         'can': 753,\n",
       "         'unsourced': 1,\n",
       "         'from': 1131,\n",
       "         'together': 189,\n",
       "         'russian': 53,\n",
       "         'poll': 308,\n",
       "         'numbers': 163,\n",
       "         'may': 139,\n",
       "         'corrupt': 37,\n",
       "         'truly': 120,\n",
       "         'continuing': 13,\n",
       "         'making': 216,\n",
       "         'stay': 117,\n",
       "         'tuned': 24,\n",
       "         'lady': 22,\n",
       "         'leaving': 85,\n",
       "         'hospital': 14,\n",
       "         'days': 105,\n",
       "         'heading': 88,\n",
       "         'see': 484,\n",
       "         'melania': 88,\n",
       "         'procedure': 2,\n",
       "         'spirits': 2,\n",
       "         'wellwishers': 1,\n",
       "         'socalled': 27,\n",
       "         'leaks': 19,\n",
       "         'white': 143,\n",
       "         'exaggeration': 1,\n",
       "         'look': 298,\n",
       "         'possible': 64,\n",
       "         'said': 384,\n",
       "         'leakers': 8,\n",
       "         'traitors': 2,\n",
       "         'cowards': 5,\n",
       "         'find': 106,\n",
       "         'large': 55,\n",
       "         'chinese': 72,\n",
       "         'phone': 17,\n",
       "         'company': 50,\n",
       "         'buys': 4,\n",
       "         'percentage': 5,\n",
       "         'individual': 20,\n",
       "         'parts': 17,\n",
       "         'companies': 75,\n",
       "         'reflective': 2,\n",
       "         'personal': 15,\n",
       "         'relationship': 34,\n",
       "         'xi': 30,\n",
       "         'usembassyjerusalem': 1,\n",
       "         'israel': 57,\n",
       "         'embassy': 19,\n",
       "         'opening': 45,\n",
       "         'jerusalem': 10,\n",
       "         'covered': 18,\n",
       "         'attweetatfoxnews': 243,\n",
       "         'amp': 2187,\n",
       "         'pausesemicolon': 2246,\n",
       "         'attweetatfoxbusiness': 15,\n",
       "         'lead': 115,\n",
       "         'am': 715,\n",
       "         'eastern': 9,\n",
       "         'event': 93,\n",
       "         'already': 113,\n",
       "         'begun': 14,\n",
       "         'sad': 194,\n",
       "         'terror': 39,\n",
       "         'paris': 23,\n",
       "         'some': 233,\n",
       "         'point': 89,\n",
       "         'countries': 88,\n",
       "         'open': 137,\n",
       "         'eyes': 34,\n",
       "         'going': 616,\n",
       "         'kind': 46,\n",
       "         'sickness': 2,\n",
       "         'hatred': 11,\n",
       "         'compatible': 1,\n",
       "         'loving': 10,\n",
       "         'peaceful': 11,\n",
       "         'changes': 14,\n",
       "         'thought': 76,\n",
       "         'process': 29,\n",
       "         'remember': 230,\n",
       "         'how': 535,\n",
       "         'badly': 114,\n",
       "         'iran': 204,\n",
       "         'behaving': 3,\n",
       "         'trying': 103,\n",
       "         'take': 308,\n",
       "         'middle': 63,\n",
       "         'east': 45,\n",
       "         'whatever': 30,\n",
       "         'happen': 98,\n",
       "         'states': 206,\n",
       "         'past': 75,\n",
       "         'sided': 10,\n",
       "         'favor': 37,\n",
       "         'benefits': 15,\n",
       "         'both': 155,\n",
       "         'cool': 20,\n",
       "         'back': 515,\n",
       "         'business': 264,\n",
       "         'fast': 150,\n",
       "         'too': 227,\n",
       "         'jobs': 460,\n",
       "         'lost': 175,\n",
       "         'commerce': 9,\n",
       "         'department': 30,\n",
       "         'instructed': 5,\n",
       "         'happy': 285,\n",
       "         'august': 12,\n",
       "         'break': 13,\n",
       "         'go': 429,\n",
       "         'home': 113,\n",
       "         'wall': 161,\n",
       "         'border': 230,\n",
       "         'security': 181,\n",
       "         'included': 2,\n",
       "         'waiting': 30,\n",
       "         'approval': 48,\n",
       "         'almost': 85,\n",
       "         'nominations': 6,\n",
       "         'worst': 124,\n",
       "         'everything': 114,\n",
       "         'obstruct': 10,\n",
       "         'know': 343,\n",
       "         'budget': 95,\n",
       "         'since': 115,\n",
       "         'negotiated': 10,\n",
       "         'nuclear': 90,\n",
       "         'reachedjust': 1,\n",
       "         'another': 297,\n",
       "         'indicator': 3,\n",
       "         'lie': 67,\n",
       "         'anymore': 39,\n",
       "         'north': 180,\n",
       "         'korea': 129,\n",
       "         'announced': 69,\n",
       "         'dismantle': 3,\n",
       "         'test': 19,\n",
       "         'site': 27,\n",
       "         'ahead': 49,\n",
       "         'summit': 32,\n",
       "         'meeting': 165,\n",
       "         'june': 30,\n",
       "         'smart': 209,\n",
       "         'gracious': 3,\n",
       "         'gesture': 1,\n",
       "         'state': 267,\n",
       "         'antitrust': 1,\n",
       "         'division': 4,\n",
       "         'opposed': 15,\n",
       "         'atampt': 1,\n",
       "         'purchase': 23,\n",
       "         'warner': 7,\n",
       "         'currently': 14,\n",
       "         'ongoing': 3,\n",
       "         'trial': 25,\n",
       "         'disgrace': 26,\n",
       "         'reporting': 70,\n",
       "         'next': 205,\n",
       "         'moved': 10,\n",
       "         'deserve': 29,\n",
       "         'system': 93,\n",
       "         'takes': 44,\n",
       "         'advantage': 34,\n",
       "         'every': 215,\n",
       "         'ensure': 10,\n",
       "         'americans': 174,\n",
       "         'access': 25,\n",
       "         'quality': 18,\n",
       "         'affordable': 11,\n",
       "         'medication': 1,\n",
       "         'rest': 35,\n",
       "         'until': 118,\n",
       "         'launching': 3,\n",
       "         'sweeping': 3,\n",
       "         'action': 80,\n",
       "         'lower': 39,\n",
       "         'price': 62,\n",
       "         'prescription': 4,\n",
       "         'drugs': 35,\n",
       "         'tougher': 23,\n",
       "         'negotiation': 53,\n",
       "         'competition': 18,\n",
       "         'prices': 74,\n",
       "         'pharmacy': 1,\n",
       "         'counter': 5,\n",
       "         'indiana': 44,\n",
       "         'highly': 76,\n",
       "         'anticipated': 12,\n",
       "         'between': 67,\n",
       "         'kim': 22,\n",
       "         'jong': 14,\n",
       "         'myself': 27,\n",
       "         'singapore': 2,\n",
       "         'try': 62,\n",
       "         'moment': 19,\n",
       "         'five': 56,\n",
       "         'wanted': 57,\n",
       "         'leaders': 147,\n",
       "         'isis': 146,\n",
       "         'captured': 4,\n",
       "         'chuck': 48,\n",
       "         'schumer': 21,\n",
       "         'fought': 15,\n",
       "         'terminated': 8,\n",
       "         'same': 130,\n",
       "         'comey': 57,\n",
       "         'fired': 90,\n",
       "         'him': 419,\n",
       "         'behalf': 32,\n",
       "         'did': 370,\n",
       "         'spectacular': 39,\n",
       "         'close': 83,\n",
       "         'forward': 274,\n",
       "         'greeting': 1,\n",
       "         'hostages': 5,\n",
       "         'longer': 77,\n",
       "         'failing': 163,\n",
       "         'york': 199,\n",
       "         'times': 140,\n",
       "         'criticized': 10,\n",
       "         'secretary': 55,\n",
       "         'pompeo': 8,\n",
       "         'awol': 1,\n",
       "         'missing': 15,\n",
       "         'fact': 123,\n",
       "         'flying': 12,\n",
       "         'guests': 6,\n",
       "         'landing': 13,\n",
       "         'andrews': 3,\n",
       "         'air': 68,\n",
       "         'force': 60,\n",
       "         'base': 20,\n",
       "         'morning': 212,\n",
       "         'greet': 3,\n",
       "         'exciting': 68,\n",
       "         'pleased': 15,\n",
       "         'inform': 10,\n",
       "         'mike': 59,\n",
       "         'wonderful': 245,\n",
       "         'gentlemen': 1,\n",
       "         'seem': 29,\n",
       "         'health': 43,\n",
       "         'date': 13,\n",
       "         'set': 79,\n",
       "         'dewine': 1,\n",
       "         'ohio': 114,\n",
       "         'governor': 95,\n",
       "         'heavy': 11,\n",
       "         'focus': 162,\n",
       "         'socialist': 5,\n",
       "         'november': 62,\n",
       "         'failure': 72,\n",
       "         'candace': 1,\n",
       "         'owens': 2,\n",
       "         'turning': 22,\n",
       "         'usa': 125,\n",
       "         'having': 133,\n",
       "         'impact': 25,\n",
       "         'represents': 7,\n",
       "         'ever': 274,\n",
       "         'expanding': 23,\n",
       "         'group': 48,\n",
       "         'thinkers': 1,\n",
       "         'watch': 374,\n",
       "         'hear': 99,\n",
       "         'dialogue': 2,\n",
       "         'onso': 1,\n",
       "         'overtime': 6,\n",
       "         'reported': 39,\n",
       "         'tremendous': 108,\n",
       "         'success': 248,\n",
       "         'things': 193,\n",
       "         'else': 93,\n",
       "         'network': 31,\n",
       "         'negative': 71,\n",
       "         'away': 133,\n",
       "         'credentials': 4,\n",
       "         'republican': 196,\n",
       "         'party': 176,\n",
       "         'voter': 30,\n",
       "         'energy': 116,\n",
       "         'excitement': 14,\n",
       "         'candidates': 54,\n",
       "         'those': 147,\n",
       "         'chance': 92,\n",
       "         'sooo': 2,\n",
       "         'wanting': 16,\n",
       "         'end': 127,\n",
       "         'cuts': 125,\n",
       "         'raise': 65,\n",
       "         'taxes': 151,\n",
       "         'defective': 1,\n",
       "         'core': 24,\n",
       "         'short': 47,\n",
       "         'sponsor': 2,\n",
       "         'cusp': 2,\n",
       "         'acquiring': 3,\n",
       "         'dangerous': 59,\n",
       "         'weapons': 33,\n",
       "         'statement': 93,\n",
       "         'pausecolon': 1947,\n",
       "         'john': 174,\n",
       "         'kerry': 22,\n",
       "         'blew': 22,\n",
       "         'hurting': 14,\n",
       "         'your': 1146,\n",
       "         'speaking': 71,\n",
       "         'primary': 75,\n",
       "         'topics': 11,\n",
       "         'where': 231,\n",
       "         'relationships': 4,\n",
       "         'trust': 32,\n",
       "         'building': 93,\n",
       "         'respected': 52,\n",
       "         'nominee': 14,\n",
       "         'praised': 7,\n",
       "         'alway': 1,\n",
       "         'tough': 195,\n",
       "         'woman': 53,\n",
       "         'leader': 102,\n",
       "         'wherever': 4,\n",
       "         'gone': 60,\n",
       "         'wants': 176,\n",
       "         'bright': 9,\n",
       "         'glorious': 3,\n",
       "         'future': 91,\n",
       "         'announcing': 11,\n",
       "         'decision': 87,\n",
       "         'national': 346,\n",
       "         'drug': 38,\n",
       "         'hashtweethashtakebackday': 2,\n",
       "         'record': 313,\n",
       "         'broken': 37,\n",
       "         'nearly': 14,\n",
       "         'million': 185,\n",
       "         'pounds': 3,\n",
       "         'rx': 2,\n",
       "         'pills': 2,\n",
       "         'disposed': 5,\n",
       "         'fighting': 68,\n",
       "         'opioid': 8,\n",
       "         'epidemic': 11,\n",
       "         'possibly': 23,\n",
       "         'shadow': 4,\n",
       "         'diplomacy': 4,\n",
       "         'created': 34,\n",
       "         'mess': 72,\n",
       "         'wrongfully': 1,\n",
       "         'impacts': 1,\n",
       "         'midterm': 2,\n",
       "         'elections': 21,\n",
       "         'intended': 4,\n",
       "         'republicans': 242,\n",
       "         'better': 324,\n",
       "         'late': 65,\n",
       "         'lisa': 6,\n",
       "         'page': 27,\n",
       "         'hold': 40,\n",
       "         'emails': 72,\n",
       "         'shortest': 2,\n",
       "         'period': 18,\n",
       "         'lover': 3,\n",
       "         'peter': 14,\n",
       "         's': 62,\n",
       "         'attorney': 29,\n",
       "         'baker': 3,\n",
       "         'part': 68,\n",
       "         'probers': 1,\n",
       "         'getting': 212,\n",
       "         'caught': 50,\n",
       "         'total': 275,\n",
       "         'luck': 172,\n",
       "         'ric': 2,\n",
       "         'grenell': 1,\n",
       "         'ambassador': 14,\n",
       "         'germany': 21,\n",
       "         'talented': 26,\n",
       "         'revolt': 2,\n",
       "         'salena': 1,\n",
       "         'zito': 1,\n",
       "         'brad': 4,\n",
       "         'todd': 13,\n",
       "         'tell': 72,\n",
       "         'victory': 101,\n",
       "         'forgotten': 19,\n",
       "         'angry': 35,\n",
       "         'charge': 27,\n",
       "         'starting': 59,\n",
       "         'court': 72,\n",
       "         'actually': 61,\n",
       "         'protects': 3,\n",
       "         'injusticeand': 1,\n",
       "         'wait': 40,\n",
       "         'courts': 20,\n",
       "         'unrevealed': 1,\n",
       "         'interest': 60,\n",
       "         'rapidly': 28,\n",
       "         'credibility': 34,\n",
       "         'intelligence': 41,\n",
       "         'committee': 45,\n",
       "         'found': 40,\n",
       "         'coordination': 5,\n",
       "         'anything': 95,\n",
       "         'probe': 9,\n",
       "         'ok': 23,\n",
       "         'crimethere': 1,\n",
       "         'o': 19,\n",
       "         'called': 128,\n",
       "         'fire': 51,\n",
       "         'terrorists': 40,\n",
       "         'think': 524,\n",
       "         'these': 139,\n",
       "         'person': 124,\n",
       "         'want': 425,\n",
       "         'west': 80,\n",
       "         'problem': 130,\n",
       "         'don': 29,\n",
       "         'blankenship': 1,\n",
       "         'running': 91,\n",
       "         'general': 95,\n",
       "         'stateno': 1,\n",
       "         'alabama': 57,\n",
       "         'rep': 16,\n",
       "         'jenkins': 1,\n",
       "         'ag': 45,\n",
       "         'morrisey': 1,\n",
       "         'cleveland': 19,\n",
       "         'level': 47,\n",
       "         'delegation': 10,\n",
       "         'representatives': 19,\n",
       "         'determine': 5,\n",
       "         'results': 87,\n",
       "         'become': 85,\n",
       "         'spoiled': 2,\n",
       "         'wins': 52,\n",
       "         'returned': 20,\n",
       "         'beautiful': 165,\n",
       "         'dallas': 28,\n",
       "         'arena': 20,\n",
       "         'packed': 35,\n",
       "         'rafters': 2,\n",
       "         'fans': 41,\n",
       "         'supporters': 84,\n",
       "         'attweetatnra': 7,\n",
       "         'book': 180,\n",
       "         'businessman': 13,\n",
       "         'attweetatandypuzder': 1,\n",
       "         'known': 31,\n",
       "         'somebody': 23,\n",
       "         'knows': 80,\n",
       "         'capitalist': 2,\n",
       "         'comeback': 19,\n",
       "         'hit': 130,\n",
       "         'friends': 147,\n",
       "         'patriots': 18,\n",
       "         'fail': 35,\n",
       "         'protect': 82,\n",
       "         'amendment': 38,\n",
       "         'demand': 45,\n",
       "         'congress': 146,\n",
       "         'secure': 33,\n",
       "         'upcoming': 27,\n",
       "         'cr': 3,\n",
       "         'immigration': 146,\n",
       "         'liberals': 7,\n",
       "         'disarm': 3,\n",
       "         'lawabiding': 1,\n",
       "         'releasing': 15,\n",
       "         'criminal': 21,\n",
       "         'aliens': 9,\n",
       "         'savage': 3,\n",
       "         'onto': 10,\n",
       "         'streets': 8,\n",
       "         'politicians': 69,\n",
       "         'citizens': 27,\n",
       "         'here': 141,\n",
       "         'timeless': 4,\n",
       "         'values': 20,\n",
       "         'liberty': 26,\n",
       "         'gift': 29,\n",
       "         'creator': 7,\n",
       "         'government': 173,\n",
       "         'rule': 35,\n",
       "         'pride': 18,\n",
       "         'unemployment': 99,\n",
       "         'meantime': 6,\n",
       "         'soon': 227,\n",
       "         'nbc': 150,\n",
       "         'cite': 3,\n",
       "         'sources': 32,\n",
       "         'constantly': 24,\n",
       "         'others': 158,\n",
       "         'exist': 16,\n",
       "         'fabricated': 9,\n",
       "         'fiction': 12,\n",
       "         'former': 42,\n",
       "         'apprentice': 221,\n",
       "         'andy': 25,\n",
       "         'tonight': 471,\n",
       "         'pm': 276,\n",
       "         'receiving': 21,\n",
       "         'aid': 18,\n",
       "         'lowest': 35,\n",
       "         'southern': 32,\n",
       "         'siege': 4,\n",
       "         'change': 167,\n",
       "         'ineffective': 21,\n",
       "         'laws': 55,\n",
       "         'build': 112,\n",
       "         'mexico': 128,\n",
       "         'nationaldayofprayer': 1,\n",
       "         'spring': 20,\n",
       "         'marks': 6,\n",
       "         'phoenix': 26,\n",
       "         'crisis': 29,\n",
       "         'wont': 120,\n",
       "         ...})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_counts = collections.Counter( tokens )\n",
    "word_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the:   15056\n",
      "endperiod:   14523\n",
      "to:    9223\n",
      "a:    6911\n",
      "and:    6876\n",
      "tweetlink:    6742\n",
      "is:    6207\n",
      "of:    6100\n",
      "in:    5728\n",
      "on:    4075\n",
      "for:    4062\n",
      "i:    4061\n",
      "you:    3687\n",
      "be:    3470\n",
      "will:    3460\n",
      "endexclamation:    3153\n",
      "great:    3051\n",
      "that:    2463\n",
      "it:    2446\n",
      "at:    2413\n",
      "with:    2401\n",
      "are:    2397\n",
      "pausesemicolon:    2246\n",
      "amp:    2187\n",
      "have:    2009\n",
      "we:    1986\n",
      "our:    1955\n",
      "pausecolon:    1947\n",
      "my:    1816\n",
      "trump:    1777\n",
      "he:    1684\n",
      "by:    1651\n",
      "not:    1648\n",
      "was:    1489\n",
      "all:    1448\n",
      "has:    1370\n",
      "thank:    1341\n",
      "this:    1339\n",
      "just:    1334\n",
      "they:    1330\n",
      "smartquoteopen:    1300\n",
      "smartquoteclose:    1283\n",
      "people:    1207\n",
      "me:    1173\n",
      "so:    1162\n",
      "your:    1146\n",
      "from:    1131\n",
      "obama:    1092\n",
      "us:    1091\n",
      "as:    1074\n",
      "new:    1074\n",
      "very:    1062\n",
      "thanks:    1048\n",
      "who:    1028\n",
      "his:    1015\n",
      "now:     977\n",
      "do:     936\n",
      "out:     935\n",
      "no:     919\n",
      "but:     912\n",
      "about:     910\n",
      "get:     896\n",
      "what:     870\n",
      "should:     867\n",
      "big:     856\n",
      "if:     842\n",
      "more:     812\n",
      "america:     807\n",
      "make:     788\n",
      "time:     783\n",
      "endquestion:     761\n",
      "good:     758\n",
      "an:     758\n",
      "can:     753\n",
      "like:     734\n",
      "via:     728\n",
      "am:     715\n",
      "one:     670\n",
      "never:     665\n",
      "many:     665\n",
      "president:     663\n",
      "country:     661\n",
      "up:     653\n",
      "would:     650\n",
      "again:     635\n",
      "or:     627\n",
      "when:     625\n",
      "going:     616\n",
      "today:     611\n",
      "donald:     600\n",
      "their:     598\n",
      "over:     579\n",
      "hillary:     578\n",
      "must:     572\n",
      "than:     568\n",
      "why:     555\n",
      "there:     544\n",
      "much:     540\n",
      "last:     536\n",
      "how:     535\n"
     ]
    }
   ],
   "source": [
    "for word, count in word_counts.most_common( 100 ):\n",
    "    print( '%s: %7d' % ( word, count ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.05.22 09:33\n",
      "Total Sequences: 399086\n",
      "2018.05.22 09:33\n",
      "Time to process: [0.2942061424255371] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "\n",
    "# organize into sequences of tokens\n",
    "sequence_len = 50 + 1\n",
    "sequences = list()\n",
    "\n",
    "for i in range( sequence_len, len( tokens ) ):\n",
    "    \n",
    "    # select sequence of tokens\n",
    "    seq = tokens[ i - sequence_len:i ]\n",
    "    \n",
    "    # convert into a line\n",
    "    line = ' '.join( seq )\n",
    "    \n",
    "    # store\n",
    "    sequences.append( line )\n",
    "    \n",
    "print( 'Total Sequences: %d' % len( sequences ) )\n",
    "print_time( start_time, get_time() )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save tokens to file, one dialog per line\n",
    "def save_doc( lines, filename ):\n",
    "    \n",
    "    data = '\\n'.join( lines )\n",
    "    file = open( filename, 'w' )\n",
    "    file.write( data )\n",
    "    file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save sequences to file\n",
    "out_filename = \"../texts/trump-tweets-sequences-01.txt\"\n",
    "save_doc( sequences, out_filename )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['just met with un secretarygeneral antónio guterres who is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job',\n",
       " 'met with un secretarygeneral antónio guterres who is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job endexclamation',\n",
       " 'with un secretarygeneral antónio guterres who is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job endexclamation tweetlink',\n",
       " 'un secretarygeneral antónio guterres who is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job endexclamation tweetlink america',\n",
       " 'secretarygeneral antónio guterres who is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job endexclamation tweetlink america is',\n",
       " 'antónio guterres who is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job endexclamation tweetlink america is a',\n",
       " 'guterres who is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job endexclamation tweetlink america is a nation',\n",
       " 'who is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job endexclamation tweetlink america is a nation that',\n",
       " 'is working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job endexclamation tweetlink america is a nation that believes',\n",
       " 'working hard to smartquoteopen make the united nations great again smartquoteclose when the un does more to solve conflicts around the world it means the us endperiod has less to do and we save money endperiod attweetatnikkihaley is doing a fantastic job endexclamation tweetlink america is a nation that believes in']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "in_filename = \"../texts/trump-tweets-sequences-01.txt\"\n",
    "doc = load_doc( in_filename )\n",
    "lines = doc.split( '\\n' )\n",
    "lines[ 0:10 ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert Words to Index Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# integer encode sequences of words\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts( lines )\n",
    "sequences = tokenizer.texts_to_sequences( lines )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'w'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print( sequences[ 0 ][ 0 ] )\n",
    "line[ 0 ][ 0 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# elegant! https://stackoverflow.com/questions/41971587/how-to-convert-predicted-sequence-back-to-text-in-keras\n",
    "sequences_to_texts = dict( map( reversed, tokenizer.word_index.items() ) ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'just'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequences_to_texts[ 39 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "399086\n"
     ]
    }
   ],
   "source": [
    "print( len( sequences[ 0 ] ) == sequence_len )\n",
    "print( len( sequences ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21148\n",
      "<class 'dict'>\n",
      "39\n"
     ]
    }
   ],
   "source": [
    "print( len( tokenizer.word_index ) )\n",
    "print( type( tokenizer.word_index ) )\n",
    "print( tokenizer.word_index[ \"just\" ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21149"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# vocabulary size\n",
    "vocab_size = len( tokenizer.word_index ) + 1\n",
    "vocab_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# separate into input and output: for now it's 50 words input and 1 word output\n",
    "sequences = np.array( sequences )\n",
    "X = sequences[ :,:-1 ] # all rows, from word 0 up to, but not including, the last word\n",
    "y = sequences[ :,-1 ]  # all rows, last word only\n",
    "y = to_categorical( y, num_classes=vocab_size )\n",
    "seq_length = X.shape[ 1 ]\n",
    "seq_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and Filter GloVe Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.05.22 09:35\n",
      "\n",
      "Loaded 13766 word vectors.\n",
      "\n",
      "Words not found 7382.\n",
      "2018.05.22 09:37\n",
      "Time to process: [109.39622044563293] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "\n",
    "# load the whole embedding into memory\n",
    "embeddings_index = dict()\n",
    "embeddings_dimension = 300 #must be 50, 100, 200, 300\n",
    "glove = open( \"../glove/glove.6B.\" + str( embeddings_dimension ) + \"d.txt\" )\n",
    "\n",
    "for line in glove:\n",
    "    \n",
    "    values = line.split()\n",
    "    # 1st string is word...\n",
    "    word = values[ 0 ]\n",
    "    \n",
    "    if word in tokens_unique:\n",
    "        \n",
    "        # ...the rest are coefficients\n",
    "        coefs = np.asarray( values[ 1: ], dtype='float32' )\n",
    "        embeddings_index[ word ] = coefs\n",
    "        #print( \"*\", end=\"\" )\n",
    "    \n",
    "glove.close()\n",
    "print( '\\nLoaded %s word vectors.' % len( embeddings_index ) )\n",
    "print( '\\nWords not found %d.' % ( len( tokenizer.word_index ) - len( embeddings_index ) ) )\n",
    "print_time( start_time, get_time() )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transform into Matrix That Maps Coefs by Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7382\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['endperiod',\n",
       " 'tweetlink',\n",
       " 'endexclamation',\n",
       " 'pausesemicolon',\n",
       " 'pausecolon',\n",
       " 'smartquoteopen',\n",
       " 'smartquoteclose',\n",
       " 'endquestion',\n",
       " 'attweetatbarackobama',\n",
       " 'attweetatfoxnews',\n",
       " 'attweetatmittromney',\n",
       " 'attweetatfoxandfriends',\n",
       " 'hashtweethashmakeamericagreatagain',\n",
       " 'attweetatapprenticenbc',\n",
       " 'attweetatcnn',\n",
       " 'hashtweethashcelebapprentice',\n",
       " 'attweetatbarackobamas',\n",
       " 'attweetatnytimes',\n",
       " 'attweetatcelebapprentice',\n",
       " 'hashtweethashmaga',\n",
       " 'hashtweethashtimetogettough',\n",
       " 'attweetatnbc',\n",
       " 'twitlonger',\n",
       " 'attweetatgretawire',\n",
       " 'attweetativankatrump',\n",
       " 'attweetatnewsmaxmedia',\n",
       " 'realdonaldtrump',\n",
       " 'attweetatseanhannity',\n",
       " 'attweetatbillmaher',\n",
       " 'attweetatmacys',\n",
       " 'attweetatrealdonaldtrump',\n",
       " 'attweetatoreillyfactor',\n",
       " 'hashtweethashamericafirst',\n",
       " 'barackobama',\n",
       " 'attweetatwhitehouse',\n",
       " 'hashtweethashdraintheswamp',\n",
       " 'attweetattrumpdoral',\n",
       " 'arod',\n",
       " 'attweetatyankees',\n",
       " 'daca',\n",
       " 'makeamericagreatagain',\n",
       " 'hashtweethashtrumpvlog',\n",
       " 'attweetatgop',\n",
       " 'attweetatbreitbartnews',\n",
       " 'attweetaterictrump',\n",
       " 'maralago',\n",
       " 'hashtweethashvotetrump',\n",
       " 'attweetattrumptowerny',\n",
       " 'hashtweethashimwithyou',\n",
       " 'attweetatpiersmorgan',\n",
       " 'attweetatsquawkcnbc',\n",
       " 'attweetatkarlrove',\n",
       " 'hashtweethashdebate',\n",
       " 'attweetattrumpchicago',\n",
       " 'attweetatagschneiderman',\n",
       " 'attweetatalexsalmond',\n",
       " 'mittromney',\n",
       " 'attweetatmegynkelly',\n",
       " 'attweetatabc',\n",
       " 'hashtweethashbigleaguetruth',\n",
       " 'attweetatmissuniverse',\n",
       " 'tonights',\n",
       " 'attweetatmorningjoe',\n",
       " 'attweetatrosie',\n",
       " 'attweetatmittromneys',\n",
       " 'youve',\n",
       " 'attweetattodayshow',\n",
       " 'attweetatwsj',\n",
       " 'hashtweethashfitn',\n",
       " 'attweetatwashingtonpost',\n",
       " 'attweetatdonaldjtrumpjr',\n",
       " 'attweetatamspec',\n",
       " 'icymi',\n",
       " 'attweetattrumpnewyork',\n",
       " 'hashtweethashgopdebate',\n",
       " 'attweetatomarosa',\n",
       " 'attweetattrumpsoho',\n",
       " 'attweetatgreta',\n",
       " 'attweetatpolitico',\n",
       " 'attweetatvanityfair',\n",
       " 'attweetatmelaniatrump',\n",
       " 'attweetatjebbush',\n",
       " 'attweetatcadillacchamp',\n",
       " 'trumpvlog',\n",
       " 'shouldnt',\n",
       " 'attweetatlateshow',\n",
       " 'alexsalmond',\n",
       " 'attweetatrnc',\n",
       " 'attweetatflotus',\n",
       " 'attweetatchucktodd',\n",
       " 'clientthe',\n",
       " 'attweetatap',\n",
       " 'attweetatisrael',\n",
       " 'attweetatthegarybusey',\n",
       " 'attweetatwwe',\n",
       " 'attweetatnydailynews',\n",
       " 'attweetatcpacnews',\n",
       " 'celebapprentice',\n",
       " 'tahmooressi',\n",
       " 'attweetatdoralresort',\n",
       " 'attweetathuffingtonpost',\n",
       " 'hashtweethashoscars',\n",
       " 'attweetatmissusa',\n",
       " 'attweetatnbcnews',\n",
       " 'hashtweethashobamacare',\n",
       " 'crookedhillary',\n",
       " 'attweetattrumpturnberry',\n",
       " 'attweetatmcuban',\n",
       " 'attweetatfundanything',\n",
       " 'attweetatnetanyahu',\n",
       " 'attweetatmsnbc',\n",
       " 'hillaryclinton',\n",
       " 'hashtweethashvpdebate',\n",
       " 'oreilly',\n",
       " 'attweetatricksantorum',\n",
       " 'attweetatteamcavuto',\n",
       " 'clientsmartquoteopen',\n",
       " 'attweetatjoanrivers',\n",
       " 'hashtweethashwwehof',\n",
       " 'attweetatnypost',\n",
       " 'ocare',\n",
       " 'attweetatmeetthepress',\n",
       " 'attweetatmikepence',\n",
       " 'hashtweethashcrookedhillary',\n",
       " 'attweetatlawrence',\n",
       " 'attweetattraceadkins',\n",
       " 'dannyzuker',\n",
       " 'newsmaxiontv',\n",
       " 'attweetatvp',\n",
       " 'attweetatpennjillette',\n",
       " 'attweetatdmregister',\n",
       " 'attweetattrumpwaikiki',\n",
       " 'clienti',\n",
       " 'attweetatfoxbusiness',\n",
       " 'foxandfriends',\n",
       " 'hashtweethashfakenews',\n",
       " 'hashtweethashicymi',\n",
       " 'smartquoteclosetweetlink',\n",
       " 'youmakeamericagreatagain',\n",
       " 'attweetatcnbc',\n",
       " 'hashtweethashiacaucus',\n",
       " 'attweetatjacknicklaus',\n",
       " 'attweetatunionleader',\n",
       " 'attweetatbarbarajwalters',\n",
       " 'attweetatmarklevinshow',\n",
       " 'attweetattrumplasvegas',\n",
       " 'attweetattrumpgolfla',\n",
       " 'attweetatdannyzuker',\n",
       " 'hashtweethashtrumptuesday',\n",
       " 'attweetatpaulryanvp',\n",
       " 'attweetatingrahamangle',\n",
       " 'hashtweethashmagatweetlink',\n",
       " 'attweetatwashtimes',\n",
       " 'selffunding',\n",
       " 'hashtweethashtrump',\n",
       " 'attweetatpressjournal',\n",
       " 'attweetatbretmichaels',\n",
       " 'attweetatlordsugar',\n",
       " 'againtweetlink',\n",
       " 'attweetatflgovscott',\n",
       " 'attweetatcbsnews',\n",
       " 'ivankatrump',\n",
       " 'attweetatlibertyu',\n",
       " 'hashtweethashtrumptrain',\n",
       " 'attweetatjimmyfallon',\n",
       " 'karlrove',\n",
       " 'attweetatgolfchannel',\n",
       " 'attweetatgstephanopoulos',\n",
       " 'attweetattime',\n",
       " 'attweetatforbes',\n",
       " 'attweetatdennisrodman',\n",
       " 'attweetattrumpferrypoint',\n",
       " 'attweetatsrqrepublicans',\n",
       " 'hashtweethashmissuniverse',\n",
       " 'attweetattrumpcharlotte',\n",
       " 'attweetattheviewtv',\n",
       " 'mcuban',\n",
       " 'attweetatnymag',\n",
       " 'antitrump',\n",
       " 'djt',\n",
       " 'onesided',\n",
       " 'attweetatthebrodyfile',\n",
       " 'attweetatbretbaier',\n",
       " 'attweetatstjude',\n",
       " 'attweetatsarahpalinusa',\n",
       " 'attweetatkrauthammer',\n",
       " 'attweetathardballchris',\n",
       " 'hashtweethashasktrump',\n",
       " 'attweetatnfl',\n",
       " 'attweetattrumptoronto',\n",
       " 'attweetatstevekingia',\n",
       " 'attweetattherealmarilu',\n",
       " 'clientmy',\n",
       " 'commanderinchief',\n",
       " 'attweetathillaryclinton',\n",
       " 'attweetatmorningmika',\n",
       " 'attweetatericbolling',\n",
       " 'hashtweethashrncincle',\n",
       " 'attweetattrumpscotland',\n",
       " 'oreillyfactor',\n",
       " 'attweetatgma',\n",
       " 'attweetatdrudgereport',\n",
       " 'attweetatnro',\n",
       " 'jebbush',\n",
       " 'attweetatgatewaypundit',\n",
       " 'attweetatariannahuff',\n",
       " 'attweetatdailycaller',\n",
       " 'attweetatthehill',\n",
       " 'attweetattrumppanama',\n",
       " 'attweetatjackwelch',\n",
       " 'attweetatcitizensunited',\n",
       " 'attweetatliljon',\n",
       " 'sotu',\n",
       " 'attweetatthefamilyleader',\n",
       " 'attweetatmacmiller',\n",
       " 'attweetatjoebiden',\n",
       " 'attweetatemmanuelmacron',\n",
       " 'attweetatsenatemajldr',\n",
       " 'hashtweethashlesm',\n",
       " 'hashtweethashtaxreform',\n",
       " 'attweetatpatriots',\n",
       " 'attweetatfoxnewssunday',\n",
       " 'attweetatncgop',\n",
       " 'megynkelly',\n",
       " 'attweetatnewtgingrich',\n",
       " 'odonnell',\n",
       " 'attweetatandersoncooper',\n",
       " 'attweetathowardstern',\n",
       " 'attweetatcnnpolitics',\n",
       " 'attweetatforbesinspector',\n",
       " 'attweetaterictrumpfdn',\n",
       " 'attweetattrumpireland',\n",
       " 'attweetatmediaite',\n",
       " 'mattginellagc',\n",
       " 'attweetatjohnrich',\n",
       " 'blackdog',\n",
       " 'attweetathannityshow',\n",
       " 'antbaxter',\n",
       " 'attweetatdnc',\n",
       " 'attweetatjudgejeanine',\n",
       " 'attweetatbillygraham',\n",
       " 'attweetatmarcorubio',\n",
       " 'attweetatisraelipm',\n",
       " 'hashtweethashimwithyoutweetlink',\n",
       " 'brexit',\n",
       " 'attweetatusatoday',\n",
       " 'attweetatjaketapper',\n",
       " 'melaniatrump',\n",
       " 'attweetatglennbeck',\n",
       " 'attweetatdavidaxelrod',\n",
       " 'worldclass',\n",
       " 'androidi',\n",
       " 'trumpadvice',\n",
       " 'attweetattrumpcollection',\n",
       " 'nycs',\n",
       " 'attweetatespn',\n",
       " 'attweetattigerwoods',\n",
       " 'attweetatautismspeaks',\n",
       " 'piersmorgan',\n",
       " 'attweetatvincemcmahon',\n",
       " 'attweetatmacyscom',\n",
       " 'attweetattherealkiyosaki',\n",
       " 'attweetatjayleno',\n",
       " 'thegarybusey',\n",
       " 'attweetatloudobbs',\n",
       " 'flotus',\n",
       " 'attweetatdcexaminer',\n",
       " 'attweetatusnavy',\n",
       " 'progrowth',\n",
       " 'hashtweethashhurricaneharvey',\n",
       " 'attweetatfacethenation',\n",
       " 'secy',\n",
       " 'hashtweethashmakeamericagreatagaintweetlink',\n",
       " 'jebs',\n",
       " 'attweetatthisweekabc',\n",
       " 'hashtweethashsupertuesday',\n",
       " 'attweetattedcruz',\n",
       " 'vanityfair',\n",
       " 'attweetatgovchristie',\n",
       " 'attweetatmailonline',\n",
       " 'hashtweethashdemdebate',\n",
       " 'attweetatewerickson',\n",
       " 'hashtweethashncgopcon',\n",
       " 'clientvia',\n",
       " 'notthatactor',\n",
       " 'attweetatwrestlemania',\n",
       " 'trumpdoral',\n",
       " 'hashtweethashcelebrityapprentice',\n",
       " 'fivestar',\n",
       " 'attweetatnewyorkobserver',\n",
       " 'lordsugar',\n",
       " 'attweetatextratv',\n",
       " 'attweetattrumpwinery',\n",
       " 'attweetattrumpto',\n",
       " 'attweetatmlb',\n",
       " 'hashtweethashmissusa',\n",
       " 'attweetatbuzzfeed',\n",
       " 'attweetatnyjets',\n",
       " 'trumpvine',\n",
       " 'beaumontanthony',\n",
       " 'attweetatqvc',\n",
       " 'johnboehner',\n",
       " 'betathe',\n",
       " 'barackobamas',\n",
       " 'attweetatobamacare',\n",
       " 'attweetatjonhuntsman',\n",
       " 'attweetatnra',\n",
       " 'attweetatdhsgov',\n",
       " 'attweetatspeakerryan',\n",
       " 'hashtweethashs',\n",
       " 'hashtweethashusa',\n",
       " 'oclock',\n",
       " 'attweetatnbcnightlynews',\n",
       " 'attweetatrealbencarson',\n",
       " 'hashtweethashperiscope',\n",
       " 'hashtweethashdraintheswamptweetlink',\n",
       " 'werent',\n",
       " 'attweetatdonlemon',\n",
       " 'hashtweethashdebatenight',\n",
       " 'attweetatfallontonight',\n",
       " 'africanamerican',\n",
       " 'attweetatjoniernst',\n",
       " 'traceadkins',\n",
       " 'flashbackfriday',\n",
       " 'hashtweethashinprimary',\n",
       " 'attweetattrumpgolf',\n",
       " 'attweetatjdickerson',\n",
       " 'hashtweethashvotetrumpsc',\n",
       " 'attweetatcher',\n",
       " 'attweetatlimbaugh',\n",
       " 'attweetatfitsnews',\n",
       " 'billmaher',\n",
       " 'attweetatfoxnewsinsider',\n",
       " 'attweetatmiamiherald',\n",
       " 'attweetatgolfmagazine',\n",
       " 'attweetatleezeldin',\n",
       " 'attweetatperduesenate',\n",
       " 'hashtweethashfreeourmarine',\n",
       " 'attweetatpgachampionship',\n",
       " 'attweetatbbcnews',\n",
       " 'attweetatdavidcameron',\n",
       " 'arseniohall',\n",
       " 'attweetatoprah',\n",
       " 'attweetattheblaze',\n",
       " 'vattenfallgroup',\n",
       " 'pennjillette',\n",
       " 'hashtweethashtheartofthedeal',\n",
       " 'attweetatvattenfallgroup',\n",
       " 'dennisrodman',\n",
       " 'attweetatlatoyajackson',\n",
       " 'attweetatkatherinewebb',\n",
       " 'attweetatgiulianarancic',\n",
       " 'attweetatcabinet',\n",
       " 'seanhannity',\n",
       " 'theyve',\n",
       " 'attweetatmariabartiromo',\n",
       " 'africanamericans',\n",
       " 'attweetatrandpaul',\n",
       " 'attweetatbpolitics',\n",
       " 'attweetatfema',\n",
       " 'attweetatredcross',\n",
       " 'attweetatsuperbowl',\n",
       " 'attweetatcbnnews',\n",
       " 'presidentelect',\n",
       " 'attweetatoann',\n",
       " 'attweetatnbcsnl',\n",
       " 'hashtweethashpotus',\n",
       " 'attweetatkimguilfoyle',\n",
       " 'attweetatjoenbc',\n",
       " 'hashtweethashvotetrumptweetlink',\n",
       " 'ewerickson',\n",
       " 'hashtweethashnhprimary',\n",
       " 'morningjoe',\n",
       " 'tedcruz',\n",
       " 'frankluntz',\n",
       " 'erictrump',\n",
       " 'schlonged',\n",
       " 'attweetatbusinessinsider',\n",
       " 'attweetatredstate',\n",
       " 'keynoting',\n",
       " 'trumpgolfla',\n",
       " 'attweetatsixteenchicago',\n",
       " 'everyones',\n",
       " 'clientif',\n",
       " 'clientcongratulations',\n",
       " 'attweetatgolfweekmag',\n",
       " 'attweetatjenniferjjacobs',\n",
       " 'attweetateonline',\n",
       " 'apprenticenbc',\n",
       " 'cohosting',\n",
       " 'attweetatbuffalobills',\n",
       " 'attweetatafphq',\n",
       " 'attweetataberdeencc',\n",
       " 'danscavino',\n",
       " 'attweetathbo',\n",
       " 'nymag',\n",
       " 'hashtweethashmakedclisten',\n",
       " 'attweetattimtebow',\n",
       " 'attweetataol',\n",
       " 'sexter',\n",
       " 'attweetatanthonyweiner',\n",
       " 'hashtweethashfundanything',\n",
       " 'attweetatlisarinna',\n",
       " 'attweetativankatrumps',\n",
       " 'michellemalkin',\n",
       " 'attweetatgallupnews',\n",
       " 'attweetatrepweiner',\n",
       " 'attweetatdeadspin',\n",
       " 'attweetatlisalampanelli',\n",
       " 'clientsweepstweet',\n",
       " 'clienthe',\n",
       " 'attweetatjustintrudeau',\n",
       " 'trumprussia',\n",
       " 'hashtweethashnoko',\n",
       " 'pelosischumer',\n",
       " 'attweetatreince',\n",
       " 'wthe',\n",
       " 'hashtweethashunga',\n",
       " 'deplorables',\n",
       " 'hashtweethashfake',\n",
       " 'repub',\n",
       " 'hashtweethashisis',\n",
       " 'attweetatfaithandfreedom',\n",
       " 'firstever',\n",
       " 'attweetatscottwalker',\n",
       " 'washingtonpost',\n",
       " 'obamacarefail',\n",
       " 'crookeds',\n",
       " 'hashtweethashamericafirsttweetlink',\n",
       " 'attweetatsentedcruz',\n",
       " 'donaldtrump',\n",
       " 'rowanne',\n",
       " 'attweetatspecialreport',\n",
       " 'hashtweethashnyprimary',\n",
       " 'hashtweethashwiprimary',\n",
       " 'hashtweethashcaucusfortrump',\n",
       " 'attweetatcolbertlateshow',\n",
       " 'hashtweethashvotetrumpnh',\n",
       " 'donaldjtrumpjr',\n",
       " 'chucktodd',\n",
       " 'attweetatrichlowry',\n",
       " 'postdebate',\n",
       " 'carlyfiorina',\n",
       " 'attweetatoutfrontcnn',\n",
       " 'attweetatrushlimbaugh',\n",
       " 'governorperry',\n",
       " 'ericbolling',\n",
       " 'attweetatturnberrybuzz',\n",
       " 'attweetatunivision',\n",
       " 'attweetattripadvisor',\n",
       " 'attweetattrumpnationalny',\n",
       " 'attweetathuffpostpol',\n",
       " 'attweetatpgatour',\n",
       " 'attweetattuohy',\n",
       " 'attweetatbostonherald',\n",
       " 'attweetatdavidbossie',\n",
       " 'attweetatjonahnro',\n",
       " 'attweetatcspan',\n",
       " 'attweetatfoxsports',\n",
       " 'bretbaier',\n",
       " 'attweetattrumpgolfdc',\n",
       " 'welldeserved',\n",
       " 'attweetatowentew',\n",
       " 'attweetattrumprink',\n",
       " 'celebrityapprentice',\n",
       " 'attweetatalgemeiner',\n",
       " 'attweetatletterman',\n",
       " 'attweetatemilymiller',\n",
       " 'hashtweethashmidastouch',\n",
       " 'attweetatteammitch',\n",
       " 'pistorious',\n",
       " 'katyperry',\n",
       " 'vescio',\n",
       " 'attweetatcnnmoney',\n",
       " 'attweetatmikebloomberg',\n",
       " 'attweetatsharkgregnorman',\n",
       " 'attweetatthedailyshow',\n",
       " 'attweetatarseniohall',\n",
       " 'attweetataberdeenshire',\n",
       " 'ivankas',\n",
       " 'obamacares',\n",
       " 'attweetatkatieshow',\n",
       " 'attweetatcondenastcorp',\n",
       " 'attweetattelemundo',\n",
       " 'attweetatgoangelo',\n",
       " 'attweetatamandatmiller',\n",
       " 'attweetatscotsmandotcom',\n",
       " 'hardballchris',\n",
       " 'attweetatdeesnider',\n",
       " 'barbarajwalters',\n",
       " 'attweetatlancearmstrong',\n",
       " 'attweetatmarleematlin',\n",
       " 'clientcelebrityapprentice',\n",
       " 'attweetatamericanowradio',\n",
       " 'newswashington',\n",
       " 'attweetatastros',\n",
       " 'attweetattuckercarlson',\n",
       " 'meritbased',\n",
       " 'sampp',\n",
       " 'wapo',\n",
       " 'attweetatgeraldorivera',\n",
       " 'attweetatricardorossello',\n",
       " 'attweetatusga',\n",
       " 'hashtweethashuswomensopen',\n",
       " 'attweetatgovwalker',\n",
       " 'attweetatcbs',\n",
       " 'hashtweethashsuperbowl',\n",
       " 'polltrump',\n",
       " 'hashtweethashdebates',\n",
       " 'hashtweethashmagatickets',\n",
       " 'attweetatjerryjrfalwell',\n",
       " 'metweetlink',\n",
       " 'imwithyou',\n",
       " 'havnt',\n",
       " 'americafirst',\n",
       " 'endofmonth',\n",
       " 'hashtweethashtbt',\n",
       " 'melanias',\n",
       " 'attweetatserenawilliams',\n",
       " 'attweetatjasondovesq',\n",
       " 'attweetatbillkristol',\n",
       " 'thehill',\n",
       " 'attweetatlindseygrahamsc',\n",
       " 'hashtweethashscprimary',\n",
       " 'attweetatstephenathome',\n",
       " 'hashtweethashnhpolitics',\n",
       " 'bobvanderplaats',\n",
       " 'pollthank',\n",
       " 'thebrodyfile',\n",
       " 'missuniverse',\n",
       " 'attweetatamazon',\n",
       " 'attweetatreuters',\n",
       " 'attweetaterinburnett',\n",
       " 'hashtweethashtrumptower',\n",
       " 'attweetattwitternyc',\n",
       " 'hashtweethashcnndebate',\n",
       " 'rushlimbaugh',\n",
       " 'attweetatschwarzenegger',\n",
       " 'brandenroderick',\n",
       " 'attweetatfacebook',\n",
       " 'attweetatdallasmavs',\n",
       " 'breitbartnews',\n",
       " 'genesimmons',\n",
       " 'attweetatmets',\n",
       " 'attweetatclareoc',\n",
       " 'androidthe',\n",
       " 'clientthere',\n",
       " 'clientour',\n",
       " 'hashtweethashwhcd',\n",
       " 'attweetatnhgop',\n",
       " 'tmobile',\n",
       " 'attweetatjoshmcelveen',\n",
       " 'attweetatjeangeorges',\n",
       " 'attweetatdloesch',\n",
       " 'attweetatmelaniebatley',\n",
       " 'davidaxelrod',\n",
       " 'attweetatwandacarruthers',\n",
       " 'hashtweethashaskthedonald',\n",
       " 'attweetatibdeditorials',\n",
       " 'attweetatlandexpo',\n",
       " 'attweetatpeoplescompany',\n",
       " 'attweetatrobbreport',\n",
       " 'attweetatshawnjohnson',\n",
       " 'attweetatthomasaroberts',\n",
       " 'attweetattheeconomicclub',\n",
       " 'attweetatdamacofficial',\n",
       " 'attweetatbillcassidy',\n",
       " 'hashtweethashbluemonster',\n",
       " 'attweetatjeangeorgesnyc',\n",
       " 'attweetatsenscottbrown',\n",
       " 'flgovscott',\n",
       " 'attweetatthomtillis',\n",
       " 'agschneiderman',\n",
       " 'attweetatstevedeaceshow',\n",
       " 'attweetatcahillforag',\n",
       " 'attweetatbriarcliffmanor',\n",
       " 'mcilroyrory',\n",
       " 'attweetatboonepickens',\n",
       " 'attweetattheopen',\n",
       " 'fivediamond',\n",
       " 'attweetatchicagotribune',\n",
       " 'nypost',\n",
       " 'attweetatbiggovt',\n",
       " 'attweetatseniorpgachamp',\n",
       " 'cliententrepreneurs',\n",
       " 'tigerwoods',\n",
       " 'attweetattmz',\n",
       " 'attweetatbillclinton',\n",
       " 'attweetatlodgeatdoonbeg',\n",
       " 'attweetateminofficial',\n",
       " 'dorals',\n",
       " 'presobama',\n",
       " 'attweetatsaintanselm',\n",
       " 'jcope',\n",
       " 'attweetatehasselbeck',\n",
       " 'latoyajackson',\n",
       " 'gretawire',\n",
       " 'attweetatbillrancic',\n",
       " 'attweetatbw',\n",
       " 'winwin',\n",
       " 'rampd',\n",
       " 'attweetatralphreeds',\n",
       " 'lisarinna',\n",
       " 'attweetatboeing',\n",
       " 'attweetatbrandenroderick',\n",
       " 'attweetatsalon',\n",
       " 'erolyalim',\n",
       " 'attweetatsternshow',\n",
       " 'clientin',\n",
       " 'clientwhen',\n",
       " 'attweetatbravoandy',\n",
       " 'attweetatjohnboehner',\n",
       " 'hashtweethashsandy',\n",
       " 'attweetatisraels',\n",
       " 'attweetatjeep',\n",
       " 'attweetatanndromney',\n",
       " 'attweetatmedvedevrussiae',\n",
       " 'lancearmstrong',\n",
       " 'attweetatarsenioofficial',\n",
       " 'attweetatrasmussenpoll',\n",
       " 'attweetatwolfblitzercnn',\n",
       " 'attweetatteresagiudice',\n",
       " 'ricksantorum',\n",
       " 'betawe',\n",
       " 'timetogettough',\n",
       " 'tweetdeckmy',\n",
       " 'attweetatthehermancain',\n",
       " 'hashtweethashtrumproast',\n",
       " 'wwwyoutubecomusermattressserta',\n",
       " 'attweetatgreggutfeld',\n",
       " 'attweetatabeshinzo',\n",
       " 'attweetatusmc',\n",
       " 'attweetatturnbullmalcolm',\n",
       " 'attweetatmarthamaccallum',\n",
       " 'smallbusiness',\n",
       " 'attweetatjohnkasich',\n",
       " 'attweetatgopleader',\n",
       " 'halfstaff',\n",
       " 'schumerpelosi',\n",
       " 'attweetatuscg',\n",
       " 'wouldve',\n",
       " 'attweetatgaryplayer',\n",
       " 'attweetatsenategop',\n",
       " 'attweetatstevescalise',\n",
       " 'attweetatgovabbott',\n",
       " 'reince',\n",
       " 'danaperino',\n",
       " 'attweetatheritage',\n",
       " 'hashtweethashstanleycup',\n",
       " 'cutsreform',\n",
       " 'attweetatgovmikehuckabee',\n",
       " 'nbcnews',\n",
       " 'hashtweethashfema',\n",
       " 'attweetatpotus',\n",
       " 'hcare',\n",
       " 'grahamcassidy',\n",
       " 'hashtweethashharvey',\n",
       " 'hashtweethashflashbackfriday',\n",
       " 'wifes',\n",
       " 'abcwashington',\n",
       " 'middleeast',\n",
       " 'hashtweethashkateslaw',\n",
       " 'ossoff',\n",
       " 'jobstweetlink',\n",
       " 'attweetatrepmarkmeadows',\n",
       " 'attweetatjimjordan',\n",
       " 'attweetatexxonmobil',\n",
       " 'bernies',\n",
       " 'hashtweethashrepealobamacare',\n",
       " 'noontweetlink',\n",
       " 'presidenttweetlink',\n",
       " 'hashtweethashcrookedhillarytweetlink',\n",
       " 'bigleaguetruth',\n",
       " 'wikileakes',\n",
       " 'attweetatsenjohnmccain',\n",
       " 'timkaine',\n",
       " 'mikepence',\n",
       " 'attweetatterrybranstad',\n",
       " 'hashtweethashhannity',\n",
       " 'makeamericasafeagain',\n",
       " 'milliondollar',\n",
       " 'pennsylvaniamakeamericagreatagain',\n",
       " 'hashtweethashrncincletweetlink',\n",
       " 'attweetatsinow',\n",
       " 'raisedgave',\n",
       " 'kimguilfoyle',\n",
       " 'attweetatthetodaysgolfer',\n",
       " 'attweetattgowdysc',\n",
       " 'soonmakeamericagreatagain',\n",
       " 'votetrump',\n",
       " 'attweetatnewday',\n",
       " 'attweetatteamsters',\n",
       " 'ticketstweetlink',\n",
       " 'againinprimary',\n",
       " 'lyinted',\n",
       " 'ittweetlink',\n",
       " 'hashtweethashtrumptraintweetlink',\n",
       " 'hashtweethashnewyorkvalues',\n",
       " 'yorkmakeamericagreatagain',\n",
       " 'hashtweethashwiprimarytweetlink',\n",
       " 'attweetatrealsheriffjoe',\n",
       " 'jessebwatters',\n",
       " 'itmakeamericagreatagain',\n",
       " 'hashtweethashontherecord',\n",
       " 'hashtweethashnevadacaucus',\n",
       " 'hashtweethashvotetrumpnv',\n",
       " 'attweetatkayleighmcenany',\n",
       " 'markhalperin',\n",
       " 'findertweetlink',\n",
       " 'theyd',\n",
       " 'attweetatbobvanderplaats',\n",
       " 'attweetatcnnsitroom',\n",
       " 'attweetatwolfblitzer',\n",
       " 'hashtweethashsotu',\n",
       " 'stuartpstevens',\n",
       " 'attweetattheslystallone',\n",
       " 'jonahnro',\n",
       " 'attweetatthefix',\n",
       " 'attweetattheview',\n",
       " 'deucecrew',\n",
       " 'hashtweethashmsm',\n",
       " 'attweetatsecupp',\n",
       " 'attweetatfox',\n",
       " 'attweetatjheil',\n",
       " 'attweetatkilmeade',\n",
       " 'superpac',\n",
       " 'govmikehuckabee',\n",
       " 'attweetatcharleshurt',\n",
       " 'thisweekabc',\n",
       " 'marcorubio',\n",
       " 'asktrump',\n",
       " 'qampa',\n",
       " 'cnndebate',\n",
       " 'attweetatbobbyjindal',\n",
       " 'governorpataki',\n",
       " 'attweetatdanaperino',\n",
       " 'attweetataswoyer',\n",
       " 'attweetatcrainschicago',\n",
       " 'attweetatpgagrandslam',\n",
       " 'hashtweethashgolf',\n",
       " 'lincolnreagan',\n",
       " 'hashtweethashtcot',\n",
       " 'attweetatswaninvestor',\n",
       " 'attweetatprnewswire',\n",
       " 'attweetatkingjames',\n",
       " 'attweetatasavagenation',\n",
       " 'attweetatalexpappas',\n",
       " 'clientwith',\n",
       " 'attweetatespngolf',\n",
       " 'attweetatedshow',\n",
       " 'attweetatstephenfhayes',\n",
       " 'serenawilliams',\n",
       " 'attweetatlasvegassun',\n",
       " 'wellearned',\n",
       " 'attweetatthescotsman',\n",
       " 'trumpsoho',\n",
       " 'hashtweethashliub',\n",
       " 'attweetatkwrcrow',\n",
       " 'attweetatwsoctv',\n",
       " 'trumpknow',\n",
       " 'attweetatrwildewrites',\n",
       " 'attweetatnickjonas',\n",
       " 'attweetatexaminercom',\n",
       " 'attweetataaanews',\n",
       " 'attweetatmyrbeachonline',\n",
       " 'attweetatew',\n",
       " 'attweetatjonkarl',\n",
       " 'attweetatworldnetdaily',\n",
       " 'obamacareromneycare',\n",
       " 'hashtweethashveteransday',\n",
       " 'attweetatnicholasballasy',\n",
       " 'attweetatmlauer',\n",
       " 'attweetatlaraleayunaska',\n",
       " 'attweetatcntraveler',\n",
       " 'dailycaller',\n",
       " 'attweetatrustyrockets',\n",
       " 'zegarelli',\n",
       " 'graememcdowell',\n",
       " 'menie',\n",
       " 'attweetatmikeandmike',\n",
       " 'attweetatkingdommag',\n",
       " 'hashtweethashicebucketchallenge',\n",
       " 'hashtweethashbringbackourmarine',\n",
       " 'trumpnewyork',\n",
       " 'trumplasvegas',\n",
       " 'attweetatcnsnews',\n",
       " 'floortoceiling',\n",
       " 'attweetatmissteenusa',\n",
       " 'attweetatusopen',\n",
       " 'attweetatmiamiheat',\n",
       " 'attweetatprimeministersx',\n",
       " 'attweetatnba',\n",
       " 'trumpchicago',\n",
       " 'attweetatnyrangers',\n",
       " 'attweetatpressclubdc',\n",
       " 'attweetatspeakerboehner',\n",
       " 'hashtweethashnhfreedomsummit',\n",
       " 'lassner',\n",
       " 'attweetatmckaycoppins',\n",
       " 'attweetatrobastorino',\n",
       " 'birdkilling',\n",
       " 'hoaxsters',\n",
       " 'iamstevent',\n",
       " 'attweetatredskins',\n",
       " 'newyorkgop',\n",
       " 'attweetatjamesokeefeiii',\n",
       " 'hashtweethashsavesaeed',\n",
       " 'illconceived',\n",
       " 'attweetatmichaelphelps',\n",
       " 'rspbscotland',\n",
       " 'frankmdavisjr',\n",
       " 'attweetatnetflix',\n",
       " 'attweetatredsox',\n",
       " 'attweetatpiersmorganlive',\n",
       " 'attweetatguardian',\n",
       " 'pierpaolomonni',\n",
       " 'attweetatkingsthings',\n",
       " 'israelipm',\n",
       " 'attweetatmileycyrus',\n",
       " 'floydmayweather',\n",
       " 'attweetatthedailybeast',\n",
       " 'dangerweiner',\n",
       " 'attweetattoure',\n",
       " 'hashtweethashroadhard',\n",
       " 'attweetatarod',\n",
       " 'realsonnynewman',\n",
       " 'missusa',\n",
       " 'arods',\n",
       " 'attweetatkatiecouric',\n",
       " 'attweetatdannyzucker',\n",
       " 'therealmarilu',\n",
       " 'attweetatapple',\n",
       " 'macmiller',\n",
       " 'attweetatdanscavino',\n",
       " 'riggsdeb',\n",
       " 'attweetatsammartinobruno',\n",
       " 'attweetatmariamenounos',\n",
       " 'attweetatmariolopezextra',\n",
       " 'attweetattodayclicker',\n",
       " 'misskhan',\n",
       " 'attweetataccesshollywood',\n",
       " 'attweetatmarissamayer',\n",
       " 'clientwhat',\n",
       " 'attweetatlatenightjimmy',\n",
       " 'goangelo',\n",
       " 'moorereva',\n",
       " 'eawiii',\n",
       " 'mayorgas',\n",
       " 'zwlykins',\n",
       " 'attweetatjohnnypaulcole',\n",
       " 'clientlance',\n",
       " 'clientwe',\n",
       " 'clientit',\n",
       " 'attweetatbillmahers',\n",
       " 'arsenioofficial',\n",
       " 'attweetatuberfacts',\n",
       " 'danabrams',\n",
       " 'clientwhy',\n",
       " 'attweetatbbc',\n",
       " 'abfalecbaldwin',\n",
       " 'attweetatnewyorkpost',\n",
       " 'attweetatcadillac',\n",
       " 'mittromneys',\n",
       " 'attweetatnationals',\n",
       " 'attweetatcomedycentral',\n",
       " 'paternos',\n",
       " 'clientbarackobama',\n",
       " 'hashtweethashsayfie',\n",
       " 'betai',\n",
       " 'onshoring',\n",
       " 'jonhuntsman',\n",
       " 'attweetatsarahpalinusas',\n",
       " 'attweetatricksantorums',\n",
       " 'attweetatreppaulryans',\n",
       " 'clienttrumpvlog',\n",
       " 'attweetattimetogettough',\n",
       " 'attweetatralphreed',\n",
       " 'attweetatbobbeckel',\n",
       " 'susterens',\n",
       " 'attweetatnikkihaley',\n",
       " 'peaceofficersmemorialday',\n",
       " 'hashtweethashtakebackday',\n",
       " 'hashtweethashnationaldayofprayer',\n",
       " 'attweetatarmywpfootball',\n",
       " 'councel',\n",
       " 'jimrenacci',\n",
       " 'drunkdrugged',\n",
       " 'accountlawyer',\n",
       " 'flunkie',\n",
       " 'attweetatjpnpmo',\n",
       " 'hashtweethashtaxcuts',\n",
       " 'attweetatinterior',\n",
       " 'attweetatnatlparkservice',\n",
       " 'barriersalso',\n",
       " 'hashtweethashmedalofhonorday',\n",
       " 'attweetatambjohnbolton',\n",
       " 'hashtweethashbuildthewall',\n",
       " 'rallytweetlink',\n",
       " 'markburnetttv',\n",
       " 'attweetatrealromadowney',\n",
       " 'hashtweethashamerica',\n",
       " 'attweetatcpac',\n",
       " 'attweetatwestervillepd',\n",
       " 'hashtweethashnationalprayerbreakfast',\n",
       " 'attweetatcolts',\n",
       " 'attweetaticegov',\n",
       " 'jayz',\n",
       " 'strzok',\n",
       " 'attweetatsenschumer',\n",
       " 'icegov',\n",
       " 'attweetatjamiejmcintyre',\n",
       " 'russiatrump',\n",
       " 'cutreform',\n",
       " 'attweetatshopfloornam',\n",
       " 'hashtweethashhanukkah',\n",
       " 'daveweigel',\n",
       " 'attweetatusarmy',\n",
       " 'attweetatsenorrinhatch',\n",
       " 'attweetatjbanafw',\n",
       " 'attweetatvarneyco',\n",
       " 'indiv',\n",
       " 'mtgs',\n",
       " 'hashtweethashaseansummit',\n",
       " 'indopacific',\n",
       " 'antisecond',\n",
       " 'militaryveterans',\n",
       " 'hashtweethashpearlharbor',\n",
       " 'outand',\n",
       " 'comeys',\n",
       " 'billionyr',\n",
       " 'attweetatnycmayor',\n",
       " 'attweetatnygovcuomo',\n",
       " 'hashtweethashjfkfiles',\n",
       " 'hashtweethashopioidepidemic',\n",
       " 'attweetattheleegreenwood',\n",
       " 'onceinageneration',\n",
       " 'attweetatantonioguterres',\n",
       " 'hashtweethashprstrong',\n",
       " 'fakenews',\n",
       " 'mediathe',\n",
       " 'todaytweetlink',\n",
       " 'hashtweethashpuertorico',\n",
       " 'attweetattbn',\n",
       " 'hashtweethashusvi',\n",
       " 'recordhigh',\n",
       " 'warmbier',\n",
       " 'hashtweethashusaatunga',\n",
       " 'attweetatusairforce',\n",
       " 'hashtweethashlaborday',\n",
       " 'attweetatamericanlegion',\n",
       " 'draintheswamp',\n",
       " 'hashtweethashussjohnsmccain',\n",
       " 'attweetatdeptvetaffairs',\n",
       " 'attweetatnarendramodi',\n",
       " 'jobsborder',\n",
       " 'wstate',\n",
       " 'hashtweethashteamscalise',\n",
       " 'attweetatsecretaryperry',\n",
       " 'attweetatsecretaryzinke',\n",
       " 'attweetatsecpricemd',\n",
       " 'hashtweethashussarizona',\n",
       " 'attweetatemmanuelmacronthank',\n",
       " 'hashtweethashbastilleday',\n",
       " 'rdy',\n",
       " 'washtimes',\n",
       " 'shouldertoshoulder',\n",
       " 'hashtweethashpotusinpoland',\n",
       " 'happyindependenceday',\n",
       " 'hashtweethashvotekarenhandel',\n",
       " 'cubanamerican',\n",
       " 'attweetathousegop',\n",
       " 'covfefe',\n",
       " 'attweetatcommercegov',\n",
       " 'hashtweethashnato',\n",
       " 'attweetatpresidentruvi',\n",
       " 'hashtweethashpotusabroad',\n",
       " 'hashtweethashmothersday',\n",
       " 'rexnord',\n",
       " 'yeartweetlink',\n",
       " 'attweetatusedgov',\n",
       " 'attweetatnasas',\n",
       " 'attweetatainsleyearhardt',\n",
       " 'hashtweethashrepealandreplace',\n",
       " 'attweetatsnoopdogg',\n",
       " 'attweetatthebigeasy',\n",
       " 'hashtweethashmarchforlife',\n",
       " 'miamidade',\n",
       " 'innercities',\n",
       " 'llbean',\n",
       " 'hightweetlink',\n",
       " 'placethe',\n",
       " 'carolinatweetlink',\n",
       " 'iowatweetlink',\n",
       " 'hashtweethashelectionnight',\n",
       " 'nevadatweetlink',\n",
       " 'lawtweetlink',\n",
       " 'hashtweethashwattersworld',\n",
       " 'attweetatjessebwatters',\n",
       " 'ohiotweetlink',\n",
       " 'emailstweetlink',\n",
       " ...]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a weight matrix for words in training docs\n",
    "embedding_matrix = np.zeros( ( vocab_size, embeddings_dimension ) )\n",
    "missing_words = []\n",
    "\n",
    "# we need this to create empty coefficients array\n",
    "dummy_shape = embeddings_index[ \"the\" ].shape\n",
    "\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    \n",
    "    embedding_vector = embeddings_index.get( word )\n",
    "    \n",
    "    # not all words in our token list are in the wikipedia 400K set!\n",
    "    if embedding_vector is None:\n",
    "        \n",
    "        # report and create empty coefficients array\n",
    "        missing_words.append( word )\n",
    "        embedding_vector = np.zeros( dummy_shape )\n",
    "        \n",
    "    embedding_matrix[ i ] = embedding_vector\n",
    "    \n",
    "print( len( missing_words ) )\n",
    "missing_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300\n",
      "0.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "7383"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# confirm visually that \n",
    "print( len( embedding_matrix[ 0 ] ) )\n",
    "print( sum( embedding_matrix[ 0 ] ) )\n",
    "empty_coefficients_count = 0\n",
    "\n",
    "for i in range( len( embedding_matrix ) ):\n",
    "    if sum( embedding_matrix[ i ] ) == 0:\n",
    "        empty_coefficients_count += 1\n",
    "        \n",
    "empty_coefficients_count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.3\n",
      "1.4.1\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "print( keras.__version__ )\n",
    "\n",
    "import tensorflow as tf\n",
    "print( tf.__version__ )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, 50, 300)           6344700   \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 50, 100)           160400    \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, 100)               80400     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 100)               10100     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 21149)             2136049   \n",
      "=================================================================\n",
      "Total params: 8,731,649\n",
      "Trainable params: 8,731,649\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# define model\n",
    "model = Sequential()\n",
    "\n",
    "# now using a pre-trained, non-trainable embedding from glove's wiki analysis\n",
    "model.add( Embedding( vocab_size, embeddings_dimension, weights=[embedding_matrix], input_length=seq_length, trainable=True ) )\n",
    "model.add( LSTM( seq_length * 2, return_sequences=True ) )\n",
    "model.add( LSTM( seq_length * 2 ) )\n",
    "model.add( Dense( seq_length * 2, activation='relu' ) )\n",
    "\n",
    "# fixed TypeError below, downgraded keras from 2.1.5 to 2.1.3: https://github.com/keras-team/keras/issues/9621\n",
    "# TypeError: softmax() got an unexpected keyword argument 'axis'\n",
    "model.add( Dense( vocab_size, activation='softmax' ) )\n",
    "\n",
    "print( model.summary() )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3117.859375\n",
      "388.215953307393\n"
     ]
    }
   ],
   "source": [
    "# calc batch size\n",
    "print( len( sequences ) / 128 )\n",
    "print( len( sequences ) / 1028 )\n",
    "# Was:\n",
    "#batch_size = 124\n",
    "#batch_size = 1024\n",
    "\n",
    "# can't remember where I read that batch sizes larger than 512 cause erratic convergence patterns.\n",
    "# TODO: find that article!\n",
    "batch_size = 512\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Model?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load model? [y/n]y\n",
      "Loading model models/trump-tweets-w-links-n-ats-02.h5\n"
     ]
    }
   ],
   "source": [
    "load = input( \"Load model? [y/n]\" )\n",
    "\n",
    "if load == \"y\":\n",
    "    \n",
    "    model_name = \"models/trump-tweets-w-links-n-ats-02.h5\"\n",
    "    print( \"Loading model %s\" % model_name )\n",
    "    model = load_model( model_name )\n",
    "    \n",
    "else:\n",
    "    \n",
    "    print( \"NOT loading model\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.05.22 17:19\n",
      "Epoch 1/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.7566 - acc: 0.6225\n",
      "Epoch 2/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.7074 - acc: 0.6338\n",
      "Epoch 3/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.7036 - acc: 0.6351\n",
      "Epoch 4/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.6995 - acc: 0.6358\n",
      "Epoch 5/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.6920 - acc: 0.6370\n",
      "Epoch 6/50\n",
      "399086/399086 [==============================] - 115s 288us/step - loss: 1.6872 - acc: 0.6372\n",
      "Epoch 7/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.6872 - acc: 0.6374\n",
      "Epoch 8/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.6850 - acc: 0.6374\n",
      "Epoch 9/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.6688 - acc: 0.6418\n",
      "Epoch 10/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.6622 - acc: 0.6426\n",
      "Epoch 11/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.6498 - acc: 0.6456\n",
      "Epoch 12/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.6533 - acc: 0.6447\n",
      "Epoch 13/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.6551 - acc: 0.6437\n",
      "Epoch 14/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.6526 - acc: 0.6432\n",
      "Epoch 15/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.6430 - acc: 0.6453\n",
      "Epoch 16/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.6301 - acc: 0.6491\n",
      "Epoch 17/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.6218 - acc: 0.6507\n",
      "Epoch 18/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.6146 - acc: 0.6524\n",
      "Epoch 19/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.6104 - acc: 0.6526\n",
      "Epoch 20/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.6108 - acc: 0.6516\n",
      "Epoch 21/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.6089 - acc: 0.6524\n",
      "Epoch 22/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.5984 - acc: 0.6539\n",
      "Epoch 23/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.5955 - acc: 0.6547\n",
      "Epoch 24/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.5970 - acc: 0.6546\n",
      "Epoch 25/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.5857 - acc: 0.6572\n",
      "Epoch 26/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.5720 - acc: 0.6600\n",
      "Epoch 27/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.5763 - acc: 0.6592\n",
      "Epoch 28/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.5792 - acc: 0.6573\n",
      "Epoch 29/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.5706 - acc: 0.6593\n",
      "Epoch 30/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.5580 - acc: 0.6618\n",
      "Epoch 31/50\n",
      "399086/399086 [==============================] - 116s 289us/step - loss: 1.5522 - acc: 0.6635\n",
      "Epoch 32/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.5450 - acc: 0.6644\n",
      "Epoch 33/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.5323 - acc: 0.6682\n",
      "Epoch 34/50\n",
      "399086/399086 [==============================] - 116s 289us/step - loss: 1.5450 - acc: 0.6639\n",
      "Epoch 35/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.5497 - acc: 0.6632\n",
      "Epoch 36/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.5471 - acc: 0.6627\n",
      "Epoch 37/50\n",
      "399086/399086 [==============================] - 116s 289us/step - loss: 1.5263 - acc: 0.6678\n",
      "Epoch 38/50\n",
      "399086/399086 [==============================] - 116s 290us/step - loss: 1.5198 - acc: 0.6697\n",
      "Epoch 39/50\n",
      "399086/399086 [==============================] - 116s 289us/step - loss: 1.5156 - acc: 0.6693\n",
      "Epoch 40/50\n",
      "399086/399086 [==============================] - 116s 290us/step - loss: 1.5020 - acc: 0.6737\n",
      "Epoch 41/50\n",
      "399086/399086 [==============================] - 116s 289us/step - loss: 1.5021 - acc: 0.6727\n",
      "Epoch 42/50\n",
      "399086/399086 [==============================] - 116s 290us/step - loss: 1.4975 - acc: 0.6738\n",
      "Epoch 43/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.4915 - acc: 0.6748\n",
      "Epoch 44/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.5025 - acc: 0.6719\n",
      "Epoch 45/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.5007 - acc: 0.6724\n",
      "Epoch 46/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.4999 - acc: 0.6724\n",
      "Epoch 47/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.4724 - acc: 0.6788\n",
      "Epoch 48/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.4742 - acc: 0.6781\n",
      "Epoch 49/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.4700 - acc: 0.6790\n",
      "Epoch 50/50\n",
      "399086/399086 [==============================] - 115s 289us/step - loss: 1.4656 - acc: 0.6798\n",
      "2018.05.22 18:55\n",
      "Time to process: [1.6028001518381967] hours\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "# compile model\n",
    "model.compile( loss='categorical_crossentropy', optimizer='adam', metrics=[ 'accuracy' ] )\n",
    "# fit model: recent version takes ~1.5 hrs for 50 epochs = ~33% accuracy\n",
    "model.fit( X, y, batch_size=batch_size, epochs=50 )\n",
    "end_time = get_time()\n",
    "print_time( start_time, end_time, interval=\"hours\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the model to file\n",
    "model.save( \"models/trump-tweets-w-links-n-ats-02.h5\" )\n",
    "\n",
    "# save the tokenizer\n",
    "dump( tokenizer, open( \"tokenizers/trump-tweets-w-links-n-ats-02.dump\", 'wb' ) )\n",
    "\n",
    "# save embedding_matrix based on wiki embeddings, complete w/ missing coefficients array dummies\n",
    "dump( embedding_matrix, open( \"embeddings/trump-tweats-w-links-n-ats-02.glove\", 'wb' ) )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use The Model to Generate Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seq_length = len( lines[ 0 ].split() ) - 1\n",
    "seq_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'”'"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "punctuation_dict.get( \"smartquoteclose\", \"bar\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_seq( model, tokenizer, seq_length, seed_text, n_words ):\n",
    "    \n",
    "    result = list()\n",
    "    in_text = seed_text\n",
    "    yhat = [ 0.1 ]\n",
    "    \n",
    "    # generate a fixed number of words\n",
    "    for _ in range( n_words ):\n",
    "        \n",
    "        # encode the text as integer\n",
    "        encoded = tokenizer.texts_to_sequences( [ in_text ] )[ 0 ] \n",
    "        \n",
    "        # truncate sequences to a fixed length\n",
    "        encoded = pad_sequences( [ encoded ], maxlen=seq_length, truncating='pre' ) \n",
    "        \n",
    "        # predict probabilities for each word\n",
    "        yhat = model.predict_classes( encoded, verbose=0 )\n",
    "        \n",
    "        # map predicted word index to word\n",
    "        out_word = sequences_to_texts[ yhat[ 0 ] ]\n",
    "                \n",
    "        # append to input\n",
    "        in_text += ' ' + out_word\n",
    "        \n",
    "        #result.append( out_word )\n",
    "        # substitute punctuation tags for actual punctuation\n",
    "        result.append( punctuation_dict.get( out_word, out_word ) )\n",
    "        \n",
    "    print( yhat )\n",
    "    return ' '.join( result )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type( sequences_to_texts[ 1 ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reformat_punctuation( doc ):\n",
    "    \n",
    "    doc = doc.replace( ' . ', '. ' )\n",
    "    doc = doc.replace( ' ! ', '! ' )\n",
    "    doc = doc.replace( ' ? ', '? ' )\n",
    "    doc = doc.replace( ' , ', ', ' )\n",
    "    doc = doc.replace( ' : ', ': ' )\n",
    "    doc = doc.replace( ' ; ', '; ' )\n",
    "    \n",
    "    doc = doc.replace( '“ ', '“' )\n",
    "    doc = doc.replace( ' ”', '”' )\n",
    "    doc = doc.replace( \"attweetat\", '@' )\n",
    "    doc = doc.replace( \"hashtweethash\", '#' )\n",
    "    \n",
    "    return doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[link] congratulations to tom brady on yet another great victory tom is my friend and a total winner the deal with iran will go down as one of the most incompetent ever made. the us. lost on virtually every point. we just dont win anymore brithume i am... \n",
      "\n",
      "[438]\n",
      "... in first place and a major election of my administration has the fbi . he strongly refused to look into the first days of with surveillance . who would invest the opportunity to the end they had a product . not to participate ! unbelievable evening live watch the lead\n",
      "\n",
      "\n",
      "... in first place and a major election of my administration has the fbi. he strongly refused to look into the first days of with surveillance. who would invest the opportunity to the end they had a product. not to participate! unbelievable evening live watch the lead\n"
     ]
    }
   ],
   "source": [
    "# select a seed text\n",
    "seed_text = lines[ randint( 0, len( lines ) ) ]\n",
    "# substitute the seed words\n",
    "raw_text = seed_text.split( \" \" )\n",
    "\n",
    "clean_text = [ punctuation_dict.get( word, word ) for word in raw_text ]\n",
    "clean_text = ' '.join( clean_text )\n",
    "\n",
    "print( reformat_punctuation( clean_text ) + '... \\n' )\n",
    "#print( len( seed_text.split( \" \" ) ) )\n",
    "\n",
    "# generate new text\n",
    "generated = generate_seq( model, tokenizer, seq_length, seed_text, 50 )\n",
    "print( \"... \" + generated )\n",
    "print( \"\\n\\n... \" + reformat_punctuation( generated ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mexico\n",
      "[77]\n",
      "... agents dont jump until trump according to michigan first in history dept. [link] pennsylvania. send build wall smart [link] my @foxnews interview on @teamcavuto @seanhannity @todayshow [link] just arrived to quantico! [link] in the park hyatt in washington dc. on january! #americafirst [link] past am\n"
     ]
    }
   ],
   "source": [
    "my_input = input()\n",
    "\n",
    "generated = generate_seq( model, tokenizer, seq_length, my_input, 50 )\n",
    "print( \"... \" + reformat_punctuation( generated ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def generate_seq_word_by_word( model, tokenizer, seq_length, seed_text, n_words ):\n",
    "    \n",
    "#     print( \"...\", end='' )\n",
    "#     #result = list()\n",
    "#     in_text = seed_text\n",
    "    \n",
    "#     # generate a fixed number of words\n",
    "#     for _ in range( n_words ):\n",
    "        \n",
    "#         # encode the text as integer\n",
    "#         encoded = tokenizer.texts_to_sequences( [ in_text ] )[ 0 ] \n",
    "        \n",
    "#         # truncate sequences to a fixed length\n",
    "#         encoded = pad_sequences( [ encoded ], maxlen=seq_length, truncating='pre' ) \n",
    "        \n",
    "#         # predict probabilities for each word\n",
    "#         yhat = model.predict_classes( encoded, verbose=0 )\n",
    "        \n",
    "#         # map predicted word index to word\n",
    "#         out_word = ''\n",
    "#         for word, index in tokenizer.word_index.items():\n",
    "#             if index == yhat:\n",
    "#                 out_word = word\n",
    "#                 print( word, end=' ' )\n",
    "#                 break \n",
    "                \n",
    "#         # append to input for next iteration\n",
    "#         in_text += ' ' + out_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# select a seed text\n",
    "seed_text = lines[ randint( 0, len( lines ) ) ]\n",
    "print( seed_text + '...\\n' )\n",
    "\n",
    "# generate new text\n",
    "generate_seq_word_by_word( model, tokenizer, seq_length, seed_text, 25 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "america is\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'generate_seq_word_by_word' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-58-b64e50cb86a5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m# generate new text\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mgenerate_seq_word_by_word\u001b[0m\u001b[0;34m(\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseq_length\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmy_input\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m25\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'generate_seq_word_by_word' is not defined"
     ]
    }
   ],
   "source": [
    "my_input = input()\n",
    "\n",
    "# generate new text\n",
    "generate_seq_word_by_word( model, tokenizer, seq_length, my_input, 25 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}