word-level-rnn-w-word2vec-tweets-w-boundaries-take-III.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Based on: https://machinelearningmastery.com/use-word-embedding-layers-deep-learning-keras/ and https://machinelearningmastery.com/how-to-develop-a-word-level-neural-language-model-in-keras/ and https://github.com/stanfordnlp/GloVe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Choose GPU to Run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Select GPU [0 or 1]: 0\n"
     ]
    }
   ],
   "source": [
    "# From: https://github.com/keras-team/keras/issues/6031\n",
    "import os\n",
    "gpu_id = input( \"Select GPU [0 or 1]: \" )\n",
    "\n",
    "if gpu_id in [ \"0\", \"1\" ]:\n",
    "    os.environ[ \"CUDA_VISIBLE_DEVICES\" ] = gpu_id\n",
    "else:\n",
    "    print( \"Invalid GPU id.  Defaulting to '0,1'\" )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Choose CPU Cores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Share CPU cores w/ other models? [y/n]: y\n",
      "Allocating 6 cores to this notebook\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "cores = 12\n",
    "share_cores = input( \"Share CPU cores w/ other models? [y/n]: \" )\n",
    "\n",
    "if share_cores == \"y\":\n",
    "    \n",
    "    cores = int( cores / 2 )\n",
    "\n",
    "print( \"Allocating %d cores to this notebook\" % cores )\n",
    "\n",
    "# From: https://stackoverflow.com/questions/46421258/limit-number-of-cores-used-in-keras\n",
    "\n",
    "from keras import backend as K\n",
    "K.set_session(\n",
    "    K.tf.Session(\n",
    "        config=K.tf.ConfigProto(\n",
    "            intra_op_parallelism_threads=cores, inter_op_parallelism_threads=cores \n",
    "        )\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.07.10 12:32\n",
      "2018.07.10 12:32\n",
      "Time to process: [0.33643603324890137] seconds\n"
     ]
    }
   ],
   "source": [
    "import gc\n",
    "import re\n",
    "import time\n",
    "import datetime\n",
    "import string\n",
    "import pickle\n",
    "import tqdm\n",
    "from pickle import dump\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils import to_categorical\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.layers import LSTM\n",
    "from keras.layers import Bidirectional\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import Embedding\n",
    "from keras.models import load_model\n",
    "import collections\n",
    "from wordsegment import load, segment\n",
    "import numpy as np\n",
    "from spacy.vectors import Vectors\n",
    "from random import randint\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "def get_time( output=True ):\n",
    "    \n",
    "    temp = time.time()\n",
    "    if output:\n",
    "        now = datetime.datetime.now()\n",
    "        print( now.strftime( \"%Y.%m.%d %H:%M\" ) )\n",
    "        \n",
    "    return temp\n",
    "\n",
    "start_time = get_time()\n",
    "\n",
    "def print_time( start_time, end_time, interval=\"seconds\" ):\n",
    "    \n",
    "    if interval == \"hours\":\n",
    "        print ( \"Time to process: [%s] hours\" % ( str( ( end_time - start_time ) / 60 / 60 ) ) )\n",
    "    else:\n",
    "        print ( \"Time to process: [%s] seconds\" % ( str( end_time - start_time ) ) )\n",
    "\n",
    "in_filename = \"../texts/trump-tweets.txt\"\n",
    "#in_filename = \"../texts/trump-speeches.txt\"\n",
    "\n",
    "# load segmentation dictionary: http://www.grantjenks.com/docs/wordsegment/\n",
    "load()\n",
    "\n",
    "print_time( start_time, get_time() )\n",
    "\n",
    "# Global paths\n",
    "training_X_path = \"data/training-X-trump-tweets-w-links-n-ats-take-III.dump\"\n",
    "training_y_path = \"data/training-y-trump-tweets-w-links-n-ats-take-III.dump\"\n",
    "model_path = \"models/trump-tweets-w-links-n-ats-take-III.h5\"\n",
    "tokenizer_path = \"tokenizers/trump-tweets-w-links-n-ats-take-III.dump\"\n",
    "embedding_path = \"embeddings/trump-tweats-w-links-n-ats-take-III.glove\"\n",
    "embedding_keys_path = \"embeddings/trump-tweats-w-links-n-ats-take-III.keys\"\n",
    "#embeddings_spacy_numpy_path = \"embeddings/trump-tweats-w-links-n-ats-take-III-spacy.np\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Doc, Line by Line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.07.10 12:32\n",
      "2018.07.10 12:32\n",
      "Time to process: [0.009757280349731445] seconds\n"
     ]
    }
   ],
   "source": [
    "# http://cmdlinetips.com/2011/08/three-ways-to-read-a-text-file-line-by-line-in-python/\n",
    "def load_doc_by_line( filename ):\n",
    "    \n",
    "    # Open the file with read only permit\n",
    "    file = open( filename, \"r\" )\n",
    "    \n",
    "    # use readlines to read all lines in the file\n",
    "    # The variable \"lines\" is a list containing all lines in the file\n",
    "    lines = file.readlines()\n",
    "    \n",
    "    # close the file after reading the lines.\n",
    "    file.close()\n",
    "    \n",
    "    return lines\n",
    "\n",
    "start_time = get_time()\n",
    "tweets = load_doc_by_line( in_filename )\n",
    "print_time( start_time, get_time() )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Look at Tweet Stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tweets [22322], total words [392403], (mean words & chars)/tweet [17.58] words, [112.10] chars\n",
      "Max/Min words [62/1] per tweet\n",
      "Max tweet: Just met with UN Secretary-General António Guterres who is working hard to “Make the United Nations Great Again.” When the UN does more to solve conflicts around the world it means the U.S. has less to do and we save money. @NikkiHaley is doing a fantastic job! https://t.co/pqUv6cyH2z\n",
      "\n",
      "Min tweet: https://t.co/6VLQYAlcto\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# approximate words per tweet\n",
    "word_count = 0\n",
    "tweet_lens = 0\n",
    "max_words = 0\n",
    "max_idx = 0\n",
    "min_idx = 0\n",
    "min_words = 100\n",
    "\n",
    "for i, tweet in enumerate( tweets ):\n",
    "    \n",
    "    tweet_lens += len( tweet )\n",
    "    words = len( tweet.split( \" \" ) )\n",
    "    word_count += words\n",
    "    \n",
    "    if words > max_words:\n",
    "        max_words = words\n",
    "        max_id = i\n",
    "    if words < min_words:\n",
    "        min_words = words\n",
    "        min_idx = i\n",
    "\n",
    "words_per_tweet = word_count / len( tweets )\n",
    "chars_pre_tweet = tweet_lens / len( tweets )\n",
    "\n",
    "print( \"Tweets [%d], total words [%d], (mean words & chars)/tweet [%.2f] words, [%.2f] chars\" % ( len( tweets ), word_count, words_per_tweet, chars_pre_tweet ) )\n",
    "print( \"Max/Min words [%d/%d] per tweet\" % ( max_words, min_words ) )\n",
    "print( \"Max tweet:\", tweets[ max_idx ] )\n",
    "print( \"Min tweet:\", tweets[ min_idx ] )\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add Tweet OPEN/CLOSE Tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "opentweetopen Just met with UN Secretary-General António Guterres who is working hard to “Make the United Nations Great Again.” When the UN does more to solve conflicts around the world it means the U.S. has less to do and we save money. @NikkiHaley is doing a fantastic job! https://t.co/pqUv6cyH2z\n",
      " closetweetclose\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'opentweetopen Just met with UN Secretary-General António Guterres who is working hard to “Make the United Nations Great Again.” When the UN does more to solve conflicts around the world it means the U.S. has less to do and we save money. @NikkiHaley is doing a fantastic job! https://t.co/pqUv6cyH2z\\n closetweetclose opentweetopen America is a Nation that believes in the power of redemption. America'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add tweet oepn/close tags\n",
    "tweets = [ \"opentweetopen {} closetweetclose\".format( tweet ) for tweet in tweets ]\n",
    "print( tweets[ 0 ] )\n",
    "          \n",
    "# create doc from individual tweets\n",
    "doc = \" \".join( tweets )\n",
    "doc[ :400 ]\n",
    "# tweets = None\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Encoded Punctuation to Punctuation Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "punctuation_dict = {}\n",
    "punctuation_dict[ \"endperiod\" ] = \".\"\n",
    "punctuation_dict[ \"endquestion\" ] = \"?\"\n",
    "punctuation_dict[ \"endexclamation\" ] = \"!\"\n",
    "punctuation_dict[ \"pausecomma\" ] = \",\"\n",
    "punctuation_dict[ \"pausecolon\" ] = \":\"\n",
    "punctuation_dict[ \"pausesemicolon\" ] = \";\"\n",
    "punctuation_dict[ \"pauseemdash\" ] = \"-\"\n",
    "punctuation_dict[ \"pausedash\" ] = \"-\"\n",
    "punctuation_dict[ \"smartquoteopen\" ] = '“'\n",
    "punctuation_dict[ \"smartquoteclose\" ] = '”'\n",
    "punctuation_dict[ \"quoteopen\" ] = '\"'\n",
    "punctuation_dict[ \"quoteclose\" ] = '\"'\n",
    "punctuation_dict[ \"attweetat\" ] = '@'\n",
    "punctuation_dict[ \"tweetlink\" ] = \"[link]\"\n",
    "punctuation_dict[ \"hashtweethash\" ] = '#'\n",
    "punctuation_dict[ \"opentweetopen\" ] = '[start]'\n",
    "punctuation_dict[ \"closetweetclose\" ] = '[end]'\n",
    "punctuation_dict[ \"ampersand\" ] = '&'\n",
    "punctuation_dict[ \"tweetelipsis\" ] = \"...\"\n",
    "\n",
    "punctuation_dict[ \"contractionopen\" ] = \"contractionopen\"\n",
    "punctuation_dict[ \"contractionclose\" ] = \"contractionclose\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string.punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DIDNT\n",
      "italiaN hyphentweethyphen aAmerican hyphentweethyphen FOOD\n",
      "How's this for a lot of space?\n"
     ]
    }
   ],
   "source": [
    "print( re.sub( r\"([a-z])'([a-z])\", r\"\\1\\2\", \"DIDN'T\" , 0, re.IGNORECASE ) )\n",
    "print( re.sub( r\"([a-z])-([a-z])\", r\"\\1 hyphentweethyphen \\2\", \"italiaN-aAmerican-FOOD\" , 0, re.IGNORECASE ) )\n",
    "print( re.sub( ' +', ' ', \"How's this     for a   lot  of space?\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dont\n",
      "goin \n",
      "till\n",
      "False\n"
     ]
    }
   ],
   "source": [
    "print( re.sub( r\"([a-z])’([a-z])\", r\"\\1\\2\", \"don’t\" , 0, re.IGNORECASE ) )\n",
    "print( re.sub( r\"([a-z])’ \", r\"\\1 \", \"goin’ \" , 0, re.IGNORECASE ) )\n",
    "print( re.sub( r\"’([a-z])\", r\"\\1\", \"’till\" , 0, re.IGNORECASE ) )\n",
    "print( \"–\" == \"—\" )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# turn a doc into clean tokens\n",
    "punctuation_string = '!‘\"\\'#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~'\n",
    "\n",
    "def clean_doc( doc ):\n",
    "    \n",
    "    # multiple kinds of emdash!\n",
    "    doc = doc.replace( '--', ' pauseemdash ' )\n",
    "    doc = doc.replace( '—', ' pauseemdash ' )\n",
    "    doc = doc.replace( '–', ' pauseemdash ' )\n",
    "    doc = doc.replace( 'u.s.', ' US ' )\n",
    "    doc = doc.replace( 'U.S.', ' US ' )\n",
    "    doc = doc.replace( \"…\", \" tweetelipsis \" )\n",
    "    doc = doc.replace( \"...\", \" tweetelipsis \" )\n",
    "    doc = doc.replace( '. ', ' endperiod ' )\n",
    "    doc = doc.replace( '! ', ' endexclamation ' )\n",
    "    doc = doc.replace( '? ', ' endquestion ' )\n",
    "    doc = doc.replace( '•', ' endbullet ' )\n",
    "    doc = doc.replace( ', ', ' pausecomma ' )\n",
    "    doc = doc.replace( ': ', ' pausecolon ' )\n",
    "    doc = doc.replace( '; ', ' pausesemicolon ' )\n",
    "    doc = doc.replace( ' - ', ' pausedash ' )\n",
    "    doc = doc.replace( '“', ' smartquoteopen ' )\n",
    "    doc = doc.replace( '”', ' smartquoteclose ' )\n",
    "    doc = doc.replace( ' \"', ' quoteopen ' )\n",
    "    doc = doc.replace( '\" ', ' quoteclose ' )\n",
    "    doc = doc.replace( \"@ \", \" \" ) # remove trailing @'s first...\n",
    "    doc = doc.replace( \"@\", \"attweetat\" ) # ...then prefix 1st char @ as word\n",
    "    doc = doc.replace( \"# \", \" \" ) # remove trailing #'s first...\n",
    "    doc = doc.replace( \"#\", \"hashtweethash\" ) # ...then prefix 1st char # as word\n",
    "    doc = doc.replace( \"&amp;\", \" ampersand \" )\n",
    "    \n",
    "    # From: https://stackoverflow.com/questions/33113338/how-to-replace-dash-between-characters-with-space-using-regex\n",
    "    # replace hyphenated words w/ spaces\n",
    "    doc = re.sub( r\"([a-z])\\-([a-z])\", r\"\\1 \\2\", doc , 0, re.IGNORECASE )\n",
    "    \n",
    "    # replace comma separated words w/0 spaces\n",
    "    doc = re.sub( r\"([a-z]),([a-z])\", r\"\\1 \\2\", doc , 0, re.IGNORECASE )\n",
    "    \n",
    "    # replace links w/ \"tweetlink\"\n",
    "    # basic regex here: https://bytes.com/topic/python/answers/741677-find-replace-hyperlinks-string\n",
    "    http_pattern = r'http[^\\s\\n\\r]+'\n",
    "    doc = re.sub( http_pattern , \"tweetlink\", doc )\n",
    "    \n",
    "    # this overgenerates texttexttexttweetlink, so insert space where it occurs\n",
    "    doc = re.sub( r\"([a-z])(tweetlink)\", r\"\\1 \\2\", doc , 0, re.IGNORECASE )\n",
    "    \n",
    "    # above isn't catching all (wuh?!?), so just bruteforce it\n",
    "    doc = doc.replace( \"tweetlink\", \" tweetlink\" )\n",
    "    \n",
    "    # overgeneration of foooooooooooooooohashtweethash, so insert space where it occurs\n",
    "    doc = re.sub( r\"([a-z])(hashtweethash)\", r\"\\1 \\2\", doc , 0, re.IGNORECASE )\n",
    "    # above isn't catching all (wuh?!?), so just bruteforce it\n",
    "    doc = doc.replace( \"hashtweethash\", \" hashtweethash\" )\n",
    "    \n",
    "    # overgeneration of fooooooooooooooooattweetat, so insert space where it occurs\n",
    "    doc = re.sub( r\"([a-z])(attweetat)\", r\"\\1 \\2\", doc , 0, re.IGNORECASE )\n",
    "    # above isn't catching all (wuh?!?), so just bruteforce it\n",
    "    doc = doc.replace( \"attweetat\", \" attweetat\" )\n",
    "    \n",
    "    # tag all hyphenated words to protect from deletion\n",
    "    doc = re.sub( r\"([a-z])-([a-z])\", r\"\\1hyphentweethyphen\\2\", doc , 0, re.IGNORECASE )\n",
    "    \n",
    "    # do big ad-hoc global replacement, instead of using maketrans and string.punctuation\n",
    "    # single quote is special case: don't leave a space, so that contractions are collapsed\n",
    "    doc = re.sub( r\"([a-z]+)'([a-z]+)\", r\"contractionopen \\1\\2 contractionclose\", doc , 0, re.IGNORECASE )\n",
    "    # some fool's using a DIFFERENT apostrophe... ’\n",
    "    doc = re.sub( r\"([a-z]+)’([a-z]+)\", r\"contractionopen \\1\\2 contractionclose\", doc , 0, re.IGNORECASE )\n",
    "    # replaced by above to wrap contractions\n",
    "    # doc = re.sub( r\"([a-z])'([a-z])\", r\"\\1\\2\", doc , 0, re.IGNORECASE )\n",
    "    # # some fool's using a DIFFERENT apostrophe... ’\n",
    "    # doc = re.sub( r\"([a-z])’([a-z])\", r\"\\1\\2\", doc , 0, re.IGNORECASE )\n",
    "    \n",
    "    # trailing deletion: goin' to goin\n",
    "    doc = re.sub( r\"([a-z])’ \", r\"\\1 \", doc , 0, re.IGNORECASE )\n",
    "    # not working?!?\n",
    "    #doc = re.sub( r\"’([a-z])\", r\"\\1\", doc , 0, re.IGNORECASE )\n",
    "    \n",
    "    # otherwise, delete all chars not already tagged as having semantic interest\n",
    "    # moved up!\n",
    "    #punctuation_string = '!‘\"\\'#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~'\n",
    "    for punctuation_char in punctuation_string:\n",
    "        doc = doc.replace( punctuation_char, ' ' )\n",
    "    \n",
    "    # ...now that global deletion of stragging dashes has been performed, replaced hyphenated words\n",
    "    doc = doc.replace( \"hyphentweethyphen\", \"-\" )\n",
    "    \n",
    "    # finally, reduce duplicate spaces to just one: https://stackoverflow.com/questions/1546226/simple-way-to-remove-multiple-spaces-in-a-string/15913564\n",
    "    # doc = doc.replace( \"  \", ' ' )\n",
    "    doc = re.sub( ' +', ' ', doc )\n",
    "    \n",
    "    # split into tokens by white space\n",
    "    tokens = doc.split()\n",
    "    \n",
    "#     # remove punctuation from each token\n",
    "#     table = str.maketrans( '', '', string.punctuation ) # will strip all .?!,:; that don't fit replace expr above.\n",
    "#     #table = str.maketrans( '', '', my_punctuation )\n",
    "#     tokens = [ w.translate( table ) for w in tokens ]\n",
    "    \n",
    "    # remove remaining tokens that are not alphabetic\n",
    "    # if to_lower:\n",
    "    #tokens = [ word for word in tokens if word.isalpha() ]\n",
    "    \n",
    "    # make lower case\n",
    "    #tokens = [ word.lower() for word in tokens ] \n",
    "    \n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Feels',\n",
       " 'like',\n",
       " 'contractionopen',\n",
       " 'Im',\n",
       " 'contractionclose',\n",
       " 'goin',\n",
       " 'crazy',\n",
       " 'pausecomma',\n",
       " 'this',\n",
       " 'contractionopen',\n",
       " 'isnt',\n",
       " 'contractionclose',\n",
       " 'some',\n",
       " 'contractionopen',\n",
       " 'fools',\n",
       " 'contractionclose',\n",
       " 'errand',\n",
       " 'pausecomma',\n",
       " 'contractionopen',\n",
       " 'wouldnt',\n",
       " 'contractionclose',\n",
       " 'you',\n",
       " 'agree']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_doc( \"Feels like I'm goin’ crazy, this isn’t some fool's errand, wouldn't you agree?\" )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Vocabulary and Segment Doc Tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.07.10 12:32\n",
      "2018.07.10 12:32\n",
      "Time to process: [0.03898358345031738] seconds\n",
      "vocabulary_list 400000\n",
      "vocabulary_dict 400000\n",
      "False\n",
      "True\n",
      "True\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# load dictionary\n",
    "start_time = get_time()\n",
    "\n",
    "embeddings_dimension = 300 #must be 50, 100, 200, 300\n",
    "with open( \"output/vocabulary-glove.6B.\" + str( embeddings_dimension ) + \"d.txt\", 'r' ) as vocabulary_file:\n",
    "    \n",
    "    # omit newline char: https://stackoverflow.com/questions/12330522/reading-a-file-without-newlines\n",
    "    vocabulary_list = vocabulary_file.read().splitlines()\n",
    "\n",
    "print_time( start_time, get_time() )\n",
    "\n",
    "vocabulary_dict = dict.fromkeys( vocabulary_list )\n",
    "print( \"vocabulary_list\", len( vocabulary_list ) )\n",
    "print( \"vocabulary_dict\", len( vocabulary_dict ) )\n",
    "\n",
    "print( \"1234123412341234\" in vocabulary_dict )\n",
    "print( \"earth\" in vocabulary_dict )\n",
    "print( \"earth\" in vocabulary_list )\n",
    "print( vocabulary_dict[ \"earth\" ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.07.10 12:32\n",
      "2018.07.10 12:32\n",
      "Time to process: [1.929260015487671] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "\n",
    "# clean document\n",
    "tokens_raw = clean_doc( doc )\n",
    "\n",
    "print_time( start_time, get_time() )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.07.10 12:32\n",
      "------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n",
      "Total tokens_raw: 488581\n",
      "Total tokens_segmented: 541867\n",
      "Unique Tokens: 24497\n",
      "Novel Tokens: 2054\n",
      "Unique Novel Tokens: 1186\n",
      "trumpisms: 0\n",
      "digits: 194\n",
      "Unique digits: 141\n",
      "digits_dropped 552\n",
      "2018.07.10 12:32\n",
      "Time to process: [7.471104383468628] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "\n",
    "tokens_segmented = []\n",
    "tokens_novel = []\n",
    "digits = []\n",
    "trumpisms = []\n",
    "\n",
    "disposable_digits_len = len( \"467821512981233666\" )\n",
    "\n",
    "# iterate raw tokens and segment hashtags\n",
    "hashtag_prefix_len = len( \"hashtweethash\" )\n",
    "attag_prefix_len = len( \"attweetat\" )\n",
    "\n",
    "digits_dropped = 0\n",
    "\n",
    "for token_raw in tokens_raw:\n",
    "    \n",
    "    if token_raw.startswith( \"attweetat\" ):\n",
    "        \n",
    "        # segment what comes after \"attweetat\"\n",
    "        #print( token_raw, token_raw[ attag_prefix_len: ] )\n",
    "        attag_segments = segment( token_raw[ attag_prefix_len: ] )\n",
    "        #print( \"attag_segments\", attag_segments )\n",
    "        \n",
    "        # iterate identified segments\n",
    "        tokens_segmented.append( \"attagopen\" )\n",
    "        for attag_segment in attag_segments:\n",
    "            tokens_segmented.append( attag_segment.capitalize() )\n",
    "        tokens_segmented.append( \"attagclose\" )\n",
    "        \n",
    "    elif token_raw.startswith( \"hashtweethash\" ):\n",
    "        \n",
    "        # segment what comes after \"hashtweethash\"\n",
    "        hashtag_segments = segment( token_raw[ hashtag_prefix_len: ] )\n",
    "        #print( \"hashtag_segments\", hashtag_segments )\n",
    "        \n",
    "        # iterate identified segments\n",
    "        tokens_segmented.append( \"hashtagopen\" )\n",
    "        for hashtag_segment in hashtag_segments:\n",
    "            tokens_segmented.append( hashtag_segment.capitalize() )\n",
    "        tokens_segmented.append( \"hashtagclose\" )\n",
    "        \n",
    "    else:\n",
    "        \n",
    "        if token_raw in punctuation_dict:\n",
    "        \n",
    "            #print( \"Punctuation RESERVED word:\", token_raw )\n",
    "            tokens_segmented.append( token_raw )\n",
    "            \n",
    "        else:\n",
    "            \n",
    "            if token_raw.lower() in vocabulary_dict:\n",
    "            \n",
    "                #print( \"IN dictionary:\", token_raw )\n",
    "                tokens_segmented.append( token_raw )\n",
    "                \n",
    "            else:\n",
    "                \n",
    "                # there's some timestamp digit leakage, and they're almost all the same len.  Drop them!\n",
    "                if token_raw.isdigit() and len( token_raw ) == disposable_digits_len:\n",
    "                    \n",
    "                    digits_dropped += 1\n",
    "                    print( \"-\", end=\"\" )\n",
    "                    \n",
    "                # keep the other ones\n",
    "                elif token_raw.isdigit():\n",
    "                    \n",
    "                    digits.append( token_raw )\n",
    "                    tokens_segmented.append( \"digitsopen\" )\n",
    "                    tokens_segmented.append( token_raw )\n",
    "                    tokens_segmented.append( \"digitsclose\" )\n",
    "                    tokens_novel.append( token_raw )\n",
    "                \n",
    "                else:\n",
    "                    \n",
    "                    # just tag it for now, and get on with it...\n",
    "                    # ...later we'll add emoji, segmentation and mispelling handling\n",
    "                    tokens_segmented.append( \"trumpismopen\" )\n",
    "                    tokens_segmented.append( token_raw )\n",
    "                    tokens_segmented.append( \"trumpismclose\" )\n",
    "                    \n",
    "                    tokens_novel.append( token_raw )\n",
    "                \n",
    "#                     trumpism_segments = segment( token_raw )\n",
    "#                     trumpisms.append( trumpism_segments )\n",
    "#                   \n",
    "#                     print( \"\\ntrumpism_segments: \", trumpism_segments, \"\\n\" )\n",
    "#    \n",
    "#                     tokens_segmented.append( \"trumpismopen\" )\n",
    "#                     for segment in trumpism_segments:\n",
    "#                         tokens_segmented.append( segment.capitalize() )\n",
    "#                     tokens_segmented.append( \"trumpismclose\" )\n",
    "\n",
    "# Don't lower case, it's now relevant\n",
    "# tokens_segmented = [ word.lower() for word in tokens_segmented ] \n",
    "            \n",
    "# # 'compress' into list of unique tokens\n",
    "tokens_unique = list( set( tokens_segmented ) )\n",
    "tokens_unique_lowercase = [ token.lower() for token in tokens_unique ]\n",
    "\n",
    "print()\n",
    "#print( \"tokens_raw\", tokens_raw[ :100 ] )\n",
    "#print( \"tokens_segmented\", tokens_segmented[ :100 ] )\n",
    "print( 'Total tokens_raw: %d' % len( tokens_raw ) )\n",
    "print( 'Total tokens_segmented: %d' % len( tokens_segmented ) )\n",
    "print( 'Unique Tokens: %d' % len( tokens_unique ) )\n",
    "print( 'Novel Tokens: %d' % len( tokens_novel ) )\n",
    "print( 'Unique Novel Tokens: %d' % len( set( tokens_novel ) ) )\n",
    "print( 'trumpisms: %d' % len( trumpisms ) )\n",
    "print( 'digits: %d' % len( digits ) )\n",
    "print( 'Unique digits: %d' % len( set( digits ) ) )\n",
    "print( \"digits_dropped\", digits_dropped )\n",
    "\n",
    "print_time( start_time, get_time() )\n",
    "\n",
    "# 2018.06.07 09:32\n",
    "# Total Tokens: 448210\n",
    "# Unique Tokens: 19564\n",
    "# Time to process: [0.35286879539489746] seconds\n",
    "\n",
    "# 2018.06.18 15:01\n",
    "# Total tokens_raw: 448583\n",
    "# Total tokens_segmented: 457501\n",
    "# Unique Tokens: 18911\n",
    "# Time to process: [3.0585978031158447] seconds\n",
    "\n",
    "# after processing @'s\n",
    "# 2018.06.18 16:28\n",
    "# Total tokens_raw: 448583\n",
    "# Total tokens_segmented: 493688\n",
    "# Unique Tokens: 17673\n",
    "# Time to process: [9.280703783035278] seconds\n",
    "\n",
    "# after processing _____\n",
    "# 2018.06.19 17:50\n",
    "# Total tokens_raw: 472558\n",
    "# Total tokens_segmented: 522310\n",
    "# Unique Tokens: 20747\n",
    "# Time to process: [9.574301481246948] seconds\n",
    "\n",
    "# 2018.06.21 12:43\n",
    "# Total tokens_raw: 488581\n",
    "# Total tokens_segmented: 541479\n",
    "# Unique Tokens: 24495\n",
    "# Novel Tokens: 2054\n",
    "# Unique Novel Tokens: 1186\n",
    "# trumpisms: 0\n",
    "# digits: 194\n",
    "# Unique digits: 141\n",
    "# digits_dropped 552\n",
    "# Time to process: [7.744369983673096] seconds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from emoji import UNICODE_EMOJI\n",
    "\n",
    "def has_emoji( s ):\n",
    "    count = 0\n",
    "    for emoji in UNICODE_EMOJI:\n",
    "        count += s.count( emoji )\n",
    "        if count > 1:\n",
    "            return False \n",
    "        \n",
    "    return bool( count )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['', '', '', '', '', '', '', '', '✅', '', '', '', '']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import emoji\n",
    "# From: https://stackoverflow.com/questions/43146528/how-to-extract-all-the-emojis-from-text\n",
    "# Create the function to extract the emojis\n",
    "def extract_emojis( a_list ):\n",
    "    \n",
    "    emojis_list = map(lambda x: ''.join(x.split()), emoji.UNICODE_EMOJI.keys())\n",
    "    r = re.compile('|'.join(re.escape(p) for p in emojis_list))\n",
    "    aux=[ ' '.join(r.findall(s)) for s in a_list ]\n",
    "    return(aux)\n",
    "\n",
    "## Execute the function\n",
    "extract_emojis( \"Carolina✅Ohio\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['', '', '', '']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extract_emojis( \"🇺🇸🇪🇸\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['carolina', 'ohio']\n",
      "['nikki', 'haley']\n",
      "['obama', 'trade']\n",
      "['production', 'up7']\n",
      "True\n",
      "False\n"
     ]
    }
   ],
   "source": [
    "# load()\n",
    "# print( 'isnt' in vocabulary_dict )\n",
    "# print( 'isnt' in vocabulary_dict )\n",
    "# print( 'McCabes' in vocabulary_dict )\n",
    "\n",
    "# foo = \"NikkiHaley\".lower()\n",
    "# print( foo )\n",
    "print( segment( \"Carolina✅Ohio\" ) )\n",
    "print( segment( \"NikkiHaley\" ) ) \n",
    "print( segment( \"OBAMATRADE\" ) )\n",
    "print( segment( \"PRODUCTIONUp📈7\" ) )\n",
    "\n",
    "print( has_emoji( \"Carolina✅Ohio\" ) )\n",
    "print( has_emoji( \"🇺🇸🇪🇸\" ) )\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "token 8838 segment(s) ['8838']\n",
      "token Frankenstien segment(s) ['franken', 'stien']\n",
      "token thinkking segment(s) ['think', 'king']\n",
      "token KatherineWebb segment(s) ['katherine', 'webb']\n",
      "token Spolier segment(s) ['spo', 'lier']\n",
      "token EWErickson segment(s) ['ew', 'erickson']\n",
      "token Sharyl segment(s) ['sharyl']\n",
      "token rolemodeland segment(s) ['role', 'model', 'and']\n",
      "token failurecountries segment(s) ['failure', 'countries']\n",
      "token tonights segment(s) ['to', 'nights']\n",
      "token Jeters segment(s) ['jeter', 's']\n",
      "token JCena segment(s) ['j', 'cena']\n",
      "token Bergdhal segment(s) ['berg', 'dhal']\n",
      "token everyones segment(s) ['everyones']\n",
      "token ’79 segment(s) ['79']\n",
      "token 96K segment(s) ['96k']\n",
      "token 2yrs segment(s) ['2yrs']\n",
      "token erictrump segment(s) ['eric', 'trump']\n",
      "token Leachs segment(s) ['leach', 's']\n",
      "token Pepp segment(s) ['pepp']\n",
      "token haterseven segment(s) ['haters', 'even']\n",
      "token CubaMemorandum segment(s) ['cuba', 'memorandum']\n",
      "token freedomsit segment(s) ['freedoms', 'it']\n",
      "token oddsin segment(s) ['odds', 'in']\n",
      "token Kaufstache segment(s) ['kauf', 'stache']\n",
      "token pollitically segment(s) ['poll', 'it', 'ically']\n",
      "token AmazonWashingtonPost segment(s) ['amazon', 'washington', 'post']\n",
      "token impossibl segment(s) ['im', 'possibl']\n",
      "token BREXIT segment(s) ['br', 'exit']\n",
      "token Tonights segment(s) ['to', 'nights']\n",
      "token iontv segment(s) ['i', 'on', 'tv']\n",
      "token bosss segment(s) ['boss', 's']\n",
      "token 40mph segment(s) ['40mph']\n",
      "token 3pm6 segment(s) ['3pm6']\n",
      "token kface segment(s) ['k', 'face']\n",
      "token Janeiros segment(s) ['janeiro', 's']\n",
      "token overregulates segment(s) ['over', 'regulates']\n",
      "token DillonCJTF segment(s) ['dillon', 'cjtf']\n",
      "token Romneywho segment(s) ['romney', 'who']\n",
      "token DEPLORABLES segment(s) ['deplorable', 's']\n",
      "token 13164 segment(s) ['13164']\n",
      "token 8amE segment(s) ['8ame']\n",
      "token today2 segment(s) ['today', '2']\n",
      "token GROWNG segment(s) ['grown', 'g']\n",
      "token thiswe segment(s) ['this', 'we']\n",
      "token 8pm6 segment(s) ['8pm6']\n",
      "token Trumpative segment(s) ['trump', 'ative']\n",
      "token Theyve segment(s) ['theyve']\n",
      "token flunkie segment(s) ['flunk', 'ie']\n",
      "token 31318 segment(s) ['31318']\n",
      "token ofour segment(s) ['of', 'our']\n",
      "token Enough’ segment(s) ['enough']\n",
      "token SuperPACs segment(s) ['super', 'pacs']\n",
      "token Growthwhich segment(s) ['growth', 'which']\n",
      "token soireé segment(s) ['soire']\n",
      "token 365000 segment(s) ['365000']\n",
      "token Erics segment(s) ['erics']\n",
      "token Warmbier segment(s) ['warm', 'bier']\n",
      "token ginrnnr2 segment(s) ['ginrnnr2']\n",
      "token ADDRESS🇺🇸 segment(s) ['address']\n",
      "token mililtary segment(s) ['mi', 'lil', 'tary']\n",
      "token strageic segment(s) ['strage', 'ic']\n",
      "token Willthe segment(s) ['will', 'the']\n",
      "token 70B segment(s) ['70b']\n",
      "token Algemeiner segment(s) ['alge', 'meiner']\n",
      "token €10bn segment(s) ['10bn']\n",
      "token hershe segment(s) ['her', 'she']\n",
      "token RealClear segment(s) ['realclear']\n",
      "token Obrian segment(s) ['o', 'brian']\n",
      "token telepromter segment(s) ['tel', 'eprom', 'ter']\n",
      "token Foldo segment(s) ['fold', 'o']\n",
      "token 16T segment(s) ['16t']\n",
      "token Iraelis segment(s) ['i', 'rael', 'is']\n",
      "token episodeover segment(s) ['episode', 'over']\n",
      "token theyll segment(s) ['theyll']\n",
      "token Meadia segment(s) ['me', 'a', 'dia']\n",
      "token Champions⚾️ segment(s) ['champions']\n",
      "token raceFAKE segment(s) ['race', 'fake']\n",
      "token insticts segment(s) ['inst', 'icts']\n",
      "token Everyones segment(s) ['everyones']\n",
      "token MSNBCS segment(s) ['msnbc', 's']\n",
      "token Wednesday1 segment(s) ['wednesday', '1']\n",
      "token Dorals segment(s) ['doral', 's']\n",
      "token 173K segment(s) ['173k']\n",
      "token Jemele segment(s) ['je', 'mele']\n",
      "token speechthe segment(s) ['speech', 'the']\n",
      "token everybod segment(s) ['every', 'bod']\n",
      "token Shamrock☘️Bowl segment(s) ['shamrock', 'bowl']\n",
      "token MadeInTheUSA🇺🇸 segment(s) ['made', 'in', 'the', 'usa']\n",
      "token 300g segment(s) ['300g']\n",
      "token bankrutcy segment(s) ['bankr', 'utc', 'y']\n",
      "token Bighurt segment(s) ['big', 'hurt']\n",
      "token PollDonald segment(s) ['poll', 'donald']\n",
      "token Bulgers segment(s) ['bulger', 's']\n",
      "token Down📉31 segment(s) ['down', '31']\n",
      "token unaccepting segment(s) ['un', 'accepting']\n",
      "token 2537 segment(s) ['2537']\n",
      "token polititians segment(s) ['politi', 'tians']\n",
      "token includeTrump segment(s) ['include', 'trump']\n",
      "token 5600000 segment(s) ['5600000']\n",
      "token Wowthe segment(s) ['wow', 'the']\n",
      "token her🇺🇸☑️LOVE segment(s) ['her', 'love']\n",
      "token 225000 segment(s) ['225000']\n",
      "token 4pmE segment(s) ['4pme']\n",
      "token RiversWas segment(s) ['rivers', 'was']\n",
      "token Weiners segment(s) ['weiners']\n",
      "token 23116928 segment(s) ['23116928']\n",
      "token Andeavor segment(s) ['and', 'eav', 'or']\n",
      "token Conneticuts segment(s) ['conneticut', 's']\n",
      "token LukasHall segment(s) ['luka', 'shall']\n",
      "token shouldtrumprun segment(s) ['should', 'trump', 'run']\n",
      "token busineses segment(s) ['busineses']\n",
      "token 9amE segment(s) ['9ame']\n",
      "token KayyArrDee segment(s) ['kay', 'yarr', 'dee']\n",
      "token Univisions segment(s) ['univision', 's']\n",
      "token Mirziyoyev segment(s) ['mirziyoyev']\n",
      "token Syriafix segment(s) ['syria', 'fix']\n",
      "token ncislover segment(s) ['nc', 'is', 'lover']\n",
      "token ºº segment(s) []\n",
      "token Lexxay segment(s) ['lexx', 'ay']\n",
      "token KLZA segment(s) ['kl', 'za']\n",
      "token AmyMek segment(s) ['amy', 'mek']\n",
      "token spinnnn segment(s) ['spin', 'nnn']\n",
      "token havnt segment(s) ['havnt']\n",
      "token qoute segment(s) ['qoute']\n",
      "token W20 segment(s) ['w20']\n",
      "token Shabbab segment(s) ['shabba', 'b']\n",
      "token Courtmarshal segment(s) ['court', 'marshal']\n",
      "token 64B segment(s) ['64b']\n",
      "token Foerderer segment(s) ['foer', 'der', 'er']\n",
      "token 600000 segment(s) ['600000']\n",
      "token 75X segment(s) ['75x']\n",
      "token WJIM segment(s) ['w', 'jim']\n",
      "token booketc segment(s) ['book', 'etc']\n",
      "token Iowas segment(s) ['iowa', 's']\n",
      "token HEREOS segment(s) ['here', 'os']\n",
      "token indiv segment(s) ['indiv']\n",
      "token sexter segment(s) ['sex', 'ter']\n",
      "token theoriginal segment(s) ['the', 'original']\n",
      "token 400000 segment(s) ['400000']\n",
      "token Hallmar segment(s) ['hall', 'mar']\n",
      "token retweets segment(s) ['re', 'tweet', 's']\n",
      "token Ebolathen segment(s) ['ebola', 'then']\n",
      "token 375B segment(s) ['375b']\n",
      "token GoAngelo segment(s) ['go', 'angelo']\n",
      "token HCare segment(s) ['h', 'care']\n",
      "token Zuckerbergs segment(s) ['zucker', 'bergs']\n",
      "token derekhough segment(s) ['derek', 'hough']\n",
      "token 🚨BREAKING🚨 segment(s) ['breaking']\n",
      "token Qaedas segment(s) ['qaedas']\n",
      "token 118M segment(s) ['118m']\n",
      "token 88419000 segment(s) ['88419000']\n",
      "token Eliots segment(s) ['eliot', 's']\n",
      "token willbe segment(s) ['will', 'be']\n",
      "token unempoyment segment(s) ['un', 'empoyment']\n",
      "token 4485 segment(s) ['4485']\n",
      "token 92M segment(s) ['92m']\n",
      "token Comeys segment(s) ['come', 'ys']\n",
      "token OlympicCommittee segment(s) ['olympic', 'committee']\n",
      "token 30am segment(s) ['30am']\n",
      "token Scotlands segment(s) ['scotlands']\n",
      "token corectly segment(s) ['corectly']\n",
      "token 41to14 segment(s) ['41to14']\n",
      "token 3249 segment(s) ['3249']\n",
      "token DOLLARS💰RICHER segment(s) ['dollars', 'richer']\n",
      "token assaulter segment(s) ['assault', 'er']\n",
      "token goAngelo segment(s) ['go', 'angelo']\n",
      "token chickenshit segment(s) ['chickenshit']\n",
      "token Omarosas segment(s) ['omarosa', 's']\n",
      "token toTrump segment(s) ['to', 'trump']\n",
      "token yearsIve segment(s) ['years', 'ive']\n",
      "token PEAC segment(s) ['peac']\n",
      "token HollyRod segment(s) ['holly', 'rod']\n",
      "token loadedshould segment(s) ['loaded', 'should']\n",
      "token murphy13 segment(s) ['murphy', '13']\n",
      "token trumpdoral segment(s) ['trump', 'doral']\n",
      "token FighterJet segment(s) ['fighter', 'jet']\n",
      "token 910000 segment(s) ['910000']\n",
      "token kir96 segment(s) ['kir96']\n",
      "token Hagels segment(s) ['hagel', 's']\n",
      "token JayZ segment(s) ['jay', 'z']\n",
      "token figthing segment(s) ['fig', 'thing']\n",
      "token substantialy segment(s) ['substantial', 'y']\n",
      "token 31334 segment(s) ['31334']\n",
      "token fearwe segment(s) ['fear', 'we']\n",
      "token ’08 segment(s) ['08']\n",
      "token jobsborder segment(s) ['jobs', 'border']\n",
      "token NobamaCare segment(s) ['no', 'bama', 'care']\n",
      "token 00PM segment(s) ['00pm']\n",
      "token Buseyisms segment(s) ['busey', 'is', 'ms']\n",
      "token interivewed segment(s) ['inter', 'ive', 'wed']\n",
      "token RushLimbaugh segment(s) ['rush', 'limbaugh']\n",
      "token waythe segment(s) ['way', 'the']\n",
      "token NFLor segment(s) ['nfl', 'or']\n",
      "token 770M segment(s) ['770m']\n",
      "token recinded segment(s) ['rec', 'in', 'ded']\n",
      "token 42k segment(s) ['42k']\n",
      "token WPOST segment(s) ['w', 'post']\n",
      "token OPECs segment(s) ['opec', 's']\n",
      "token Honolulus segment(s) ['honolulu', 's']\n",
      "token 557B segment(s) ['557b']\n",
      "token TwitLonger segment(s) ['twit', 'longer']\n",
      "token 6700000 segment(s) ['6700000']\n",
      "token 432000 segment(s) ['432000']\n",
      "token 1Trillion segment(s) ['1', 'trillion']\n",
      "token Jusr segment(s) ['j', 'usr']\n",
      "token thered segment(s) ['the', 'red']\n",
      "token Jamess segment(s) ['james', 's']\n",
      "token 6000000 segment(s) ['6000000']\n",
      "token OMalley segment(s) ['omalley']\n",
      "token Lubitz segment(s) ['lubitz']\n",
      "token awaywe segment(s) ['a', 'way', 'we']\n",
      "token 242K segment(s) ['242k']\n",
      "token 8pmE segment(s) ['8pme']\n",
      "token MakeAmericaGreatAgain segment(s) ['make', 'america', 'great', 'again']\n",
      "token Bronxs segment(s) ['bronx', 's']\n",
      "token BreitbartNews segment(s) ['breitbart', 'news']\n",
      "token 831B segment(s) ['831b']\n",
      "token deannayoung segment(s) ['deanna', 'young']\n",
      "token Greeces segment(s) ['greece', 's']\n",
      "token 800B segment(s) ['800b']\n",
      "token ISIL segment(s) ['is', 'il']\n",
      "token AlabamaTickets segment(s) ['alabama', 'tickets']\n",
      "token Pacificas segment(s) ['pacific', 'as']\n",
      "token Shouldnt segment(s) ['shouldnt']\n",
      "token Morsis segment(s) ['mors', 'is']\n",
      "token mattressserta segment(s) ['mattress', 'serta']\n",
      "token Crysta segment(s) ['crysta']\n",
      "token ❌LIBYA❌SYRIA❌IRAN❌IRAQ❌ASIA segment(s) ['libya', 'syria', 'iran', 'iraq', 'asia']\n",
      "token Eyesa segment(s) ['eyes', 'a']\n",
      "token difficultmay segment(s) ['difficult', 'may']\n",
      "token discriptive segment(s) ['di', 'script', 'ive']\n",
      "token 45pm segment(s) ['45pm']\n",
      "token CBSWashDC segment(s) ['cbs', 'wash', 'dc']\n",
      "token rdy segment(s) ['rdy']\n",
      "token Clownstick segment(s) ['clown', 'stick']\n",
      "token 99M segment(s) ['99m']\n",
      "token retweeted segment(s) ['re', 'tweet', 'ed']\n",
      "token abirpatel segment(s) ['abir', 'patel']\n",
      "token NAFW segment(s) ['na', 'fw']\n",
      "token bypoliticians segment(s) ['by', 'politicians']\n",
      "token Superpac segment(s) ['super', 'pac']\n",
      "token supplymaking segment(s) ['supply', 'making']\n",
      "token byTrump segment(s) ['by', 'trump']\n",
      "token 2004📈 segment(s) ['2004']\n",
      "token 3ssaf segment(s) ['3ssaf']\n",
      "token 3263 segment(s) ['3263']\n",
      "token JohnDaly segment(s) ['john', 'daly']\n",
      "token PIVOT❌RUSSIAN segment(s) ['pivot', 'russian']\n",
      "token 120000 segment(s) ['120000']\n",
      "token 21T segment(s) ['21t']\n",
      "token RUBIOS segment(s) ['rubio', 's']\n",
      "token 🇺🇸🇵🇱 segment(s) []\n",
      "token Melanias segment(s) ['melani', 'as']\n",
      "token wbtw segment(s) ['w', 'btw']\n",
      "token Saeeds segment(s) ['saeed', 's']\n",
      "token Metropolitian segment(s) ['metropolitian']\n",
      "token Westchesters segment(s) ['westchester', 's']\n",
      "token unclassify segment(s) ['un', 'classify']\n",
      "token 9’s segment(s) ['9s']\n",
      "token 🇺🇸🇰🇷 segment(s) []\n",
      "token toooo segment(s) ['to', 'ooo']\n",
      "token Scheiderman segment(s) ['scheider', 'man']\n",
      "token Supercommittee segment(s) ['super', 'committee']\n",
      "token inBuffalo segment(s) ['in', 'buffalo']\n",
      "token onshoring segment(s) ['on', 'shoring']\n",
      "token Africahas segment(s) ['africa', 'has']\n",
      "token ⬇ segment(s) []\n",
      "token mugg segment(s) ['mugg']\n",
      "token mboyle1 segment(s) ['mboyle1']\n",
      "token Snowdens segment(s) ['snowden', 's']\n",
      "token schlonged segment(s) ['schlong', 'ed']\n",
      "token 32417 segment(s) ['32417']\n",
      "token 130000 segment(s) ['130000']\n",
      "token grunch segment(s) ['grunch']\n",
      "token Keysyone segment(s) ['keys', 'y', 'one']\n",
      "token A-X segment(s) ['a', 'x']\n",
      "token 17T segment(s) ['17t']\n",
      "token 175000 segment(s) ['175000']\n",
      "token ODonnell segment(s) ['odonnell']\n",
      "token Detroits segment(s) ['detroit', 's']\n",
      "token fastbefore segment(s) ['fast', 'before']\n",
      "token 57728 segment(s) ['57728']\n",
      "token here➡️ segment(s) ['here']\n",
      "token 🇺🇸🇮🇪 segment(s) []\n",
      "token cties segment(s) ['c', 'ties']\n",
      "token 17500 segment(s) ['17500']\n",
      "token Linkins segment(s) ['link', 'ins']\n",
      "token 32º segment(s) ['32']\n",
      "token 7531 segment(s) ['7531']\n",
      "token AddressJoin segment(s) ['address', 'join']\n",
      "token THEGaryBusey segment(s) ['the', 'gary', 'busey']\n",
      "token 388000 segment(s) ['388000']\n",
      "token missteenusa segment(s) ['miss', 'teen', 'usa']\n",
      "token Harveymy segment(s) ['harvey', 'my']\n",
      "token Philadelphias segment(s) ['philadelphia', 's']\n",
      "token FoxNewsInsider segment(s) ['foxnews', 'insider']\n",
      "token 30pmE segment(s) ['30pme']\n",
      "token changeits segment(s) ['change', 'its']\n",
      "token 91RTG segment(s) ['91rtg']\n",
      "token Milania segment(s) ['milani', 'a']\n",
      "token AJPerez segment(s) ['aj', 'perez']\n",
      "token stockh segment(s) ['stock', 'h']\n",
      "token 🇺🇸🇵🇦➡️ segment(s) []\n",
      "token Petreus segment(s) ['petre', 'us']\n",
      "token wayso segment(s) ['way', 'so']\n",
      "token CutsSecurity segment(s) ['cuts', 'security']\n",
      "token covfefe segment(s) ['cov', 'fefe']\n",
      "token kimguilfoyle segment(s) ['kim', 'guilfoyle']\n",
      "token 113826 segment(s) ['113826']\n",
      "token whrn segment(s) ['whr', 'n']\n",
      "token Solyndras segment(s) ['so', 'lynd', 'ras']\n",
      "token 15Trillion segment(s) ['15', 'trillion']\n",
      "token Americans🇺🇸 segment(s) ['americans']\n",
      "token globeandmail segment(s) ['globe', 'and', 'mail']\n",
      "token Korea🇺🇸🇰🇷 segment(s) ['korea']\n",
      "token imcompetently segment(s) ['im', 'competently']\n",
      "token onrob segment(s) ['on', 'rob']\n",
      "token 30PM segment(s) ['30pm']\n",
      "token OmikronDreamer segment(s) ['omikron', 'dreamer']\n",
      "token STANDARD’ segment(s) ['standard']\n",
      "token CNNS segment(s) ['cnn', 's']\n",
      "token distrbution segment(s) ['distr', 'bution']\n",
      "token CPAC2013 segment(s) ['cpac2013']\n",
      "token 16500 segment(s) ['16500']\n",
      "token ➡️ segment(s) []\n",
      "token NATIONVideo segment(s) ['nation', 'video']\n",
      "token Michiga segment(s) ['michiga']\n",
      "token 2783 segment(s) ['2783']\n",
      "token ratingleaves segment(s) ['rating', 'leaves']\n",
      "token Daviss segment(s) ['davis', 's']\n",
      "token ourselvesnot segment(s) ['ourselves', 'not']\n",
      "token youve segment(s) ['youve']\n",
      "token 217Billion segment(s) ['217', 'billion']\n",
      "token perh segment(s) ['per', 'h']\n",
      "token 2years segment(s) ['2', 'years']\n",
      "token overtaxes segment(s) ['over', 'taxes']\n",
      "token CNBCs segment(s) ['cnbc', 's']\n",
      "token fundraises segment(s) ['fund', 'raises']\n",
      "token todaysgolferco segment(s) ['to', 'days', 'golfer', 'co']\n",
      "token u-n segment(s) ['un']\n",
      "token Egypts segment(s) ['egypt', 's']\n",
      "token TimeToGetTough segment(s) ['time', 'to', 'get', 'tough']\n",
      "token AEIs segment(s) ['ae', 'is']\n",
      "token MIXa49 segment(s) ['mixa49']\n",
      "token ObamaMr segment(s) ['obama', 'mr']\n",
      "token 132000 segment(s) ['132000']\n",
      "token BarnesandNoble segment(s) ['barnesandnoble']\n",
      "token 00amE segment(s) ['00ame']\n",
      "token FFCs segment(s) ['ff', 'cs']\n",
      "token toAspen segment(s) ['to', 'aspen']\n",
      "token 12400000 segment(s) ['12400000']\n",
      "token Jackies segment(s) ['jackies']\n",
      "token Anniv segment(s) ['anniv']\n",
      "token Alisyn segment(s) ['ali', 'syn']\n",
      "token sacrfices segment(s) ['sacr', 'fices']\n",
      "token Parlux segment(s) ['parlux']\n",
      "token victoryhas segment(s) ['victory', 'has']\n",
      "token 4real segment(s) ['4', 'real']\n",
      "token Zuckers segment(s) ['zucker', 's']\n",
      "token NYCs segment(s) ['nyc', 's']\n",
      "token 🇺🇸🇫🇷 segment(s) []\n",
      "token 7693 segment(s) ['7693']\n",
      "token Raddatzs segment(s) ['raddatz', 's']\n",
      "token £25M segment(s) ['25m']\n",
      "token PoliticsResponsible segment(s) ['politics', 'responsible']\n",
      "token 12months segment(s) ['12', 'months']\n",
      "token 30pm segment(s) ['30pm']\n",
      "token BHOs segment(s) ['bhos']\n",
      "token DailyCaller segment(s) ['daily', 'caller']\n",
      "token tempsand segment(s) ['temps', 'and']\n",
      "token 19017 segment(s) ['19017']\n",
      "token FundAnything segment(s) ['fund', 'anything']\n",
      "token Gruters segment(s) ['gru', 'ters']\n",
      "token locationsthan segment(s) ['locations', 'than']\n",
      "token 500B segment(s) ['500b']\n",
      "token bldgs segment(s) ['bldgs']\n",
      "token Vancouvers segment(s) ['vancouver', 's']\n",
      "token phoner segment(s) ['phone', 'r']\n",
      "token favorited segment(s) ['favorite', 'd']\n",
      "token 7533692 segment(s) ['7533692']\n",
      "token celebritys segment(s) ['celebritys']\n",
      "token 200000000 segment(s) ['200000000']\n",
      "token DSRL segment(s) ['d', 'srl']\n",
      "token 00A segment(s) ['00a']\n",
      "token Ebolas segment(s) ['ebola', 's']\n",
      "token 635000000 segment(s) ['635000000']\n",
      "token 7yr segment(s) ['7yr']\n",
      "token Obsma segment(s) ['obs', 'ma']\n",
      "token assults segment(s) ['as', 'sults']\n",
      "token 200000 segment(s) ['200000']\n",
      "token 20T segment(s) ['20t']\n",
      "token 93516 segment(s) ['93516']\n",
      "token Cruzhad segment(s) ['cruz', 'had']\n",
      "token cnsnews segment(s) ['cns', 'news']\n",
      "token leverageable segment(s) ['leverage', 'able']\n",
      "token 49500000 segment(s) ['49500000']\n",
      "token Joannah segment(s) ['joanna', 'h']\n",
      "token 2200000 segment(s) ['2200000']\n",
      "token 39M segment(s) ['39m']\n",
      "token 35000000 segment(s) ['35000000']\n",
      "token teabaggers segment(s) ['tea', 'baggers']\n",
      "token RussiaRussia segment(s) ['russia', 'russia']\n",
      "token asSecretary segment(s) ['as', 'secretary']\n",
      "token 8332000 segment(s) ['8332000']\n",
      "token 10T segment(s) ['10t']\n",
      "token Humas segment(s) ['hum', 'as']\n",
      "token Reed83 segment(s) ['reed83']\n",
      "token 296000 segment(s) ['296000']\n",
      "token nobodys segment(s) ['nobodys']\n",
      "token 00pmE segment(s) ['00pme']\n",
      "token 4yrs segment(s) ['4yrs']\n",
      "token backour segment(s) ['back', 'our']\n",
      "token Heilemanns segment(s) ['heilemann', 's']\n",
      "token Bday segment(s) ['bday']\n",
      "token 90T segment(s) ['90t']\n",
      "token usthen segment(s) ['us', 'then']\n",
      "token Tri16 segment(s) ['tri16']\n",
      "token Shulkins segment(s) ['s', 'hulk', 'ins']\n",
      "token Schlapp segment(s) ['schlapp']\n",
      "token Websta segment(s) ['web', 'sta']\n",
      "token 52500 segment(s) ['52500']\n",
      "token 150000000 segment(s) ['150000000']\n",
      "token 18142 segment(s) ['18142']\n",
      "token cuttingStock segment(s) ['cutting', 'stock']\n",
      "token Kasie segment(s) ['kasie']\n",
      "token roadhardmovie segment(s) ['road', 'hard', 'movie']\n",
      "token Universitys segment(s) ['university', 's']\n",
      "token Streetsense segment(s) ['street', 'sense']\n",
      "token Yankeed segment(s) ['yankee', 'd']\n",
      "token ABCCBS segment(s) ['abc', 'cbs']\n",
      "token ☑️Pass segment(s) ['pass']\n",
      "token Giaritelli segment(s) ['gia', 'ri', 'tell', 'i']\n",
      "token GOPLegacy segment(s) ['gop', 'legacy']\n",
      "token ivankatrump segment(s) ['ivanka', 'trump']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "token Oghene segment(s) ['og', 'hene']\n",
      "token Moneynews segment(s) ['money', 'news']\n",
      "token Gasparinos segment(s) ['gaspari', 'nos']\n",
      "token NYers segment(s) ['ny', 'ers']\n",
      "token Rowanne segment(s) ['row', 'anne']\n",
      "token Garnetts segment(s) ['garnett', 's']\n",
      "token OReilly segment(s) ['oreilly']\n",
      "token MAN’ segment(s) ['man']\n",
      "token 5000000 segment(s) ['5000000']\n",
      "token SoHi segment(s) ['so', 'hi']\n",
      "token leakin segment(s) ['leak', 'in']\n",
      "token roarwatch segment(s) ['roar', 'watch']\n",
      "token ⚾️ segment(s) []\n",
      "token Sanduskys segment(s) ['sandusky', 's']\n",
      "token Ocare segment(s) ['o', 'care']\n",
      "token 790k segment(s) ['790k']\n",
      "token Geronimos segment(s) ['geronimo', 's']\n",
      "token wouldve segment(s) ['wouldve']\n",
      "token 18290 segment(s) ['18290']\n",
      "token 33000 segment(s) ['33000']\n",
      "token richs segment(s) ['rich', 's']\n",
      "token ✅Lower segment(s) ['lower']\n",
      "token baaaaack segment(s) ['b', 'aaaa', 'ack']\n",
      "token Carolina✅Ohio segment(s) ['carolina', 'ohio']\n",
      "token 4992343 segment(s) ['4992343']\n",
      "token 300ft segment(s) ['300ft']\n",
      "token justnever segment(s) ['just', 'never']\n",
      "token SOTU segment(s) ['sotu']\n",
      "token SHR1 segment(s) ['shr1']\n",
      "token ✅200000 segment(s) ['200000']\n",
      "token placelocked segment(s) ['place', 'locked']\n",
      "token WMURs segment(s) ['w', 'murs']\n",
      "token Gadhafis segment(s) ['gadhafi', 's']\n",
      "token 79M segment(s) ['79m']\n",
      "token WatchdogTime segment(s) ['watchdog', 'time']\n",
      "token grandchildrens segment(s) ['grandchildren', 's']\n",
      "token 7291 segment(s) ['7291']\n",
      "token 23000 segment(s) ['23000']\n",
      "token TNGC segment(s) ['t', 'ngc']\n",
      "token YouTubes segment(s) ['you', 'tubes']\n",
      "token Strzok segment(s) ['str', 'zok']\n",
      "token Hoft segment(s) ['ho', 'ft']\n",
      "token theyve segment(s) ['theyve']\n",
      "token 🇺🇸🇨🇳 segment(s) []\n",
      "token jihadistst segment(s) ['jihadists', 't']\n",
      "token COLLECTION™ segment(s) ['collection']\n",
      "token TJOHNSON segment(s) ['t', 'johnson']\n",
      "token cokehead segment(s) ['coke', 'head']\n",
      "token 46000 segment(s) ['46000']\n",
      "token GeorgesNYC segment(s) ['georges', 'nyc']\n",
      "token POLLThank segment(s) ['poll', 'thank']\n",
      "token apprenticenbc segment(s) ['apprentice', 'nbc']\n",
      "token 119000 segment(s) ['119000']\n",
      "token ststement segment(s) ['st', 'stem', 'ent']\n",
      "token dennis82 segment(s) ['dennis', '82']\n",
      "token soler1 segment(s) ['soler1']\n",
      "token AMERICANS🇺🇸 segment(s) ['americans']\n",
      "token 476K segment(s) ['476k']\n",
      "token Mickelsons segment(s) ['mickelson', 's']\n",
      "token DePrimo segment(s) ['de', 'primo']\n",
      "token NEWSMAXs segment(s) ['newsmax', 's']\n",
      "token SuperPAC segment(s) ['super', 'pac']\n",
      "token 1970’s segment(s) ['1970s']\n",
      "token Dareing segment(s) ['dare', 'ing']\n",
      "token 269710 segment(s) ['269710']\n",
      "token America🇺🇸FOUR segment(s) ['america', 'four']\n",
      "token bball segment(s) ['bball']\n",
      "token 9pmE segment(s) ['9pme']\n",
      "token Ambs segment(s) ['ambs']\n",
      "token 63K segment(s) ['63k']\n",
      "token financeable segment(s) ['finance', 'able']\n",
      "token firstminister segment(s) ['first', 'minister']\n",
      "token 29T segment(s) ['29t']\n",
      "token Country’ segment(s) ['country']\n",
      "token XiJinping segment(s) ['xij', 'in', 'ping']\n",
      "token Brexit segment(s) ['br', 'exit']\n",
      "token 45AM segment(s) ['45am']\n",
      "token Talackova segment(s) ['t', 'a', 'lack', 'ova']\n",
      "token Colmery segment(s) ['colmer', 'y']\n",
      "token UNIVERSE® segment(s) ['universe']\n",
      "token Medicareand segment(s) ['medicare', 'and']\n",
      "token terroristAl segment(s) ['terrorist', 'al']\n",
      "token Renn1 segment(s) ['renn1']\n",
      "token LaVars segment(s) ['la', 'vars']\n",
      "token Repub segment(s) ['repub']\n",
      "token enterainment segment(s) ['enterainment']\n",
      "token Panettas segment(s) ['panetta', 's']\n",
      "token Councel segment(s) ['counce', 'l']\n",
      "token insecureits segment(s) ['insecure', 'its']\n",
      "token shouldve segment(s) ['shouldve']\n",
      "token Sohos segment(s) ['soho', 's']\n",
      "token 325000 segment(s) ['325000']\n",
      "token KatherineWebbs segment(s) ['katherine', 'webbs']\n",
      "token waitforit segment(s) ['wait', 'for', 'it']\n",
      "token somethiing segment(s) ['some', 'thi', 'ing']\n",
      "token intetesting segment(s) ['inte', 'testing']\n",
      "token 36489 segment(s) ['36489']\n",
      "token 24000 segment(s) ['24000']\n",
      "token 635M segment(s) ['635m']\n",
      "token JCOPE segment(s) ['j', 'cope']\n",
      "token priv segment(s) ['priv']\n",
      "token leightweight segment(s) ['leight', 'weight']\n",
      "token Beachs segment(s) ['beach', 's']\n",
      "token NFLs segment(s) ['nfl', 's']\n",
      "token Thsnk segment(s) ['th', 'snk']\n",
      "token Libyas segment(s) ['libya', 's']\n",
      "token 35pmE segment(s) ['35pme']\n",
      "token latenow segment(s) ['late', 'now']\n",
      "token at《1 segment(s) ['at1']\n",
      "token INDV segment(s) ['in', 'dv']\n",
      "token interestbut segment(s) ['interest', 'but']\n",
      "token dead400 segment(s) ['dead400']\n",
      "token 4787 segment(s) ['4787']\n",
      "token 2Trillion segment(s) ['2', 'trillion']\n",
      "token BSer segment(s) ['b', 'ser']\n",
      "token propertys segment(s) ['property', 's']\n",
      "token Youve segment(s) ['youve']\n",
      "token 109B segment(s) ['109b']\n",
      "token washpost segment(s) ['washpost']\n",
      "token themnot segment(s) ['them', 'not']\n",
      "token Giaccio segment(s) ['giac', 'cio']\n",
      "token Virnelli segment(s) ['vir', 'nelli']\n",
      "token yearTrump segment(s) ['year', 'trump']\n",
      "token NBCs segment(s) ['nbc', 's']\n",
      "token £250million segment(s) ['250', 'million']\n",
      "token 530000 segment(s) ['530000']\n",
      "token Romneycare segment(s) ['romney', 'care']\n",
      "token Chance’ segment(s) ['chance']\n",
      "token Virginiais segment(s) ['virginia', 'is']\n",
      "token Spectac segment(s) ['spec', 'tac']\n",
      "token Average223 segment(s) ['average', '223']\n",
      "token 50OOO segment(s) ['50ooo']\n",
      "token contractionclose’s segment(s) ['contraction', 'closes']\n",
      "token ’09 segment(s) ['09']\n",
      "token Jibz segment(s) ['jib', 'z']\n",
      "token 300000 segment(s) ['300000']\n",
      "token drugrunners segment(s) ['drug', 'runners']\n",
      "token 1pmE segment(s) ['1pme']\n",
      "token CASSmoney segment(s) ['cass', 'money']\n",
      "token 4331 segment(s) ['4331']\n",
      "token charitys segment(s) ['charity', 's']\n",
      "token Odomirok segment(s) ['odom', 'irok']\n",
      "token Dummythanks segment(s) ['dummy', 'thanks']\n",
      "token 12T segment(s) ['12t']\n",
      "token 21456 segment(s) ['21456']\n",
      "token phoners segment(s) ['phone', 'rs']\n",
      "token Miamiis segment(s) ['miami', 'is']\n",
      "token ✅Unemployment segment(s) ['unemployment']\n",
      "token Way’ segment(s) ['way']\n",
      "token Selfies segment(s) ['self', 'ies']\n",
      "token Uptigrove segment(s) ['up', 'ti', 'grove']\n",
      "token 7T segment(s) ['7t']\n",
      "token Dornsife segment(s) ['dornsife']\n",
      "token 00am segment(s) ['00am']\n",
      "token Sexter segment(s) ['sex', 'ter']\n",
      "token Careful’ segment(s) ['careful']\n",
      "token 3Billion segment(s) ['3', 'billion']\n",
      "token ChrisCassidy segment(s) ['chris', 'cassidy']\n",
      "token 👍 segment(s) []\n",
      "token 500Billion segment(s) ['500', 'billion']\n",
      "token NYPost segment(s) ['ny', 'post']\n",
      "token Enq segment(s) ['enq']\n",
      "token motheras segment(s) ['mother', 'as']\n",
      "token 00pmEST segment(s) ['00pmest']\n",
      "token msccc segment(s) ['ms', 'ccc']\n",
      "token Amonix segment(s) ['a', 'monix']\n",
      "token baaack segment(s) ['baaack']\n",
      "token wifes segment(s) ['wifes']\n",
      "token 40000000 segment(s) ['40000000']\n",
      "token Ragazine segment(s) ['ragazine']\n",
      "token Sertas segment(s) ['sert', 'as']\n",
      "token reputatio segment(s) ['reputa', 'tio']\n",
      "token TONIGHTS segment(s) ['to', 'nights']\n",
      "token 16977 segment(s) ['16977']\n",
      "token Strassel segment(s) ['strasse', 'l']\n",
      "token DJT segment(s) ['djt']\n",
      "token McEnany segment(s) ['mc', 'en', 'any']\n",
      "token March2013 segment(s) ['march', '2013']\n",
      "token Yawwp segment(s) ['ya', 'wwp']\n",
      "token 7yrs segment(s) ['7yrs']\n",
      "token NeckNJ segment(s) ['neck', 'nj']\n",
      "token INCR segment(s) ['incr']\n",
      "token 100′ segment(s) ['100']\n",
      "token Trump® segment(s) ['trump']\n",
      "token MichaelCaputo segment(s) ['michael', 'caputo']\n",
      "token Concentrationassessment segment(s) ['concentration', 'assessment']\n",
      "token artAndy segment(s) ['art', 'andy']\n",
      "token HER🇺🇸Good segment(s) ['her', 'good']\n",
      "token storywith segment(s) ['story', 'with']\n",
      "token £63m segment(s) ['63m']\n",
      "token itiveness segment(s) ['i', 'tiveness']\n",
      "token CarolinaTed segment(s) ['carolina', 'ted']\n",
      "token belileve segment(s) ['be', 'lil', 'eve']\n",
      "token DrudgeTime segment(s) ['drudge', 'time']\n",
      "token 38000 segment(s) ['38000']\n",
      "token Reince segment(s) ['re', 'ince']\n",
      "token 51yrs segment(s) ['51yrs']\n",
      "token Looya segment(s) ['looy', 'a']\n",
      "token hesitatingtherefore segment(s) ['hesitating', 'therefore']\n",
      "token Broadcoms segment(s) ['broadcom', 's']\n",
      "token LyinTed segment(s) ['lyin', 'ted']\n",
      "token PPVs segment(s) ['pp', 'vs']\n",
      "token shouldnt segment(s) ['shouldnt']\n",
      "token 100yrs segment(s) ['100yrs']\n",
      "token CUTCAP segment(s) ['cut', 'cap']\n",
      "token commmitment segment(s) ['comm', 'mitment']\n",
      "token MoneyRacks segment(s) ['money', 'racks']\n",
      "token Names’ segment(s) ['names']\n",
      "token linemy segment(s) ['line', 'my']\n",
      "token Gretas segment(s) ['greta', 's']\n",
      "token Ivankas segment(s) ['ivanka', 's']\n",
      "token 147M segment(s) ['147m']\n",
      "token FOX2 segment(s) ['fox2']\n",
      "token JusticeDepartment segment(s) ['justice', 'department']\n",
      "token ☑️Keep segment(s) ['keep']\n",
      "token Wisconsins segment(s) ['wisconsin', 's']\n",
      "token Hvidston segment(s) ['h', 'vids', 'ton']\n",
      "token ’13 segment(s) ['13']\n",
      "token elememt segment(s) ['elem', 'emt']\n",
      "token NYJets segment(s) ['ny', 'jets']\n",
      "token Need’ segment(s) ['need']\n",
      "token lossshe segment(s) ['loss', 'she']\n",
      "token 63000000 segment(s) ['63000000']\n",
      "token Menie segment(s) ['men', 'ie']\n",
      "token Dursts segment(s) ['durst', 's']\n",
      "token Secretaty segment(s) ['secret', 'at', 'y']\n",
      "token K9s segment(s) ['k9s']\n",
      "token hadnt segment(s) ['hadnt']\n",
      "token 283938 segment(s) ['283938']\n",
      "token daddys segment(s) ['daddys']\n",
      "token cohesivness segment(s) ['co', 'he', 'siv', 'ness']\n",
      "token EO1 segment(s) ['eo1']\n",
      "token Westcot segment(s) ['west', 'cot']\n",
      "token Lettermans segment(s) ['letterman', 's']\n",
      "token cont➡️ segment(s) ['cont']\n",
      "token Cruzs segment(s) ['cruz', 's']\n",
      "token 7943 segment(s) ['7943']\n",
      "token Leaguewhose segment(s) ['league', 'whose']\n",
      "token Edddie24 segment(s) ['edddie24']\n",
      "token Garys segment(s) ['garys']\n",
      "token Shuette segment(s) ['shu', 'ette']\n",
      "token McManamon segment(s) ['mcmanamon']\n",
      "token Trumpocrats segment(s) ['trump', 'oc', 'rats']\n",
      "token 🇺🇸🇯🇲 segment(s) []\n",
      "token Pennsylvanias segment(s) ['pennsylvania', 's']\n",
      "token 88921000 segment(s) ['88921000']\n",
      "token TOMORROWFletcher segment(s) ['tomorrow', 'fletcher']\n",
      "token slimebag segment(s) ['slime', 'bag']\n",
      "token fieldand segment(s) ['field', 'and']\n",
      "token 4155 segment(s) ['4155']\n",
      "token 5841 segment(s) ['5841']\n",
      "token 766000 segment(s) ['766000']\n",
      "token Flashbk segment(s) ['flash', 'bk']\n",
      "token 50678 segment(s) ['50678']\n",
      "token Northamwho segment(s) ['northam', 'who']\n",
      "token Grandstanders segment(s) ['grand', 'standers']\n",
      "token 🇺🇸🇳🇬 segment(s) []\n",
      "token AmericaExecutive segment(s) ['america', 'executive']\n",
      "token 650000 segment(s) ['650000']\n",
      "token WAPO segment(s) ['wapo']\n",
      "token McCulloughFmr segment(s) ['mccullough', 'fmr']\n",
      "token DACA segment(s) ['daca']\n",
      "token Watch➡️ segment(s) ['watch']\n",
      "token ☑️ segment(s) []\n",
      "token Belichicks segment(s) ['belichick', 's']\n",
      "token hardagainst segment(s) ['hard', 'against']\n",
      "token figurewants segment(s) ['figure', 'wants']\n",
      "token Spitzers segment(s) ['spitzer', 's']\n",
      "token calenders segment(s) ['calenders']\n",
      "token Stirewalt segment(s) ['stir', 'ewalt']\n",
      "token Dumbass segment(s) ['dumbass']\n",
      "token 972000 segment(s) ['972000']\n",
      "token RomneyCare segment(s) ['romney', 'care']\n",
      "token 01AM segment(s) ['01am']\n",
      "token 1950’s segment(s) ['1950s']\n",
      "token 94000 segment(s) ['94000']\n",
      "token congrat segment(s) ['congrat']\n",
      "token Tiffanys segment(s) ['tiffanys']\n",
      "token Canada🇨🇦and segment(s) ['canada', 'and']\n",
      "token 3203 segment(s) ['3203']\n",
      "token 566000 segment(s) ['566000']\n",
      "token 250000 segment(s) ['250000']\n",
      "token 1000000 segment(s) ['1000000']\n",
      "token 7326 segment(s) ['7326']\n",
      "token Fan24 segment(s) ['fan24']\n",
      "token firingsTrump segment(s) ['firings', 'trump']\n",
      "token Scaramucci segment(s) ['scara', 'mucci']\n",
      "token 88000 segment(s) ['88000']\n",
      "token IstanbulTurkey segment(s) ['istanbul', 'turkey']\n",
      "token temperment segment(s) ['temperment']\n",
      "token 900Billion segment(s) ['900', 'billion']\n",
      "token OHPA segment(s) ['oh', 'pa']\n",
      "token Samsungs segment(s) ['samsung', 's']\n",
      "token dealsthey segment(s) ['deals', 'they']\n",
      "token CubaVideo segment(s) ['cuba', 'video']\n",
      "token The🇺🇸has segment(s) ['the', 'has']\n",
      "token WaPo segment(s) ['wapo']\n",
      "token 30AM segment(s) ['30am']\n",
      "token 00AM segment(s) ['00am']\n",
      "token ✔️ segment(s) []\n",
      "token deliberatuon segment(s) ['de', 'libera', 'tu', 'on']\n",
      "token 20000📈21000📈22000📈 segment(s) ['200002100022000']\n",
      "token 6125 segment(s) ['6125']\n",
      "token fanastic segment(s) ['fan', 'astic']\n",
      "token longislnd segment(s) ['long', 'islnd']\n",
      "token Hodnett segment(s) ['hodnett']\n",
      "token 75000 segment(s) ['75000']\n",
      "token betwe segment(s) ['betwe']\n",
      "token veryvery segment(s) ['very', 'very']\n",
      "token HOAXSTERS segment(s) ['hoax', 'sters']\n",
      "token SUPERPACS segment(s) ['super', 'pacs']\n",
      "token tropically segment(s) ['tropically']\n",
      "token Andey segment(s) ['and', 'ey']\n",
      "token Syrias segment(s) ['syria', 's']\n",
      "token 🇺🇸🇦🇺Press segment(s) ['press']\n",
      "token Danyella segment(s) ['danyel', 'la']\n",
      "token 179249 segment(s) ['179249']\n",
      "token nauseas segment(s) ['nausea', 's']\n",
      "token 31000 segment(s) ['31000']\n",
      "token Johnson68 segment(s) ['johnson', '68']\n",
      "token Fheile segment(s) ['f', 'he', 'ile']\n",
      "token Vescio segment(s) ['ves', 'cio']\n",
      "token anybodys segment(s) ['anybody', 's']\n",
      "token RichieRich segment(s) ['richie', 'rich']\n",
      "token Wiedefeld segment(s) ['wie', 'de', 'feld']\n",
      "token ridicilous segment(s) ['rid', 'ici', 'lous']\n",
      "token Nationak segment(s) ['nation', 'ak']\n",
      "token additionno segment(s) ['addition', 'no']\n",
      "token Pennsylvaniaan segment(s) ['pennsylvania', 'an']\n",
      "token 500000 segment(s) ['500000']\n",
      "token theyd segment(s) ['they', 'd']\n",
      "token Cat41 segment(s) ['cat41']\n",
      "token 13375 segment(s) ['13375']\n",
      "token Cokehead segment(s) ['coke', 'head']\n",
      "token 00pm segment(s) ['00pm']\n",
      "token ShouldTrumpRun segment(s) ['should', 'trump', 'run']\n",
      "token 32000 segment(s) ['32000']\n",
      "token Bernies segment(s) ['bernies']\n",
      "token 7777 segment(s) ['7777']\n",
      "token undet segment(s) ['und', 'et']\n",
      "token Rosechem1 segment(s) ['rosechem1']\n",
      "token 11pmE segment(s) ['11pme']\n",
      "token Rellim segment(s) ['rel', 'lim']\n",
      "token Berry87 segment(s) ['berry87']\n",
      "token Susterens segment(s) ['susteren', 's']\n",
      "token 180000 segment(s) ['180000']\n",
      "token YOUVE segment(s) ['youve']\n",
      "token the13th segment(s) ['the', '13th']\n",
      "token Schlonged segment(s) ['schlong', 'ed']\n",
      "token Worired segment(s) ['wor', 'i', 'red']\n",
      "token ‎In segment(s) ['in']\n",
      "token Manaforts segment(s) ['mana', 'forts']\n",
      "token Theyare segment(s) ['they', 'are']\n",
      "token historyis segment(s) ['history', 'is']\n",
      "token Nykea segment(s) ['ny', 'kea']\n",
      "token prople segment(s) ['prople']\n",
      "token Dorya segment(s) ['dory', 'a']\n",
      "token Lassner segment(s) ['lass', 'ner']\n",
      "token Waikikis segment(s) ['waikiki', 's']\n",
      "token Playef segment(s) ['playef']\n",
      "token TRILLIONAugust segment(s) ['trillion', 'august']\n",
      "token riggy segment(s) ['r', 'iggy']\n",
      "token 85B segment(s) ['85b']\n",
      "token DonaldTrump segment(s) ['donald', 'trump']\n",
      "token NationFull segment(s) ['nation', 'full']\n",
      "token Phoneix segment(s) ['phone', 'ix']\n",
      "token Chalian segment(s) ['chali', 'an']\n",
      "token Secy segment(s) ['secy']\n",
      "token Retweet segment(s) ['re', 'tweet']\n",
      "token ObamaCares segment(s) ['obama', 'cares']\n",
      "token 300B segment(s) ['300b']\n",
      "token 700000 segment(s) ['700000']\n",
      "token OCare segment(s) ['o', 'care']\n",
      "token evil➡️ segment(s) ['evil']\n",
      "token FakeNews segment(s) ['fake', 'news']\n",
      "token Warrendya segment(s) ['warren', 'dya']\n",
      "token 3000000 segment(s) ['3000000']\n",
      "token Tahmooressi segment(s) ['tah', 'moore', 'ssi']\n",
      "token 18T segment(s) ['18t']\n",
      "token 169B segment(s) ['169b']\n",
      "token Stevenswho segment(s) ['stevens', 'who']\n",
      "token 🎥 segment(s) []\n",
      "token 522000 segment(s) ['522000']\n",
      "token RedStateand segment(s) ['redstate', 'and']\n",
      "token 7310 segment(s) ['7310']\n",
      "token hustlerr segment(s) ['hustler', 'r']\n",
      "token thatwith segment(s) ['that', 'with']\n",
      "token ScienceGuy segment(s) ['science', 'guy']\n",
      "token Evanchos segment(s) ['evan', 'chos']\n",
      "token Ihave segment(s) ['i', 'have']\n",
      "token Zuckermans segment(s) ['zuckerman', 's']\n",
      "token NPRs segment(s) ['nprs']\n",
      "token Krzanich segment(s) ['krza', 'nich']\n",
      "token 20PM segment(s) ['20pm']\n",
      "token £24 segment(s) ['24']\n",
      "token securityISIS segment(s) ['security', 'isis']\n",
      "token BuseyLand segment(s) ['busey', 'land']\n",
      "token Nieporte segment(s) ['nie', 'porte']\n",
      "token ratingsreally segment(s) ['ratings', 'really']\n",
      "token Josh8J4 segment(s) ['josh8j4']\n",
      "token creati segment(s) ['creati']\n",
      "token margin13 segment(s) ['margin', '13']\n",
      "token 30’s segment(s) ['30s']\n",
      "token i4u segment(s) ['i4u']\n",
      "token 295M segment(s) ['295m']\n",
      "token Parade® segment(s) ['parade']\n",
      "token Chozick segment(s) ['ch', 'ozick']\n",
      "token Pareene segment(s) ['p', 'are', 'ene']\n",
      "token 8T segment(s) ['8t']\n",
      "token Wikileakes segment(s) ['wiki', 'leak', 'es']\n",
      "token AGAIN🇺🇸 segment(s) ['again']\n",
      "token notMexico segment(s) ['not', 'mexico']\n",
      "token TrumpU segment(s) ['trump', 'u']\n",
      "token Schlaflys segment(s) ['schlafly', 's']\n",
      "token RapidsIA segment(s) ['rapids', 'i', 'a']\n",
      "token 4466 segment(s) ['4466']\n",
      "token basketcase segment(s) ['basketcase']\n",
      "token 120K segment(s) ['120k']\n",
      "token anyones segment(s) ['anyones']\n",
      "token 18000000 segment(s) ['18000000']\n",
      "token realdonaldtrump segment(s) ['real', 'donald', 'trump']\n",
      "token mtgs segment(s) ['mtgs']\n",
      "token Welfares segment(s) ['welfare', 's']\n",
      "token 47M segment(s) ['47m']\n",
      "token Postthis segment(s) ['post', 'this']\n",
      "token Griffith1 segment(s) ['griffith', '1']\n",
      "token bankruptcynow segment(s) ['bankruptcy', 'now']\n",
      "token shoker segment(s) ['sh', 'oker']\n",
      "token wholl segment(s) ['who', 'll']\n",
      "token Swisher69 segment(s) ['swisher', '69']\n",
      "token 16’s segment(s) ['16s']\n",
      "token Congressioal segment(s) ['congress', 'io', 'al']\n",
      "token ’12 segment(s) ['12']\n",
      "token numberd segment(s) ['number', 'd']\n",
      "token Utahs segment(s) ['utah', 's']\n",
      "token THEBillMcGee segment(s) ['the', 'bill', 'mcgee']\n",
      "token discriminatoryracist segment(s) ['discriminatory', 'racist']\n",
      "token decieved segment(s) ['decieved']\n",
      "token 071067 segment(s) ['071067']\n",
      "token Ricko segment(s) ['ricko']\n",
      "token deplorables segment(s) ['deplorable', 's']\n",
      "token Pistorious segment(s) ['pistor', 'ious']\n",
      "token 81000 segment(s) ['81000']\n",
      "token 136260 segment(s) ['136260']\n",
      "token Tebows segment(s) ['te', 'bows']\n",
      "token FoxBusiness segment(s) ['fox', 'business']\n",
      "token Phadraig segment(s) ['pha', 'dr', 'aig']\n",
      "token hearby segment(s) ['hear', 'by']\n",
      "token continentally segment(s) ['continental', 'ly']\n",
      "token measurewait segment(s) ['measure', 'wait']\n",
      "token chidlren segment(s) ['chi', 'dlr', 'en']\n",
      "token Sotoro segment(s) ['so', 'toro']\n",
      "token Pageantwill segment(s) ['pageant', 'will']\n",
      "token her🇺🇸☑️Defend segment(s) ['her', 'defend']\n",
      "token TheScotsman segment(s) ['the', 'scotsman']\n",
      "token 🇺🇸🇪🇸 segment(s) []\n",
      "token launchhis segment(s) ['launch', 'his']\n",
      "token terminatrd segment(s) ['term', 'in', 'at', 'rd']\n",
      "token Skarlatos segment(s) ['s', 'karla', 'tos']\n",
      "token Keign segment(s) ['k', 'eign']\n",
      "token countr segment(s) ['countr']\n",
      "token Americ segment(s) ['americ']\n",
      "token HealthcareTax segment(s) ['healthcare', 'tax']\n",
      "token MailOnline segment(s) ['mail', 'online']\n",
      "token 68M segment(s) ['68m']\n",
      "token runnot segment(s) ['run', 'not']\n",
      "token 487K segment(s) ['487k']\n",
      "token Barnini segment(s) ['barn', 'ini']\n",
      "token bales1 segment(s) ['bales1']\n",
      "token 10pE segment(s) ['10pe']\n",
      "token friends’ segment(s) ['friends']\n",
      "token justme segment(s) ['just', 'me']\n",
      "token O0 segment(s) ['o0']\n",
      "token theToday segment(s) ['the', 'today']\n",
      "token intetview segment(s) ['int', 'et', 'view']\n",
      "token 31951 segment(s) ['31951']\n",
      "token 80’s segment(s) ['80s']\n",
      "token 33428 segment(s) ['33428']\n",
      "token 79352 segment(s) ['79352']\n",
      "token sleazebags segment(s) ['sleaze', 'bags']\n",
      "token whove segment(s) ['who', 've']\n",
      "token totaly segment(s) ['totaly']\n",
      "token Parentsmake segment(s) ['parents', 'make']\n",
      "token Shulkin segment(s) ['s', 'hulk', 'in']\n",
      "token esxcited segment(s) ['esx', 'cited']\n",
      "token GoldmanVice segment(s) ['goldman', 'vice']\n",
      "token terifies segment(s) ['teri', 'fies']\n",
      "token grea segment(s) ['grea']\n",
      "token tixs segment(s) ['tixs']\n",
      "token sobby segment(s) ['sob', 'by']\n",
      "token duscussing segment(s) ['d', 'us', 'cussing']\n",
      "token EO2 segment(s) ['eo2']\n",
      "token Maitner segment(s) ['ma', 'it', 'ner']\n",
      "token ccolvinj segment(s) ['c', 'colvin', 'j']\n",
      "token Norks segment(s) ['nor', 'ks']\n",
      "token 2000000 segment(s) ['2000000']\n",
      "token hanson800 segment(s) ['hanson', '800']\n",
      "token Giovannis segment(s) ['giovanni', 's']\n",
      "token CharlottesvilleVirginia segment(s) ['charlottesville', 'virginia']\n",
      "token Sharpstache segment(s) ['sharp', 'stache']\n",
      "token 55198 segment(s) ['55198']\n",
      "token werent segment(s) ['werent']\n",
      "token windfarmsDoral segment(s) ['wind', 'farms', 'doral']\n",
      "token staion segment(s) ['staion']\n",
      "token Metalchick segment(s) ['metal', 'chick']\n",
      "token Newsweeksoon segment(s) ['newsweek', 'soon']\n",
      "token keynoting segment(s) ['key', 'noting']\n",
      "token likeTim segment(s) ['like', 'tim']\n",
      "token USA🇺🇸 segment(s) ['usa']\n",
      "token twit29 segment(s) ['twit29']\n",
      "token Huckebee segment(s) ['hucke', 'bee']\n",
      "token Stepaneks segment(s) ['stepanek', 's']\n",
      "token Abedinis segment(s) ['abedin', 'is']\n",
      "token StaysFresh segment(s) ['stays', 'fresh']\n",
      "token onUrgent segment(s) ['on', 'urgent']\n",
      "token jewlery segment(s) ['jewlery']\n",
      "token endbullet segment(s) ['end', 'bullet']\n",
      "token saywith segment(s) ['say', 'with']\n",
      "token BarackObama segment(s) ['barack', 'obama']\n",
      "token 12000000 segment(s) ['12000000']\n",
      "token 􏰀 segment(s) []\n",
      "token ReallyI segment(s) ['really', 'i']\n",
      "token impt segment(s) ['impt']\n",
      "token Broadwells segment(s) ['broad', 'wells']\n",
      "token Genall segment(s) ['gen', 'all']\n",
      "token onThursday segment(s) ['on', 'thursday']\n",
      "token ICYMI segment(s) ['icy', 'mi']\n",
      "token ❤ segment(s) []\n",
      "token 14076 segment(s) ['14076']\n",
      "token Zegarelli segment(s) ['zeg', 'arelli']\n",
      "token FailingNewYorkTimes segment(s) ['failing', 'new', 'york', 'times']\n",
      "token 7450 segment(s) ['7450']\n",
      "token VPE segment(s) ['vpe']\n",
      "token Connecticuts segment(s) ['connecticut', 's']\n",
      "token Repubicans segment(s) ['repubic', 'ans']\n",
      "token Bergdahls segment(s) ['bergdahl', 's']\n",
      "token advanage segment(s) ['advan', 'age']\n",
      "token Schreckenger segment(s) ['schreck', 'enger']\n",
      "token couses segment(s) ['co', 'uses']\n",
      "token 1250000 segment(s) ['1250000']\n",
      "token 894520 segment(s) ['894520']\n",
      "token exsist segment(s) ['exsist']\n",
      "token Lebrons segment(s) ['lebron', 's']\n",
      "token Brets segment(s) ['bret', 's']\n",
      "token Journos segment(s) ['journos']\n",
      "token Hispanically segment(s) ['hispanic', 'ally']\n",
      "token 190000 segment(s) ['190000']\n",
      "token 430M segment(s) ['430m']\n",
      "token 6T segment(s) ['6t']\n",
      "token 7205 segment(s) ['7205']\n",
      "token councel segment(s) ['counce', 'l']\n",
      "token EXPERIENCEVideo segment(s) ['experience', 'video']\n",
      "token Whiteish segment(s) ['white', 'ish']\n",
      "token 23000📈this segment(s) ['23000', 'this']\n",
      "token 2853 segment(s) ['2853']\n",
      "token ★★★★★ segment(s) []\n",
      "token Seans segment(s) ['seans']\n",
      "token Toures segment(s) ['tour', 'es']\n",
      "token iIPod segment(s) ['i', 'ipod']\n",
      "token watch🇺🇸☑️Protect segment(s) ['watch', 'protect']\n",
      "token bordersthey segment(s) ['borders', 'they']\n",
      "token betw segment(s) ['betw']\n",
      "token Paternos segment(s) ['paterno', 's']\n",
      "token Blackdog segment(s) ['blackdog']\n",
      "token services12 segment(s) ['services', '12']\n",
      "token handbasket’ segment(s) ['handbasket']\n",
      "token 8646551 segment(s) ['8646551']\n",
      "token Berghdal segment(s) ['bergh', 'dal']\n",
      "token Jepsens segment(s) ['jepsen', 's']\n",
      "token Scocca segment(s) ['scocca']\n",
      "token Goodnews segment(s) ['good', 'news']\n",
      "token nicca segment(s) ['nic', 'ca']\n",
      "token 4Trillion segment(s) ['4', 'trillion']\n",
      "token 55000 segment(s) ['55000']\n",
      "token Tahmooressis segment(s) ['tah', 'moore', 'ssis']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "token Habberman segment(s) ['hab', 'berman']\n",
      "token Afghanistant segment(s) ['afghanistan', 't']\n",
      "token Podestas segment(s) ['podesta', 's']\n",
      "token 19000 segment(s) ['19000']\n",
      "token CNNs segment(s) ['cnn', 's']\n",
      "token POLLTrump segment(s) ['poll', 'trump']\n",
      "token Wiliem98 segment(s) ['wiliem98']\n",
      "token welchs segment(s) ['welch', 's']\n",
      "token 10000000 segment(s) ['10000000']\n",
      "token SHOULDNT segment(s) ['shouldnt']\n",
      "token upcoming13th segment(s) ['upcoming', '13th']\n",
      "token 951957 segment(s) ['951957']\n",
      "token 📸 segment(s) []\n",
      "token TrumpCollections segment(s) ['trump', 'collections']\n",
      "token Jaynie segment(s) ['jay', 'nie']\n",
      "token marshallx segment(s) ['marshall', 'x']\n",
      "token unsexiest segment(s) ['un', 'sexiest']\n",
      "token 315000 segment(s) ['315000']\n",
      "token Kesslers segment(s) ['kessler', 's']\n",
      "token America🇺🇸 segment(s) ['america']\n",
      "token 250g segment(s) ['250g']\n",
      "token Melania® segment(s) ['melania']\n",
      "token behal segment(s) ['be', 'hal']\n",
      "token triky segment(s) ['tri', 'ky']\n",
      "token TheTrumpNetwork segment(s) ['the', 'trump', 'network']\n",
      "token fotgotten segment(s) ['fot', 'gotten']\n",
      "token 14789 segment(s) ['14789']\n",
      "token ’THE segment(s) ['the']\n",
      "token Smoooth segment(s) ['smoo', 'oth']\n",
      "token ✅Average segment(s) ['average']\n",
      "token 185000 segment(s) ['185000']\n",
      "token 197M segment(s) ['197m']\n",
      "token memembers segment(s) ['me', 'members']\n",
      "token destroyin segment(s) ['destroy', 'in']\n",
      "token AngieApon segment(s) ['angie', 'ap', 'on']\n",
      "token 51152 segment(s) ['51152']\n",
      "token Leightweight segment(s) ['leight', 'weight']\n",
      "token othersto segment(s) ['others', 'to']\n",
      "token Everybodys segment(s) ['everybodys']\n",
      "token Leisures segment(s) ['leisures']\n",
      "token 350000 segment(s) ['350000']\n",
      "token 288M segment(s) ['288m']\n",
      "token recordsI segment(s) ['records', 'i']\n",
      "token Smithsonians segment(s) ['smithsonian', 's']\n",
      "token l54 segment(s) ['l54']\n",
      "token 7242 segment(s) ['7242']\n",
      "token 9trillion segment(s) ['9', 'trillion']\n",
      "token uprighting segment(s) ['up', 'righting']\n",
      "token wayif segment(s) ['way', 'if']\n",
      "token contd segment(s) ['contd']\n",
      "token 48000 segment(s) ['48000']\n",
      "token Maimi segment(s) ['maim', 'i']\n",
      "token Janise segment(s) ['janis', 'e']\n",
      "token £100m segment(s) ['100m']\n",
      "token fictious segment(s) ['fictious']\n",
      "token Obamacares segment(s) ['obama', 'cares']\n",
      "token lilmonkeys segment(s) ['lil', 'monkeys']\n",
      "token Ahmadinejads segment(s) ['ahmadinejad', 's']\n",
      "token GolfSwag segment(s) ['golf', 'swag']\n",
      "token 505K segment(s) ['505k']\n",
      "token 10pmE segment(s) ['10pme']\n",
      "token 620000 segment(s) ['620000']\n",
      "token FrankG segment(s) ['frank', 'g']\n",
      "token Passov segment(s) ['pass', 'ov']\n",
      "token 5630 segment(s) ['5630']\n",
      "token Bagans segment(s) ['bag', 'ans']\n",
      "token 🇺🇸🇯🇴 segment(s) []\n",
      "token Crookeds segment(s) ['crooked', 's']\n",
      "token campaignhis segment(s) ['campaign', 'his']\n",
      "token ’14 segment(s) ['14']\n",
      "token Jareds segment(s) ['jared', 's']\n",
      "token GROWTH📈that segment(s) ['growth', 'that']\n",
      "token VoteStand segment(s) ['vote', 'stand']\n",
      "token 460M segment(s) ['460m']\n",
      "token 227000 segment(s) ['227000']\n",
      "token session➡️ segment(s) ['session']\n",
      "token 2016Scranton segment(s) ['2016', 'scranton']\n",
      "token Schnatte segment(s) ['sch', 'natte']\n",
      "token Ossoff segment(s) ['oss', 'off']\n",
      "token 9658 segment(s) ['9658']\n",
      "token ‏ segment(s) []\n",
      "token oclock segment(s) ['oclock']\n",
      "token 21points segment(s) ['21', 'points']\n",
      "token estab segment(s) ['estab']\n",
      "token smith9 segment(s) ['smith', '9']\n",
      "token 15T segment(s) ['15t']\n",
      "token seasonHence segment(s) ['season', 'hence']\n",
      "token anot segment(s) ['a', 'not']\n",
      "token 2764 segment(s) ['2764']\n",
      "token allys segment(s) ['all', 'ys']\n",
      "token manufa segment(s) ['manufa']\n",
      "token MPrabhu segment(s) ['m', 'prabhu']\n",
      "token a-Lagos segment(s) ['a', 'lagos']\n",
      "token Revolutionar segment(s) ['revolution', 'ar']\n",
      "token Jebs segment(s) ['jeb', 's']\n",
      "token Nationsl segment(s) ['nations', 'l']\n",
      "token militarys segment(s) ['military', 's']\n",
      "token Chicagos segment(s) ['chicago', 's']\n",
      "token pushpolls segment(s) ['push', 'polls']\n",
      "token Chapos segment(s) ['chap', 'os']\n",
      "token Thrursday segment(s) ['thru', 'rs', 'day']\n",
      "token 82800 segment(s) ['82800']\n",
      "token BarkleyFrance segment(s) ['barkley', 'france']\n",
      "token LiAngelo segment(s) ['li', 'angelo']\n",
      "token currrently segment(s) ['curr', 'rently']\n",
      "token Huntleys segment(s) ['huntley', 's']\n",
      "token JeffH segment(s) ['jeff', 'h']\n",
      "token 487B segment(s) ['487b']\n",
      "token Councilhe segment(s) ['council', 'he']\n",
      "token HillaryClinton segment(s) ['hillary', 'clinton']\n",
      "token buildilng segment(s) ['build', 'i', 'lng']\n",
      "token 750000 segment(s) ['750000']\n",
      "token PRODUCTIONUp📈7 segment(s) ['production', 'up7']\n",
      "token ✅Lean segment(s) ['lean']\n",
      "token WM23 segment(s) ['wm23']\n",
      "token 53000 segment(s) ['53000']\n",
      "token beepee2004 segment(s) ['beepee2004']\n",
      "token 10543 segment(s) ['10543']\n",
      "token 3557 segment(s) ['3557']\n",
      "token Schneidermans segment(s) ['schneiderman', 's']\n",
      "token Hamilton69 segment(s) ['hamilton', '69']\n",
      "token TrumpA segment(s) ['trump', 'a']\n",
      "token JVF segment(s) ['jv', 'f']\n",
      "token information’ segment(s) ['information']\n",
      "token Kelly1 segment(s) ['kelly', '1']\n",
      "token Indiv segment(s) ['indiv']\n",
      "token Address🇺🇸 segment(s) ['address']\n",
      "token ABCWashington segment(s) ['abc', 'washington']\n",
      "token Shouldve segment(s) ['shouldve']\n",
      "token dieing segment(s) ['dieing']\n",
      "token USChamber segment(s) ['us', 'chamber']\n",
      "token eveving segment(s) ['eve', 'ving']\n",
      "token 4676 segment(s) ['4676']\n",
      "token Tylar segment(s) ['tyla', 'r']\n",
      "token outnegotiate segment(s) ['out', 'negotiate']\n",
      "token 7884 segment(s) ['7884']\n",
      "token thisThursday segment(s) ['this', 'thursday']\n",
      "token 7540 segment(s) ['7540']\n",
      "token McCabes segment(s) ['mccabe', 's']\n",
      "token henceFake segment(s) ['hence', 'fake']\n",
      "token NBPC segment(s) ['nb', 'pc']\n",
      "token IraniansSyrians segment(s) ['iranians', 'syrians']\n",
      "token Thamooressi segment(s) ['tha', 'moore', 'ssi']\n",
      "token Hawaiis segment(s) ['hawaii', 's']\n",
      "token Obamatrade segment(s) ['obama', 'trade']\n",
      "token Entrepeneurs segment(s) ['entrepeneurs']\n",
      "token Hcare segment(s) ['h', 'care']\n",
      "token mustve segment(s) ['must', 've']\n",
      "token 🇺🇸🇨🇴Joint segment(s) ['joint']\n",
      "token loooose segment(s) ['loo', 'oose']\n",
      "token Rexnord segment(s) ['rex', 'nord']\n",
      "token Shadey segment(s) ['shade', 'y']\n",
      "token 150000 segment(s) ['150000']\n",
      "token ✅ segment(s) []\n",
      "token weeklystandard segment(s) ['weekly', 'standard']\n",
      "token 13K segment(s) ['13k']\n",
      "token Conf➡️ segment(s) ['conf']\n",
      "token RESET❌BENGHAZI segment(s) ['reset', 'benghazi']\n",
      "token hoaxsters segment(s) ['hoax', 'sters']\n",
      "token namedropping segment(s) ['name', 'dropping']\n",
      "token govtt segment(s) ['govt', 't']\n",
      "token OBAMATRADE segment(s) ['obama', 'trade']\n",
      "token Moscows segment(s) ['moscow', 's']\n",
      "token 46º segment(s) ['46']\n",
      "token PollThank segment(s) ['poll', 'thank']\n",
      "token Naghmeh segment(s) ['nag', 'h', 'meh']\n",
      "token 11am6 segment(s) ['11am6']\n",
      "token Libertys segment(s) ['liberty', 's']\n",
      "token ✅Eliminate segment(s) ['eliminate']\n",
      "token destabalize segment(s) ['des', 'tab', 'alize']\n",
      "token KenStarr segment(s) ['ken', 'starr']\n",
      "token 61RTG segment(s) ['61rtg']\n",
      "token ridicuous segment(s) ['rid', 'icu', 'ous']\n",
      "token Schneidean segment(s) ['schneid', 'ean']\n",
      "token 45pmE segment(s) ['45pme']\n",
      "token 7pmE segment(s) ['7pme']\n"
     ]
    }
   ],
   "source": [
    "tokens_novel_set = set( tokens_novel )\n",
    "\n",
    "for token in tokens_novel_set:\n",
    "    \n",
    "    print( \"token\", token, \"segment(s)\", segment( token ) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Look at Token Frequencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'António': 1,\n",
       " 'General': 82,\n",
       " 'Guterres': 3,\n",
       " 'Just': 392,\n",
       " 'Secretary': 61,\n",
       " 'UN': 23,\n",
       " 'met': 43,\n",
       " 'opentweetopen': 22322,\n",
       " 'who': 955,\n",
       " 'with': 2218}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_counts = collections.Counter( tokens_segmented )\n",
    "pairs = { k: word_counts[ k ] for k in list( word_counts )[ :10 ] }\n",
    "pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "opentweetopen:   22322\n",
      "closetweetclose:   22322\n",
      "endperiod:   13920\n",
      "attagopen:   12889\n",
      "attagclose:   12889\n",
      "the:   12635\n",
      "to:    9047\n",
      "tweetlink:    7295\n",
      "and:    6807\n",
      "contractionopen:    6402\n",
      "contractionclose:    6400\n",
      "a:    6332\n",
      "is:    6067\n",
      "of:    6028\n",
      "in:    5458\n",
      "I:    4267\n",
      "on:    3949\n",
      "for:    3935\n",
      "hashtagopen:    3470\n",
      "hashtagclose:    3470\n",
      "you:    3271\n",
      "be:    3262\n",
      "endexclamation:    3181\n",
      "will:    3027\n",
      "Trump:    2947\n",
      "The:    2892\n",
      "pauseemdash:    2863\n",
      "quoteopen:    2418\n",
      "that:    2398\n",
      "at:    2351\n"
     ]
    }
   ],
   "source": [
    "for word, count in word_counts.most_common( 30 ):\n",
    "    print( '%s: %7d' % ( word, count ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "i 6\n",
      "I 4267\n",
      "me 1178\n",
      "you 3271\n",
      "we 1063\n",
      "us 414\n",
      "US 746\n",
      "they 940\n",
      "them 482\n"
     ]
    }
   ],
   "source": [
    "print( \"i\", word_counts[ \"i\" ] )\n",
    "print( \"I\", word_counts[ \"I\" ] )\n",
    "print( \"me\", word_counts[ \"me\" ] )\n",
    "\n",
    "print( \"you\", word_counts[ \"you\" ] )\n",
    "\n",
    "print( \"we\", word_counts[ \"we\" ] )\n",
    "print( \"us\", word_counts[ \"us\" ] )\n",
    "print( \"US\", word_counts[ \"US\" ] )\n",
    "\n",
    "print( \"they\", word_counts[ \"they\" ] )\n",
    "print( \"them\", word_counts[ \"them\" ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.07.10 12:32\n",
      "Total Sequences: 541841\n",
      "2018.07.10 12:32\n",
      "Time to process: [0.2904362678527832] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "\n",
    "# organize into sequences of tokens\n",
    "sequence_len = 25 + 1\n",
    "sequences = []\n",
    "\n",
    "for i in range( sequence_len, len( tokens_segmented ) ):\n",
    "    \n",
    "    # select sequence of tokens\n",
    "    seq = tokens_segmented[ i - sequence_len:i ]\n",
    "    \n",
    "    # convert into a line\n",
    "    line = ' '.join( seq )\n",
    "    \n",
    "    # store\n",
    "    sequences.append( line )\n",
    "    \n",
    "print( 'Total Sequences: %d' % len( sequences ) )\n",
    "print_time( start_time, get_time() )\n",
    "\n",
    "# 2018.06.18 12:35\n",
    "# Total Sequences: 448557\n",
    "# 2018.06.18 12:35\n",
    "# Time to process: [0.24109315872192383] seconds\n",
    "\n",
    "# 2018.06.18 15:11\n",
    "# Total Sequences: 457475\n",
    "# 2018.06.18 15:11\n",
    "# Time to process: [0.27341151237487793] seconds\n",
    "\n",
    "# 2018.06.19 14:23\n",
    "# Total Sequences: 501061\n",
    "# 2018.06.19 14:23\n",
    "# Time to process: [0.28377485275268555] seconds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save tokens to file, one dialog per line\n",
    "def save_doc( lines, filename ):\n",
    "    \n",
    "    data = '\\n'.join( lines )\n",
    "    file = open( filename, 'w' )\n",
    "    file.write( data )\n",
    "    file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save sequences to file\n",
    "out_filename = \"sequences/trump-tweets-sequences-take-III.txt\"\n",
    "save_doc( sequences, out_filename )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_doc( filename ):\n",
    "    \n",
    "    # open the file as read only\n",
    "    file = open( filename, 'r' )\n",
    "    # read all text\n",
    "    text = file.read()\n",
    "    # close the file\n",
    "    file.close()\n",
    "    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['opentweetopen Just met with UN Secretary General António Guterres who is working hard to smartquoteopen Make the United Nations Great Again smartquoteclose When the UN does',\n",
       " 'Just met with UN Secretary General António Guterres who is working hard to smartquoteopen Make the United Nations Great Again smartquoteclose When the UN does more',\n",
       " 'met with UN Secretary General António Guterres who is working hard to smartquoteopen Make the United Nations Great Again smartquoteclose When the UN does more to',\n",
       " 'with UN Secretary General António Guterres who is working hard to smartquoteopen Make the United Nations Great Again smartquoteclose When the UN does more to solve',\n",
       " 'UN Secretary General António Guterres who is working hard to smartquoteopen Make the United Nations Great Again smartquoteclose When the UN does more to solve conflicts',\n",
       " 'Secretary General António Guterres who is working hard to smartquoteopen Make the United Nations Great Again smartquoteclose When the UN does more to solve conflicts around',\n",
       " 'General António Guterres who is working hard to smartquoteopen Make the United Nations Great Again smartquoteclose When the UN does more to solve conflicts around the',\n",
       " 'António Guterres who is working hard to smartquoteopen Make the United Nations Great Again smartquoteclose When the UN does more to solve conflicts around the world',\n",
       " 'Guterres who is working hard to smartquoteopen Make the United Nations Great Again smartquoteclose When the UN does more to solve conflicts around the world it',\n",
       " 'who is working hard to smartquoteopen Make the United Nations Great Again smartquoteclose When the UN does more to solve conflicts around the world it means']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "in_filename = \"sequences/trump-tweets-sequences-take-III.txt\"\n",
    "#doc = load_doc( in_filename )\n",
    "lines = load_doc( in_filename ).split( '\\n' )\n",
    "lines[ 0:10 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# seq_len_sum = 0;\n",
    "# line_len_dict = {}\n",
    "\n",
    "# for line in lines:\n",
    "    \n",
    "#     token_count = len( line.split( \" \" ) )\n",
    "#     seq_len_sum += token_count\n",
    "    \n",
    "#     if token_count in line_len_dict:\n",
    "#         line_len_dict[ token_count ] += 1\n",
    "#     else:\n",
    "#         line_len_dict[ token_count ] = 1\n",
    "\n",
    "# print( seq_len_sum / len( lines ) )\n",
    "# print( line_len_dict )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert Words to Index Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.07.10 12:32\n",
      "1\n",
      "19\n",
      "4\n",
      "opentweetopen\n",
      "hashtagopen\n",
      "attagopen\n",
      "vocab_size 24498\n",
      "2018.07.10 12:33\n",
      "Time to process: [11.936470031738281] seconds\n"
     ]
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "\n",
    "# integer encode sequences of words\n",
    "# tokenizer = Tokenizer( lower=False, filters=punctuation_string )\n",
    "tokenizer = Tokenizer( lower=False, filters=\"\" )\n",
    "\n",
    "tokenizer.fit_on_texts( lines )\n",
    "#print( \"sequences len *before* keras:\", len( sequences ) )\n",
    "sequences = tokenizer.texts_to_sequences( lines )\n",
    "#print( \"sequences len *after* keras:\", len( sequences ) )\n",
    "\n",
    "# elegant! https://stackoverflow.com/questions/41971587/how-to-convert-predicted-sequence-back-to-text-in-keras\n",
    "words_by_id = dict( map( reversed, tokenizer.word_index.items() ) ) \n",
    "\n",
    "# Check to and from of words and idx\n",
    "print( tokenizer.word_index[ \"opentweetopen\" ] )\n",
    "print( tokenizer.word_index[ \"hashtagopen\" ] )\n",
    "print( tokenizer.word_index[ \"attagopen\" ] )\n",
    "\n",
    "print( words_by_id[ 1 ] )\n",
    "print( words_by_id[ 19 ] )\n",
    "print( words_by_id[ 4 ] )\n",
    "\n",
    "# vocabulary size\n",
    "# discrepancy between these two lengths, of by a few words...\n",
    "#vocab_size = len( tokens_unique ) + 1\n",
    "vocab_size = len( tokenizer.word_index ) + 1\n",
    "print( \"vocab_size\", vocab_size )\n",
    "\n",
    "print_time( start_time, get_time() )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print( type( sequences ) )\n",
    "# print( type( sequences[ 0 ] ) )\n",
    "# print( type( sequences[ 0 ][ 0 ] ) )\n",
    "# print()\n",
    "# print( type( sequences ) )\n",
    "# print( type( sequences[ 0:1 ] ) )\n",
    "# print( type( sequences[ 0:1 ][ 0 ] ) )\n",
    "# print( type( np.array( sequences[ 0:1 ][ 0 ] ) ) )\n",
    "\n",
    "# print( np.array( sequences[ 0:1 ][ 0 ] ).shape )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # iterate lists of lists, and get lens\n",
    "# seq_len_sum = 0\n",
    "# seq_lens = []\n",
    "# seq_len_dict = {}\n",
    "\n",
    "# for seq in sequences:\n",
    "    \n",
    "#     seq_len_sum += len( seq )\n",
    "#     seq_lens.append( len( seq ) )\n",
    "    \n",
    "#     if len( seq ) in seq_len_dict:\n",
    "#         seq_len_dict[ len( seq ) ] += 1\n",
    "#     else:\n",
    "#         seq_len_dict[ len( seq ) ] = 1\n",
    "    \n",
    "# print( seq_len_sum / len( sequences ) ) \n",
    "# print( seq_len_dict )\n",
    "\n",
    "# # Was:\n",
    "# # 26.002879073381305\n",
    "# # {26: 540281, 27: 1560}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This KLUDGE Works, But Is No Longer Needed\n",
    "*(Using an empty filter string in the tokenizer obviates the need for this workaround)*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "# start_time = get_time()\n",
    "\n",
    "# sequences_foo = np.zeros( ( len( sequences ), len( sequences[ 0 ] ) ), dtype=int )\n",
    "# print( sequences_foo.shape )\n",
    "\n",
    "# row_count = 0\n",
    "\n",
    "# for row_idx, row in enumerate( sequences ):\n",
    "    \n",
    "#     for col_idx, col in enumerate( row ):\n",
    "        \n",
    "#         if col_idx > 26:\n",
    "#             print( row_idx, col_idx )\n",
    "#         else:\n",
    "#             sequences_foo[ row_idx, col_idx ] = col\n",
    "        \n",
    "        \n",
    "# #     if row_count == 30:\n",
    "# #         break\n",
    "# #     row_count += 1\n",
    "\n",
    "# print( sequences_foo.shape )\n",
    "# print( sequences_foo[ 0 ])\n",
    "\n",
    "# print_time( start_time, get_time() )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KLUDGE: We Need to Convert List of Lists into Array of Arrays\n",
    "_(Tokenizer's output is different when asked to leave case as is!?!)_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.07.10 12:33\n",
      "<class 'list'>\n",
      "<class 'numpy.ndarray'>\n",
      "(541841, 26)\n",
      "<class 'numpy.ndarray'>\n",
      "[    1   158  1342    33  2270   966   725 24497  9724    67    13   334\n",
      "   195     7    48   106     6   328  1776    55   117    50   300     6\n",
      "  2270   335]\n",
      "2018.07.10 12:33\n",
      "Time to process: [1.4083459377288818] seconds\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "22"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start_time = get_time()\n",
    "\n",
    "sequences_np = np.array( sequences )\n",
    "\n",
    "for i in range( len( sequences ) ):\n",
    "    sequences_np[ i ] = np.array( sequences[ i ] )\n",
    "\n",
    "print( type( sequences ) )\n",
    "print( type( sequences_np ) )\n",
    "print( sequences_np.shape )\n",
    "print( type( sequences_np[ 0 ] ) )\n",
    "print( sequences_np[ 0 ] )\n",
    "\n",
    "print_time( start_time, get_time() )\n",
    "\n",
    "sequences = None\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## KLUDGE: np.array slicing WAS/is fucked up!?!\n",
    "*(Was only b0rk3d when assuming texts_to_sequences(...) would return the same object types...)*\n",
    "It doesn't! That's a bug."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n",
      "(541841, 26)\n",
      "[    1   158  1342    33  2270   966   725 24497  9724    67    13   334\n",
      "   195     7    48   106     6   328  1776    55   117    50   300     6\n",
      "  2270   335]\n",
      "<class 'numpy.ndarray'>\n",
      "X.shape (541841, 25)\n",
      "y.shape (541841,)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "25"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print( type( sequences_np ))\n",
    "print( sequences_np.shape )\n",
    "\n",
    "print( sequences_np[ 0 ] )\n",
    "print( type( sequences_np[ 0 ] ) )\n",
    "\n",
    "# separate into input and output: for now it's 50 words input and 1 word output\n",
    "#sequences = np.array( sequences )\n",
    "X = sequences_np[ :,:-1 ] # all rows, from word 0 up to, but not including, the last word\n",
    "y = sequences_np[ :,-1 ]  # all rows, last word only\n",
    "\n",
    "# Throws MemoryError\n",
    "# https://stackoverflow.com/questions/46293734/memoryerror-in-keras-utils-np-utils-to-categorical\n",
    "#y = to_categorical( y, num_classes=vocab_size )\n",
    "print( \"X.shape\", X.shape )\n",
    "print( \"y.shape\", y.shape )\n",
    "\n",
    "seq_length = len( X[ 0 ] )\n",
    "seq_length\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Write X's and y's to Local File System\n",
    "dump( X, open( training_X_path, 'wb' ) )\n",
    "dump( y, open( training_y_path, 'wb' ) )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and Filter GloVe Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2018.07.10 12:33\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.+++++++++++++++++++++++++++++++..++++++.+++++++++++++++.++++.++++++++++++++++++++++++.+++++++.++++++.++++++++..++++++++++++++++++++++++++.+.+.+.+.+.+++++++++++.++++.+++++++++.++.+++++++++++++.++.+++.+++++++.++++.++++++.++++++++++++++++++.+++.+.++.+++++++.+..+.++++++++.++..++++++++.+++++++++++++++++++++++++++++.+++.+++++.+++.++++++.+.++++++..+++++++++++++..+++.+..+.++.+..+++.++++++++++.+++.+++.+++++++.+.++++.++++.++++++++++.++++..+++++++++++.+++++.++++++++++.++.+.+++.+++.+++.++.+.++.+++.++++..+.++...+++++++.++.+++++++..+..+++..+++.++.+...+.++++.++++....+++++.+.+++.++.+.+++.++++++++++++++++++++++..+.++++++.+.++..++.+++.+..+++....++..+++.++.+++++++.+++++++.++.+..+++++..++++++++..+..++++++++++++.+++++..++.+++.+.+.++++++++.+++.++++++++++.++++..+.+++++.++.+.++++.+..++++++.++.+++....+..++.+++++++.+++...+++++.+++..+++.++++++++++++.++.++++++++....++++..+.++++.+.+++.+++.++..++++.++.++...+++++.++++++.++.+.+.+++++++++.+++.+...++..++.+.+.+.++++..++.+++++++.++.++.+..++++++..++++..++++++.++++.++++...+.+++++++.+.+.+.++.++++++.+..++++.++.+.++++.++...++++.+++++.++...++++++..+++++++++++.+..++.+.+.+....+++.+++++.++++++++.++..++.++.++++.+.+....++.+..+.+++++.+.+++++++..+.+.+.+.+.++.++++.+++++.++.+++++++.....+++++++...+..++.++.+.+...++.+.+++++++..++++++..++.+.++++.+++++++..+.++++++++.+.+.+.+++..+.++++.+.++.+++.+++.++...+.+++.+.++++.+....+..+++++++++++++.+++++.+.+.+...++.+.+.++++++++++.+....+..++..+.+++++.++++.+++++++.++.++..++++..++++.+++..+.++.+++.+..++.++.+++++.++++.+++++.++.+++.++.++.+++++...++.+++++.+..+.+.++..+++++....+++++++.++++++..+.+.++.+.+++.+.+.+.++.+.++.+.++++...+.+.+++.+.++..++.++.+.+.+.++.+.++++.+.+++.+++++.++.+.++..+++.+.++...+++.+.+.....+++....++++++.+++.++.+..++.++++.++++..+++.++.+.++++.+++...++.++....+++++..+.+++.++.+..++.+.+.+++++.++.++.+..+.++++++...+++++...+.++.++.+++..++++.++++.+.+++++.+.+....+++++..++.++..+..+.++.++++.+++......+++.+...++..+..++.+..++..+...+++...+...++..+...+++++..++++..+++.+++..++.+++.+..+.+.+.+++..+.+..++..+++.++.+++...+.+..+..+++.+.++..++++.++.+..++....++.++++++++...+.++...+++.+++++.+..+..+.+++.+..+.+.++.++....++.+++++++..+.+.+.+....+.++...+.....+...+.++++++.++..++.+...+++++.+++.+.+++++.+.+.+++.+.+++..++.+.++.....+++....+..+..+..+++++.++.+..+........+....+++.+.+..++++.+....+++++...+..+++++.+.+.++++.+.+++.++....++++....+++.+.++++++.+..+.+.+.++.++++.+.....++.+......+++..++..+++.+.+...++.+.+.++...++..+....++....+.++..+.++.+...+++.+.++.++...++..+.++++....+.++.+++++++....+..+++..+++....+...++...+++.+...+..+.....+++.+..+..+++.+++.++++.+.++.++++++.+..++....++++++......+...++.......+++.+....+.......+.........+++.+.+.+...++....++++++..+.++..++++.++..++.++++..++++.+.+...+.+.++....+.+...++......+..+.....+....++..++....+.+..+..++++..+.....+.+++.+....++..+++.+++++.++.+...+..++...+..+++.+.++.++.+++...++.+......+..+++++.+.+...+...+++.+..+.+....+...+...+.++++....+.++..+....+.....+.+....+++........+.+..+.+..++.+.+.+++.+..+..+.++.....++...+....+....+.+..++..++++.+++.+.+....+.+.+++..++.+++.++........+..+.++++.++..++..+.+..+..+..+++..+.++......+.....+...++.+..+.++..+++....++...+...+++..++++.+.++..+.+.....++.++..+...+..++..+..+..+..+.+.....+..++...+.++.++..+++..+++.+....+++..+.....++..+.++.+....+.+++....+......+.+..+...+.+..++.....+++.+.++......+....+...++.++..++...+...+++...++..+...++.+...++..+.++.++.+...+.+.++.++++.........+.+..............+..+.+.....+.++.....+.+...++..........++...+...+.++.+++.+..+..+.+.+++..........+....++.+++.++..+++.+...+...++....+...+.++..+.+.+..++++..+++.+........+.+.++.+.+.+..+.+..+++...+...++..+.....+++..++.+.++..++.+.+.++.+...++.+.....++.+......+...+..+.+.+.......+.++++....++++.+...+.+...++.+++..+++.+..+++...++.+.+..+.....+++..+..+.++.++..+....+++..+++..+.++.+.++...+.+....++++...+..+.......++.++..+++..++.++..+.++.+...+.+++++....++...+....+++...++....+....++..+....++.....++.+..+..+..+++..+..+.++..+.....++.+.....+.+++.+..........+......+..+...+++...++++........++++.+......+..+....+..+...+.+......+.+.+.....++....+.++++++..++.+..+...++..+++..++..++...++.......+.++......+..+..+..++.++...++..+++.....++.+..++.+.....++.++...+.+.++...+...+.+..++++....+.+.............+...+++......+...++..+..+..++..+.+.+++..+...........++++..+.....+...+..+...+............+.++.+..+...+.......+.+.+...+..++..+...........+.++++.+.+++.+++++.+.......+....+.......+....+.+.......+..+...+.....++.........++..+....+....+.+...........++...++.+..+....++..++++++++..++....+.++......+.........+..++........++.+.++.+.....+.+..+...+..+...++.+.+....+..+..++.+.....++..+..+.++..+...+..+..+..+.....+.++.+..+.++.............++...............+...+..+++..+.+++.+.....+...+..+..+..++..++...+....+.+.........++..+.+..++.+..++..+.++.....++....++....+...+.....+....++++..+..++..+...........++..+.....++........+..+...+.+....+..++....+...+.++.++..+......++.+.+...+..++.++...+.+.+...+..++.+.+.+...+..........+...+.....+.+..++...+.....+..++..+.......+..++.+........++........++.......++....+......+..+.++...+.+...+.+.+..+.+++.+...+.+.+++..+.+.+.......++...++++...+....+....+.+..+....+....++...+....+....+.+.+++.+...+++......+.....++.++.+......+..+..+.+..+.+...+.....++..+...+..++...+.....+......+.+...........+.....+......++.+.+......+++.+...+++++..........+.+..++....+++......+...++.........+...........+....+..+.+........++.+.+....+..+.+++...++.....+++++.....+..........+.+.+..+..+.+...+.....+...............++.+.+....+.....+..+........++.+...+.++.+..+...+.........+..++.+...+++.....+.......+.+.++..+..+.+......++......+.+.........+....+++++.+....+.........+..++..++++.++...+..+..+.++.++.+...+.++.......+.+..+......+.+.......+..+.....+.+......+.+.++.+.+......+....+..+..++.+....+++........+++..+....+++...................+....+.......+..+++....++...++.....+......+......+....+..+.++..+..+.+..+..............+++++...+....++.+.........++...........+.+.............+.+....+.......++.......+..+++....+.+.....+...+..++.....+..++....+.+.++...+.........+....+.................+....+............++...........+..+.+.++..+.+...+..+.......+.+..+.+......+.........+...+..+.....+++...+....+.+++.+.........+.+..+.+..+....+.......++.+..+..++.+..................+..++..........++...........+.+.++.+........+.........+...++..+.....++.........++.+.....+.++..+...+....+..+...++....+.+......+............+++....+..........+.....+.............+.+.+.+...++..+.++.+.++....++....+.......+.....++..++...++....+.......+....++.++..+....++...+.+.....+..+.++...+.......+.........+.+.++............+++.....++.+.......+........+.+...++........++++....++.++.....+....+++...+...+..+.+.......+..+...+......+......+.....+....+...+.++.....+..+...+.....+...+.....+...............+....+......+...+.+.............+++...+.+...+++...+.+.+.+..+++..........++.........+........+..++......+.++.+++.+............++........+.+...+.+..+..+...+......+++..+......+.+.+.....+..+......+.......+....+.................++.....++...+......+.+.....+.+....+....+..+...+++..+...............+......+...........+.+..............+..+..+.+.+..+....++........+..+....+.+...+...+......++...+.+...++...+..........+.......+.+..+....+...+..+....+.........+......++...+....++..........+.......+.....+...+............++...........+.+...+..+....++.++.......+.++.+++...........+........+.....+....++...+............................+.........+.++......+......++....+.........+..+.+.+.+.+.+...+.........+.......+......................++++....+.....+..++..+........+.+..++.++....................+..+......++..........+....................+.+.+.+.......++.+....+...........+.+....++....+................+...+.......+....+.+..+.+.+........+................+...+.....++..+..+.......+.....................+..........+...+.......+.....+.++..+.+....................+...+..+.+....+................+......+......++....+..+.........+.+...........+.....+..+..+..................................++........++.........+...+.+.....+..+...+....+.+....+..+..+.+..........+........+.......+.+.+..+..+.....+...++...+......+.+.+........+..........+..........++...+.+...........++..+....+............+..+....++..........+...+.......................+.+..............+.........++..................................+................++................+....+...................+...+..+..........+..+...........+....+......+.........+..........+........+....+.......+.+...............+.....+....+.+......++..+.+.....+.........+..+.......++....+........+...+.....+.......+..+....+..........+.....+...........+.......+..+......+.+.....+.......+..++......+.+...++.+......+...+.++.+....++...+..............+..++.....+....+++.........+.....+..................+....................+......+.+..........+..................+..........+....................+.+.+...++.........+.....+..............+.+....+........+.......+.............+....+.+.++........+.+.+..+.+..+.....+.+..+....+......+......+....+..++.......+......+...............+...++..+......++...+..........+.....+........................+..+...+..+...+..+...................+.............+..................+...............+.........+++.+....+...+.++........+............+..+....+..+.........+.+.+....+..+.....+.............+......+....+.+............+...+....+.....................................+.........+........"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "....................+..+...++......+..++.+..+.+............+....++.......+..+....+...+.......+.+..+++....+.+.....++...++.+.....+.+..+.......+++..+............+.............+.....+...+.+++......+..+.+.......+.................+..................++.++..+..............+...............+.............+..+......++..........................................+...................+.....+.....+.................+.......+.......+++.......++......................+.......+...+......+......+.................+.....+..+........+...+......++............+............+.........+.+.......+...+.......+.+......+..+.+...........+..+..........+.+............+.........+......+....+....+...........+......++..+.......+.+................+............+.......+........+.+..+......+....+...+.+...++....+........+..+.......+......+......+.....+..............++...+...........+..++..............................+....+...+...........+..+...+.+..........+......+....+...........+.+.....+..+..........................+.................+..........+..........+..+.....+++......................+............+...+.+.....+.......+.............++.......+...+.......+............++...+............++.............+.........+...........+........+..+.........+...+..+..........+.......++.+.+......+................+........+.......++...........+..+...+...+......++.....+....................+..........................+..............+.............+.......++....+..+..........+.......+......+..........+...++.+........+.+.+.....+........+...............+.........+......+...+..........+...+.....+..+.+.....+......................+.......+....+.....+.............+...+................+.....+....+...........+.........+..........++..+............................................+...+....+.....+........+...........................+...+.......+.................+.....................++................+.................+.............++....+............+....+.........................+.++....+........+.........+....+.............+....+....................+.....+...+..........+....+...................+........++.......+............................+.+........+...........+...+.+...............+............+...............+...........+..........+.............+..++........+......................+..............+.......+........+..........+.........+............................................++.................+.....................+.........+.....+........+...............+...++.+..+.......+..+......+.....+..++..+......+.+........................+.............+..............+.+................+....+.+..+.........................+.......+..+.+...............+.............+.......................+..+...+............+...........+.+.....+...+.....+....+..+.........+..+....+........+.................+..+...+............+.........+.+..+....+............+.........................+.......+............+.+.........................+..................+.......+.............................................+....................+....+....................+............................+.+....++.........................+........+...++........+...........................+..........+........+.....+...............................................+....+......+.+...+........+......+........+..............+...+.....+.+..........+..+.+.......+...+..........+.............+.................+.+...........+.........+...+.+....+...............+...+.+.........+................................................+.+.+.+.....+.....+....++.......................+......................+.................................+..+...+....+...........++......+..........+.......++.........................+............+...+........+..+...............+.............+..........+........+.........+........+...................+................................+.+....+...+.+..+.+..................+....+...................+....+....................+................+.......+...+......+.+................+.......+..........+................+.........+..+................+................+...+..+.....+...+.....+.......+....+.+..+......+....+.+..+................................++........+..+......++........+.....+.........+.......+.....................................+.....+..............................+.......+...+........................+......+......................+.+...........+..........+.+.........+.......+..............................+.+...........+.+.........+..............................+....+......................+........+..................+.........+.....................+...................+.........+..++.......+...............+.......................................+..........................................+.........+......+.......+.......+......................................................+.....+.............+...++....++......................................................+.........+.......+......+.............................+....+....................................+....+.......+.....................+..........+............................................................................+.................................+.....+....+...................+..........................+.........................++....+.........++..+..........++..............+..+...............+.....................................................................................+...+.....+.......+.........+.....................+......+........+.........+...+..........+....+........+...+.......+............................+...........................+.........+.+.....+.............+........+............................+............+......++.....++.............+............................+.......+.......+.....+.......+.........................................................+............+....+...............+........++......................+.............................+..............++....+...+...+................+........+.............+......+......................+....................................................+...............+....+..................+.........+......+.......+......+....+...............................................................+........................+................+...........................................................+........+....................+................++......................+............+.........+...................+....+..............++.+.+....+..+...........+................+......+.....+....+..................+...+..+.....+.........+...+...+........................................................+.................+..........+.+................................+...........................................................................+..........................+......+...............+...............++.........+....+.....................+..............+.......+.....................+...........................................+............................+..+.............+.........+.............+.+.............................................................+................................+...+..+.............+.....++........++.+..+......................................+...........................+..+.......................+.............+........+.....+........+............+..+.............+.............+................................+...............+...+............+......................++.........+.......+.............................+........................+..........+...+....................+.........+.....+.+...............................................+.......+...........++......+.........+...................................................+.+...+.................+......................++......................................+..........+........+....................+.........+..................+..............................................+......+..............+.....+..+.........+..........+...++........+.....+..............+..................................+......................+......+......\n",
      "Loaded 22439 word vectors.\n",
      "\n",
      "Words not found 2058.\n",
      "words_singular 12804\n",
      "words_plural 4297\n",
      "2018.07.10 12:35\n",
      "Time to process: [104.01091170310974] seconds\n"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "\n",
    "start_time = get_time()\n",
    "\n",
    "# verify that mixed and lower case lists are the same length\n",
    "# print( \"¿\", len( tokens_unique ), \"==\", len( tokens_unique_lowercase ), \"?\" )\n",
    "\n",
    "# load the whole embedding into memory and iterate it, keeping only those word=embedding pairs present in tweets\n",
    "embeddings_index = dict()\n",
    "\n",
    "# moved up!\n",
    "#embeddings_dimension = 300 #must be 50, 100, 200, 300\n",
    "glove = open( \"../glove/glove.6B.\" + str( embeddings_dimension ) + \"d.txt\" )\n",
    "\n",
    "# for stats keeping\n",
    "words_plural = 0\n",
    "words_singular = 0\n",
    "\n",
    "# ASSUME: that 1st item in list is lowercase word, vectors are 2nd item\n",
    "# for line in tqdm.tqdm( lines, leave=False ):\n",
    "for line in glove:\n",
    "    \n",
    "    values = line.split()\n",
    "    # 1st string is word...\n",
    "    word = values[ 0 ]\n",
    "    \n",
    "    # we're now searching w/in lowercase version to allow both mixed and lower case words (\"WorD\" and \"word\") to \n",
    "    # inherit same vectors\n",
    "    if word in tokens_unique_lowercase:\n",
    "        \n",
    "        # ...the rest are coefficients\n",
    "        coefs = np.asarray( values[ 1: ], dtype='float32' )\n",
    "        \n",
    "        # get indices for all occurences in lower_case list\n",
    "        indices = [ i for i, word_lower in enumerate( tokens_unique_lowercase ) if word_lower == word ]\n",
    "        \n",
    "        if len( indices ) > 1:\n",
    "            \n",
    "            #print( \"[%d] entries found for [%s]\" % ( len( indices ), word ) )\n",
    "            print( \"+\", end=\"\" )\n",
    "            words_plural += 1\n",
    "            # iterate indices for this word\n",
    "            for i in indices:\n",
    "                word_temp = tokens_unique[ i ]\n",
    "                embeddings_index[ word_temp ] = coefs\n",
    "        else:\n",
    "            \n",
    "            print( \".\", end=\"\" )\n",
    "            words_singular += 1\n",
    "            embeddings_index[ word ] = coefs \n",
    "            \n",
    "    \n",
    "glove.close()\n",
    "print( '\\nLoaded %s word vectors.' % len( embeddings_index ) )\n",
    "print( '\\nWords not found %d.' % ( len( tokenizer.word_index ) - len( embeddings_index ) ) )\n",
    "print( \"words_singular\", words_singular )\n",
    "print( \"words_plural\", words_plural )\n",
    "\n",
    "print_time( start_time, get_time(), interval=\"minute\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "index [13234] word [PRESIDENT] frequencey [9]\n",
      "index [15233] word [president] frequencey [130]\n",
      "index [22296] word [President] frequencey [550]\n"
     ]
    }
   ],
   "source": [
    "indices = [ i for i, word_lower in enumerate( tokens_unique_lowercase ) if word_lower == \"president\" ]\n",
    "\n",
    "for i in indices:\n",
    "    \n",
    "    print( \"index [%d] word [%s] frequencey [%d]\" % ( i, tokens_unique[ i ], word_counts[ tokens_unique[ i ] ] ) )\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len( words_uppercase ) 2067\n"
     ]
    }
   ],
   "source": [
    "# what words appear in upper case?\n",
    "words_uppercase = []\n",
    "\n",
    "for word in tokens_unique:\n",
    "    \n",
    "    if word.isupper():\n",
    "        words_uppercase.append( word )\n",
    "        \n",
    "print( \"len( words_uppercase )\", len( words_uppercase ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "word [BLUE], count[4]\n",
      "word [FENCE], count[2]\n",
      "word [MATTER], count[2]\n",
      "word [MICROSOFT], count[2]\n",
      "word [LEAVE], count[1]\n",
      "word [ALABAMA], count[3]\n",
      "word [90M], count[1]\n",
      "word [SMARTER], count[1]\n",
      "word [GUILT], count[1]\n",
      "word [RENOVATION], count[1]\n",
      "word [JVF], count[1]\n",
      "word [COS], count[2]\n",
      "word [YEAR], count[7]\n",
      "word [FBI], count[100]\n",
      "word [PILOTS], count[1]\n",
      "word [INFLUENCE], count[1]\n",
      "word [NEGOTIATES], count[1]\n",
      "word [CNN], count[95]\n",
      "word [INNOVATION], count[2]\n",
      "word [MESS], count[4]\n",
      "word [MY], count[4]\n",
      "word [SOUTHERN], count[3]\n",
      "word [11PM], count[3]\n",
      "word [ONLY], count[6]\n",
      "word [BBC], count[2]\n",
      "word [IRS], count[12]\n",
      "word [WEST], count[2]\n",
      "word [REPORT], count[9]\n",
      "word [MERIT], count[2]\n",
      "word [MOVEMENT], count[39]\n",
      "word [EVENING], count[2]\n",
      "word [LATER], count[1]\n",
      "word [CHANGING], count[1]\n",
      "word [COACH], count[1]\n",
      "word [PLEASED], count[1]\n",
      "word [LIED], count[4]\n",
      "word [DEAL], count[10]\n",
      "word [TRUST], count[2]\n",
      "word [WTC], count[4]\n",
      "word [KY], count[4]\n",
      "word [V01], count[1]\n",
      "word [CA], count[10]\n",
      "word [THEIR], count[1]\n",
      "word [FILES], count[1]\n",
      "word [E], count[69]\n",
      "word [LEFT], count[2]\n",
      "word [NAFTA], count[27]\n",
      "word [SHOWS], count[2]\n",
      "word [WORSE], count[2]\n",
      "word [AYA], count[1]\n",
      "word [OREGON], count[1]\n",
      "word [SAVE], count[1]\n",
      "word [USED], count[3]\n",
      "word [ERRATIC], count[1]\n",
      "word [SPEECH], count[5]\n",
      "word [CONTINUES], count[1]\n",
      "word [TRUSTED], count[2]\n",
      "word [RICH], count[2]\n",
      "word [BORDERS], count[3]\n",
      "word [ND], count[1]\n",
      "word [FDNY], count[1]\n",
      "word [FDR], count[1]\n",
      "word [MPH], count[1]\n",
      "word [CAPS], count[1]\n",
      "word [V3], count[1]\n",
      "word [GANG], count[3]\n",
      "word [100G], count[2]\n",
      "word [Y241097], count[1]\n",
      "word [CURRENCY], count[1]\n",
      "word [BASED], count[2]\n",
      "word [CREATION], count[1]\n",
      "word [LIVES], count[4]\n",
      "word [ISIS], count[149]\n",
      "word [AGAIN🇺🇸], count[1]\n",
      "word [OBAMAS], count[3]\n",
      "word [SA], count[1]\n",
      "word [CBS], count[38]\n",
      "word [A], count[1092]\n",
      "word [10M], count[1]\n",
      "word [30PM], count[3]\n",
      "word [N24], count[1]\n",
      "word [DOONBEG], count[1]\n",
      "word [ATTN], count[1]\n",
      "word [TERRORISTS], count[4]\n",
      "word [BRAINWASHED], count[1]\n",
      "word [GUIDANCE], count[1]\n",
      "word [REPLACE], count[7]\n",
      "word [VIOLENT], count[1]\n",
      "word [QUICK], count[1]\n",
      "word [4TH], count[1]\n",
      "word [RULE], count[1]\n",
      "word [TJOHNSON], count[1]\n",
      "word [FRONTRUNNER], count[1]\n",
      "word [THIRTY], count[1]\n",
      "word [6M], count[4]\n",
      "word [PRICES], count[2]\n",
      "word [DEBATE], count[5]\n",
      "word [S], count[434]\n",
      "word [GLN], count[1]\n",
      "word [DURING], count[1]\n",
      "word [NBA], count[1]\n",
      "word [NASA], count[2]\n",
      "word [🚨BREAKING🚨], count[1]\n",
      "word [FLEW], count[1]\n",
      "word [EO], count[2]\n",
      "word [WORKED], count[1]\n",
      "word [500B], count[2]\n",
      "word [SEAL], count[6]\n",
      "word [UK], count[13]\n",
      "word [AGENDA], count[1]\n",
      "word [PA], count[13]\n",
      "word [BILLIONAIRE], count[1]\n",
      "word [PREMIUM], count[1]\n",
      "word [45AM], count[2]\n",
      "word [ENTERPRISE], count[1]\n",
      "word [CASH], count[3]\n",
      "word [NASDAQ], count[1]\n",
      "word [EXTRA], count[1]\n",
      "word [MOORE], count[2]\n",
      "word [DEALS], count[2]\n",
      "word [MATTHEWS], count[1]\n",
      "word [FULL], count[1]\n",
      "word [DOUBLE], count[2]\n",
      "word [CORRUPTION], count[2]\n",
      "word [GOLD], count[1]\n",
      "word [CRUSH], count[1]\n",
      "word [SOTU], count[11]\n",
      "word [COLD], count[1]\n",
      "word [OWN], count[3]\n",
      "word [20M], count[1]\n",
      "word [INSPIRE], count[1]\n",
      "word [TOO], count[2]\n",
      "word [RICK], count[1]\n",
      "word [CNBC], count[15]\n",
      "word [THE], count[125]\n",
      "word [ELEVENTH], count[1]\n",
      "word [EARLY], count[3]\n",
      "word [DHS], count[2]\n",
      "word [BLAME], count[1]\n",
      "word [VERIFY], count[1]\n",
      "word [£25M], count[1]\n",
      "word [DONATES], count[1]\n",
      "word [GENERAL], count[1]\n",
      "word [HEADS], count[1]\n",
      "word [CHILD], count[1]\n",
      "word [6B], count[1]\n",
      "word [BOOK], count[1]\n",
      "word [TED], count[1]\n",
      "word [CAREFUL], count[3]\n",
      "word [THANK], count[89]\n",
      "word [SHOCK], count[5]\n",
      "word [PROUDLY], count[1]\n",
      "word [STATEMENT], count[3]\n",
      "word [JUSTICES], count[1]\n",
      "word [UNREDACTED], count[1]\n",
      "word [BENGHAZI], count[2]\n",
      "word [PEARL], count[2]\n",
      "word [IQ], count[6]\n",
      "word [TEA], count[2]\n",
      "word [16T], count[7]\n",
      "word [450M], count[1]\n",
      "word [REPORTS], count[1]\n",
      "word [CELEBRATE], count[1]\n",
      "word [109B], count[1]\n",
      "word [MARIST], count[1]\n",
      "word [INTELLIGENT], count[1]\n",
      "word [BREAD], count[1]\n",
      "word [RATE], count[1]\n",
      "word [DECLARES], count[1]\n",
      "word [HR], count[1]\n",
      "word [BADLY], count[1]\n",
      "word [BAITING], count[1]\n",
      "word [ADL], count[1]\n",
      "word [PASSPORT], count[1]\n",
      "word [SOD], count[4]\n",
      "word [RAID], count[1]\n",
      "word [IMMUNITY], count[1]\n",
      "word [5K], count[1]\n",
      "word [VEGAS], count[2]\n",
      "word [6PM], count[3]\n",
      "word [UPDATE], count[1]\n",
      "word [BREXIT], count[7]\n",
      "word [DELEGATES], count[1]\n",
      "word [UTTER], count[1]\n",
      "word [100M], count[4]\n",
      "word [VISION], count[1]\n",
      "word [LEVIN], count[1]\n",
      "word [DOPE], count[1]\n",
      "word [TURN], count[1]\n",
      "word [KSM], count[1]\n",
      "word [RIGGED], count[5]\n",
      "word [GB], count[1]\n",
      "word [CHIEFS], count[2]\n",
      "word [CMA], count[1]\n",
      "word [BROKEN], count[1]\n",
      "word [ANTHEM], count[1]\n",
      "word [INTELLIGENCE], count[3]\n",
      "word [OBSTRUCTIONISTS], count[3]\n",
      "word [22B], count[1]\n",
      "word [INTEREST], count[2]\n",
      "word [HOPE], count[2]\n",
      "word [MK], count[1]\n",
      "word [PEO], count[1]\n",
      "word [G1959], count[2]\n",
      "word [RESPECTING], count[1]\n",
      "word [UNNECESSARY], count[1]\n",
      "word [LIE], count[3]\n",
      "word [COLDEST], count[2]\n",
      "word [AHEAD], count[1]\n",
      "word [VULNERABLE], count[1]\n",
      "word [PIVOT❌RUSSIAN], count[2]\n",
      "word [FINALLY], count[2]\n",
      "word [ROY], count[2]\n",
      "word [JOIN], count[5]\n",
      "word [OVERSEAS], count[1]\n",
      "word [INNOVATORS], count[1]\n",
      "word [HI], count[5]\n",
      "word [TOM], count[1]\n",
      "word [DOLLARS], count[6]\n",
      "word [AIR], count[1]\n",
      "word [27M], count[2]\n",
      "word [169B], count[1]\n",
      "word [KEEPS], count[1]\n",
      "word [WONDERFUL], count[1]\n",
      "word [LEADS], count[1]\n",
      "word [THAN], count[3]\n",
      "word [LEAKER], count[1]\n",
      "word [FAMILY], count[2]\n",
      "word [ABC], count[39]\n",
      "word [WISCONSIN], count[2]\n",
      "word [TV], count[60]\n",
      "word [TN], count[4]\n",
      "word [GOVT], count[1]\n",
      "word [YOURE], count[5]\n",
      "word [EASY], count[1]\n",
      "word [SKYROCKET], count[1]\n",
      "word [TERMINATE], count[1]\n",
      "word [ANNOUNCEMENT], count[1]\n",
      "word [SB], count[1]\n",
      "word [SECOND], count[2]\n",
      "word [00AM], count[3]\n",
      "word [OBAMACARE], count[6]\n",
      "word [PEOPLE], count[14]\n",
      "word [AMERICAN], count[23]\n",
      "word [FLOYD], count[1]\n",
      "word [VI], count[1]\n",
      "word [OR], count[5]\n",
      "word [ABCCBS], count[1]\n",
      "word [FRACK], count[4]\n",
      "word [OSCARS], count[1]\n",
      "word [LOSE], count[2]\n",
      "word [TOURÉ], count[1]\n",
      "word [20T], count[3]\n",
      "word [MSNBCS], count[1]\n",
      "word [UPCOMING], count[1]\n",
      "word [60M], count[3]\n",
      "word [STRAW], count[2]\n",
      "word [EASTER], count[4]\n",
      "word [INCOME], count[1]\n",
      "word [2B], count[3]\n",
      "word [APPROVE], count[1]\n",
      "word [TRUTH], count[2]\n",
      "word [VOTE], count[58]\n",
      "word [WMUR], count[1]\n",
      "word [BUY], count[2]\n",
      "word [SUPPORTERS], count[2]\n",
      "word [BANGERS], count[1]\n",
      "word [LEVEL], count[1]\n",
      "word [FORGOT], count[2]\n",
      "word [DTS], count[1]\n",
      "word [MI], count[7]\n",
      "word [PRESIDENTS], count[1]\n",
      "word [SPY], count[4]\n",
      "word [JW], count[1]\n",
      "word [MAG], count[1]\n",
      "word [DEFENSE], count[1]\n",
      "word [FASTER], count[1]\n",
      "word [MAN’], count[1]\n",
      "word [FEAR], count[1]\n",
      "word [HOAX], count[2]\n",
      "word [KOREA], count[1]\n",
      "word [COMMUNITIES], count[2]\n",
      "word [QB], count[2]\n",
      "word [WHY], count[6]\n",
      "word [ENERGIZED], count[1]\n",
      "word [ISNT], count[1]\n",
      "word [WWI], count[2]\n",
      "word [SOLD], count[1]\n",
      "word [EDITION], count[5]\n",
      "word [WWE], count[11]\n",
      "word [OVERWHELMING], count[1]\n",
      "word [CIA], count[28]\n",
      "word [12M], count[2]\n",
      "word [DEATH], count[5]\n",
      "word [BOMBSHELLS], count[1]\n",
      "word [LAND], count[1]\n",
      "word [CASE], count[1]\n",
      "word [ANY], count[2]\n",
      "word [NSC], count[1]\n",
      "word [STRONGER], count[1]\n",
      "word [GRASSROOTS], count[2]\n",
      "word [HANDED], count[1]\n",
      "word [ESTABLISH], count[1]\n",
      "word [TRUE], count[3]\n",
      "word [SYRIA], count[3]\n",
      "word [FIELDS], count[1]\n",
      "word [OSCAR], count[1]\n",
      "word [BIRTHER], count[1]\n",
      "word [POSITION], count[2]\n",
      "word [TT], count[1]\n",
      "word [MILLION], count[9]\n",
      "word [LIES], count[4]\n",
      "word [JFK], count[4]\n",
      "word [SAD], count[18]\n",
      "word [TAKEN], count[1]\n",
      "word [ICE], count[9]\n",
      "word [TROPHY], count[3]\n",
      "word [CLIMATE], count[8]\n",
      "word [HEARS], count[1]\n",
      "word [BELIEVE], count[1]\n",
      "word [THIRD], count[2]\n",
      "word [AMERICANS], count[4]\n",
      "word [PLAN], count[3]\n",
      "word [PMO], count[2]\n",
      "word [16M], count[1]\n",
      "word [FL], count[21]\n",
      "word [118M], count[1]\n",
      "word [LOVED], count[1]\n",
      "word [FAR], count[5]\n",
      "word [CANNOT], count[3]\n",
      "word [HITS], count[2]\n",
      "word [PASTOR], count[1]\n",
      "word [ROBOCALL], count[1]\n",
      "word [QE3], count[6]\n",
      "word [SOMALIA], count[1]\n",
      "word [SYSTEM], count[1]\n",
      "word [BACKLASH], count[1]\n",
      "word [PT], count[1]\n",
      "word [WARMING], count[22]\n",
      "word [INS], count[1]\n",
      "word [RESULTS], count[1]\n",
      "word [SATURDAY], count[1]\n",
      "word [800B], count[1]\n",
      "word [INTO], count[1]\n",
      "word [DANGEROUS], count[3]\n",
      "word [3M], count[5]\n",
      "word [EVERYBODY], count[1]\n",
      "word [F], count[75]\n",
      "word [100B], count[2]\n",
      "word [NBPC], count[1]\n",
      "word [EXCUSE], count[1]\n",
      "word [HARDER], count[3]\n",
      "word [SCALE], count[1]\n",
      "word [G7], count[7]\n",
      "word [WIFE], count[1]\n",
      "word [40M], count[1]\n",
      "word [RAISES], count[1]\n",
      "word [SOMETHING], count[1]\n",
      "word [CNET], count[1]\n",
      "word [NO], count[57]\n",
      "word [NATO], count[13]\n",
      "word [INSULTING], count[1]\n",
      "word [CEDER], count[1]\n",
      "word [700M], count[1]\n",
      "word [POWER], count[1]\n",
      "word [HAS], count[8]\n",
      "word [NUMBER], count[1]\n",
      "word [487K], count[1]\n",
      "word [RECKLESS], count[1]\n",
      "word [INDEPENDENT], count[1]\n",
      "word [SACCONE], count[1]\n",
      "word [REMEMBER], count[4]\n",
      "word [BILLIONS], count[2]\n",
      "word [ACTION], count[3]\n",
      "word [LV], count[1]\n",
      "word [VOTED], count[1]\n",
      "word [HP], count[1]\n",
      "word [OIR], count[1]\n",
      "word [SECURE], count[3]\n",
      "word [STRATEGY], count[1]\n",
      "word [THANKSGIVING], count[4]\n",
      "word [MASTERS], count[2]\n",
      "word [HUD], count[2]\n",
      "word [BIGGEST], count[1]\n",
      "word [IOWA], count[7]\n",
      "word [NSA], count[9]\n",
      "word [ATTORNEY], count[1]\n",
      "word [CONFIRM], count[1]\n",
      "word [ADDRESS], count[1]\n",
      "word [GRAHAM], count[1]\n",
      "word [TICKETS], count[1]\n",
      "word [LARGE], count[1]\n",
      "word [REGS], count[1]\n",
      "word [BEGGING], count[1]\n",
      "word [TRYING], count[1]\n",
      "word [300K], count[2]\n",
      "word [SUPER], count[1]\n",
      "word [JINPING], count[3]\n",
      "word [REWARDED], count[1]\n",
      "word [LEADERS], count[2]\n",
      "word [W], count[53]\n",
      "word [PREACHES], count[1]\n",
      "word [LEARN], count[1]\n",
      "word [Q1], count[3]\n",
      "word [CUSTOMERS], count[1]\n",
      "word [VERSUS], count[1]\n",
      "word [FB], count[1]\n",
      "word [FREEBIES], count[1]\n",
      "word [MANIPULATION], count[1]\n",
      "word [MONTH], count[1]\n",
      "word [HRC], count[9]\n",
      "word [15T], count[1]\n",
      "word [DOJ], count[8]\n",
      "word [WORLD], count[11]\n",
      "word [EARTH], count[1]\n",
      "word [PGA], count[9]\n",
      "word [N555], count[1]\n",
      "word [TAKE], count[4]\n",
      "word [SHOWED], count[1]\n",
      "word [HQ], count[8]\n",
      "word [FILTHY], count[1]\n",
      "word [SANCTUARY], count[1]\n",
      "word [COMMANDER], count[2]\n",
      "word [SABOTAGE], count[1]\n",
      "word [PEACE], count[6]\n",
      "word [THEMSELVES], count[1]\n",
      "word [ELECTIONS], count[1]\n",
      "word [Q2], count[7]\n",
      "word [COULDNT], count[1]\n",
      "word [WP], count[1]\n",
      "word [SOUTH], count[3]\n",
      "word [EMIN], count[1]\n",
      "word [PLEDGE], count[1]\n",
      "word [AMAZING], count[13]\n",
      "word [FINDINGS], count[1]\n",
      "word [CLAIMS], count[1]\n",
      "word [REBELS], count[1]\n",
      "word [CAROLINA], count[4]\n",
      "word [DOE], count[2]\n",
      "word [ASIANS], count[1]\n",
      "word [RESET❌BENGHAZI], count[2]\n",
      "word [WATERLOO], count[1]\n",
      "word [BTW], count[2]\n",
      "word [EXPECT], count[1]\n",
      "word [EPIC], count[1]\n",
      "word [VOTING], count[3]\n",
      "word [BEGINS], count[1]\n",
      "word [UNITY], count[2]\n",
      "word [BY], count[7]\n",
      "word [FAILURE], count[2]\n",
      "word [OPPORTUNITY], count[1]\n",
      "word [ANYTHING], count[1]\n",
      "word [PRETEND], count[1]\n",
      "word [COLLUSION], count[10]\n",
      "word [NOTICE], count[2]\n",
      "word [TIMES], count[1]\n",
      "word [OBAMA], count[6]\n",
      "word [DEMOCRATS], count[1]\n",
      "word [LYIN], count[2]\n",
      "word [STOP], count[12]\n",
      "word [COMMANDING], count[1]\n",
      "word [RETHINK], count[1]\n",
      "word [GIVE], count[4]\n",
      "word [AOL], count[9]\n",
      "word [LIFE], count[4]\n",
      "word [BE], count[26]\n",
      "word [EMERGES], count[1]\n",
      "word [EXTREME], count[2]\n",
      "word [COUNSEL], count[1]\n",
      "word [DONALD], count[26]\n",
      "word [LGA], count[1]\n",
      "word [GMA], count[2]\n",
      "word [G], count[90]\n",
      "word [DOMINANCE], count[1]\n",
      "word [FX], count[1]\n",
      "word [DL], count[1]\n",
      "word [LED], count[1]\n",
      "word [UNBELIEVABLE], count[1]\n",
      "word [WILD], count[1]\n",
      "word [JANUARY], count[1]\n",
      "word [GREATNESS], count[3]\n",
      "word [4K], count[1]\n",
      "word [ONCE], count[2]\n",
      "word [ROVE], count[3]\n",
      "word [READY], count[2]\n",
      "word [MEETING], count[1]\n",
      "word [NOW], count[44]\n",
      "word [NOMINATION], count[1]\n",
      "word [AT], count[14]\n",
      "word [SKY], count[1]\n",
      "word [REALITY], count[1]\n",
      "word [AMENDMENT], count[1]\n",
      "word [SNL], count[4]\n",
      "word [01AM], count[1]\n",
      "word [IDEA], count[1]\n",
      "word [ENDING], count[1]\n",
      "word [LOOPHOLES], count[1]\n",
      "word [1AM], count[2]\n",
      "word [IMO], count[1]\n",
      "word [ENOUGH], count[5]\n",
      "word [YR], count[1]\n",
      "word [DELIVER], count[1]\n",
      "word [CROOK], count[1]\n",
      "word [BEST], count[15]\n",
      "word [R69], count[1]\n",
      "word [BRAVE], count[3]\n",
      "word [XXIX], count[2]\n",
      "word [IRA], count[1]\n",
      "word [POLICY], count[3]\n",
      "word [5B], count[9]\n",
      "word [FAIL], count[1]\n",
      "word [ICYMI], count[16]\n",
      "word [SUSPECT], count[1]\n",
      "word [BEHIND], count[1]\n",
      "word [CLEAR], count[1]\n",
      "word [ACN], count[1]\n",
      "word [WI], count[5]\n",
      "word [I1], count[1]\n",
      "word [INVESTMENTS], count[2]\n",
      "word [BIGOTRY], count[1]\n",
      "word [RESPONDS], count[1]\n",
      "word [PRESIDENCY], count[1]\n",
      "word [MIDDLE], count[1]\n",
      "word [GETS], count[1]\n",
      "word [NEW], count[24]\n",
      "word [WELL], count[1]\n",
      "word [TAPE], count[2]\n",
      "word [QUOTE], count[1]\n",
      "word [CHAOS], count[1]\n",
      "word [LUCKIER], count[3]\n",
      "word [NE], count[1]\n",
      "word [SWAT], count[1]\n",
      "word [BENCH], count[1]\n",
      "word [PRESIDENTIAL], count[4]\n",
      "word [SUPERPACS], count[1]\n",
      "word [UNIVERSE®], count[1]\n",
      "word [RECORDED], count[1]\n",
      "word [HEAD], count[1]\n",
      "word [OO], count[1]\n",
      "word [VISA], count[1]\n",
      "word [THINK], count[3]\n",
      "word [TOTALLY], count[4]\n",
      "word [W33157775], count[1]\n",
      "word [FAKE], count[53]\n",
      "word [GLOBAL], count[22]\n",
      "word [USC], count[10]\n",
      "word [HSI], count[1]\n",
      "word [V], count[66]\n",
      "word [OPEN], count[4]\n",
      "word [CY], count[1]\n",
      "word [HEALTH], count[2]\n",
      "word [PENALTY], count[3]\n",
      "word [SAYING], count[2]\n",
      "word [DOUBLED], count[1]\n",
      "word [NEGOTIATION], count[1]\n",
      "word [THX], count[1]\n",
      "word [REPUBLICANS], count[1]\n",
      "word [COOL], count[1]\n",
      "word [USMC], count[14]\n",
      "word [CSPAN], count[1]\n",
      "word [VALUES], count[1]\n",
      "word [DECEIT], count[1]\n",
      "word [ELECTION], count[2]\n",
      "word [TOTALITARIANS], count[1]\n",
      "word [ALS], count[2]\n",
      "word [23M], count[1]\n",
      "word [AIRED], count[1]\n",
      "word [KERRY], count[1]\n",
      "word [MISS], count[1]\n",
      "word [ANGER], count[1]\n",
      "word [NAME], count[2]\n",
      "word [LAUGHABLE], count[1]\n",
      "word [SLAMS], count[1]\n",
      "word [LA], count[15]\n",
      "word [SCOTUS], count[2]\n",
      "word [GRATEFUL], count[1]\n",
      "word [RATINGS], count[1]\n",
      "word [JAILED], count[1]\n",
      "word [197M], count[1]\n",
      "word [MATCH], count[1]\n",
      "word [SIR], count[1]\n",
      "word [NV], count[4]\n",
      "word [CHERISH], count[1]\n",
      "word [WWII], count[7]\n",
      "word [RUSSIA], count[7]\n",
      "word [HT], count[1]\n",
      "word [DISTORTING], count[1]\n",
      "word [GMP], count[1]\n",
      "word [AKOYA], count[1]\n",
      "word [DAMAC], count[4]\n",
      "word [KASH], count[1]\n",
      "word [U], count[54]\n",
      "word [WEAKNESS], count[1]\n",
      "word [SITES], count[1]\n",
      "word [GPS], count[2]\n",
      "word [STEEL], count[1]\n",
      "word [BUYING], count[1]\n",
      "word [CHARITY], count[2]\n",
      "word [HABIT], count[1]\n",
      "word [RESPECT], count[4]\n",
      "word [DROPPED], count[1]\n",
      "word [85B], count[1]\n",
      "word [NRA], count[6]\n",
      "word [VP], count[18]\n",
      "word [WITCH], count[7]\n",
      "word [EPISODES], count[1]\n",
      "word [RSPB], count[1]\n",
      "word [AN], count[5]\n",
      "word [B205], count[1]\n",
      "word [WHO], count[2]\n",
      "word [10AM], count[3]\n",
      "word [MADE], count[2]\n",
      "word [ACT], count[7]\n",
      "word [BIGGER], count[1]\n",
      "word [PERSONALLY], count[1]\n",
      "word [STATE], count[11]\n",
      "word [FM], count[3]\n",
      "word [FUTURE], count[3]\n",
      "word [WISH], count[1]\n",
      "word [HOAXSTERS], count[2]\n",
      "word [FAIR], count[7]\n",
      "word [79M], count[1]\n",
      "word [SAME], count[3]\n",
      "word [BECAUSE], count[1]\n",
      "word [AJ], count[2]\n",
      "word [BLOCKBUSTER], count[1]\n",
      "word [B], count[64]\n",
      "word [STANDARD’], count[1]\n",
      "word [CENTURY], count[1]\n",
      "word [4M], count[3]\n",
      "word [SOVEREIGN], count[1]\n",
      "word [POTUS], count[30]\n",
      "word [RECORD], count[12]\n",
      "word [242K], count[1]\n",
      "word [UNDERESTIMATE], count[1]\n",
      "word [PLENTY], count[1]\n",
      "word [NONSENSE], count[1]\n",
      "word [BOSS], count[1]\n",
      "word [MAJORITY], count[3]\n",
      "word [DEMOCRAT], count[1]\n",
      "word [COMING], count[1]\n",
      "word [NAACP], count[2]\n",
      "word [WASTE], count[1]\n",
      "word [OWS], count[6]\n",
      "word [7K], count[1]\n",
      "word [EU], count[4]\n",
      "word [HONOR], count[2]\n",
      "word [WOUNDED], count[1]\n",
      "word [TEAM], count[1]\n",
      "word [MSNBC], count[11]\n",
      "word [SENATOR], count[2]\n",
      "word [CANADIAN], count[1]\n",
      "word [POLICIES], count[2]\n",
      "word [HOW], count[1]\n",
      "word [CO], count[5]\n",
      "word [AUSTIN], count[1]\n",
      "word [GW], count[2]\n",
      "word [LIAR], count[3]\n",
      "word [TPM], count[1]\n",
      "word [GA], count[1]\n",
      "word [POLS], count[2]\n",
      "word [DOING], count[1]\n",
      "word [MIKE], count[1]\n",
      "word [REVIEWS], count[1]\n",
      "word [FAILING], count[1]\n",
      "word [REPEALED], count[1]\n",
      "word [LLP], count[1]\n",
      "word [E-A], count[4]\n",
      "word [SUN], count[1]\n",
      "word [BCS], count[1]\n",
      "word [ADVANCED], count[1]\n",
      "word [PORTLAND], count[1]\n",
      "word [STRENGTH], count[4]\n",
      "word [THING], count[1]\n",
      "word [XXIII], count[1]\n",
      "word [CONTROLLED], count[1]\n",
      "word [ORDER], count[3]\n",
      "word [XINPING], count[1]\n",
      "word [11TH], count[1]\n",
      "word [GROUPS], count[1]\n",
      "word [II], count[5]\n",
      "word [TIME], count[20]\n",
      "word [CALLED], count[2]\n",
      "word [ALL], count[41]\n",
      "word [13M], count[1]\n",
      "word [91RTG], count[1]\n",
      "word [LONGEST], count[1]\n",
      "word [NOBEL], count[1]\n",
      "word [ID], count[2]\n",
      "word [LT], count[1]\n",
      "word [FIU], count[1]\n",
      "word [REDUCTION], count[1]\n",
      "word [THIS], count[5]\n",
      "word [ILLINOIS], count[1]\n",
      "word [CORE], count[1]\n",
      "word [STOCKS], count[1]\n",
      "word [SO], count[15]\n",
      "word [RACISM], count[1]\n",
      "word [SRO], count[1]\n",
      "word [BS], count[2]\n",
      "word [2PM], count[2]\n",
      "word [COMMON], count[2]\n",
      "word [CONGRATULATIONS], count[4]\n",
      "word [TAINTED], count[1]\n",
      "word [RAPIDS], count[1]\n",
      "word [LAS], count[2]\n",
      "word [SLEAZE], count[1]\n",
      "word [THRIVE], count[1]\n",
      "word [PUPPET], count[1]\n",
      "word [60K], count[1]\n",
      "word [EXPLORATORY], count[1]\n",
      "word [FOP], count[1]\n",
      "word [IBM], count[1]\n",
      "word [PENCHANT], count[1]\n",
      "word [13K], count[1]\n",
      "word [DT], count[1]\n",
      "word [OFFICERS], count[1]\n",
      "word [UN], count[23]\n",
      "word [TALK], count[6]\n",
      "word [MR], count[1]\n",
      "word [4B], count[4]\n",
      "word [12PM], count[3]\n",
      "word [DEFENDS], count[1]\n",
      "word [FEW], count[1]\n",
      "word [WITHOUT], count[1]\n",
      "word [PLAYER], count[1]\n",
      "word [ITSELF], count[1]\n",
      "word [GOAL], count[1]\n",
      "word [HARBOR], count[2]\n",
      "word [UPFRONTS], count[1]\n",
      "word [CSI], count[1]\n",
      "word [150M], count[1]\n",
      "word [TERRIBLE], count[1]\n",
      "word [DENY], count[1]\n",
      "word [NORTHCOM], count[1]\n",
      "word [PHOTOS], count[1]\n",
      "word [TUMOR], count[1]\n",
      "word [MANY], count[6]\n",
      "word [MAGA], count[3]\n",
      "word [RACE], count[2]\n",
      "word [RIGHT], count[3]\n",
      "word [END], count[10]\n",
      "word [WORKING], count[1]\n",
      "word [UNACCEPTABLE], count[2]\n",
      "word [NATION], count[9]\n",
      "word [BULL], count[1]\n",
      "word [FRISK], count[1]\n",
      "word [BHO], count[1]\n",
      "word [OPERATIVE], count[1]\n",
      "word [CAUSE], count[1]\n",
      "word [DETERMINED], count[1]\n",
      "word [UFC], count[3]\n",
      "word [AMVETS], count[1]\n",
      "word [LYING], count[2]\n",
      "word [ERA], count[2]\n",
      "word [WARN], count[1]\n",
      "word [MORE], count[10]\n",
      "word [COSTS], count[1]\n",
      "word [HILLS], count[1]\n",
      "word [THERE], count[2]\n",
      "word [HIGHS], count[1]\n",
      "word [TALKING], count[1]\n",
      "word [RENEWED], count[1]\n",
      "word [TEXAS], count[3]\n",
      "word [MOMENTS], count[1]\n",
      "word [COORDINATION], count[1]\n",
      "word [35M], count[1]\n",
      "word [295M], count[1]\n",
      "word [NM], count[2]\n",
      "word [PRIVATE], count[1]\n",
      "word [ONE], count[4]\n",
      "word [CLAIM], count[1]\n",
      "word [7PM], count[6]\n",
      "word [WCNC], count[1]\n",
      "word [CDC], count[6]\n",
      "word [ENERGY], count[2]\n",
      "word [SCARY], count[2]\n",
      "word [SOMETIMES], count[2]\n",
      "word [DWS], count[1]\n",
      "word [ADDRESS🇺🇸], count[3]\n",
      "word [IMPROVE], count[1]\n",
      "word [LEADERSHIP], count[5]\n",
      "word [SURVEILLANCE], count[2]\n",
      "word [FIFTH], count[1]\n",
      "word [FORCE], count[1]\n",
      "word [WOULD], count[5]\n",
      "word [NYU], count[1]\n",
      "word [CAREFULLY], count[1]\n",
      "word [HOF], count[1]\n",
      "word [INCOMPETENT], count[1]\n",
      "word [BUMP], count[1]\n",
      "word [FREEDOM], count[5]\n",
      "word [RASMUSSEN], count[1]\n",
      "word [RECLAIM], count[1]\n",
      "word [7M], count[1]\n",
      "word [WORKERS], count[5]\n",
      "word [JUST], count[6]\n",
      "word [DEPLORABLES], count[2]\n",
      "word [BUSINESS], count[3]\n",
      "word [JEB], count[6]\n",
      "word [AZ], count[5]\n",
      "word [B550], count[1]\n",
      "word [ANOTHER], count[2]\n",
      "word [WATCHING], count[1]\n",
      "word [PATHOLOGICAL], count[1]\n",
      "word [C45], count[1]\n",
      "word [N], count[106]\n",
      "word [STORIES], count[1]\n",
      "word [JOHN], count[1]\n",
      "word [5M], count[42]\n",
      "word [SECRETLY], count[1]\n",
      "word [CONGRATS], count[2]\n",
      "word [VORTEX], count[1]\n",
      "word [ALREADY], count[2]\n",
      "word [PST], count[1]\n",
      "word [HIGHEST], count[1]\n",
      "word [JOBS], count[77]\n",
      "word [M25221024], count[1]\n",
      "word [PREDATORS], count[1]\n",
      "word [HES], count[1]\n",
      "word [INCLUDE], count[1]\n",
      "word [SHE], count[1]\n",
      "word [TEARS], count[1]\n",
      "word [VETO], count[1]\n",
      "word [PESSIMIST], count[1]\n",
      "word [REGISTER], count[2]\n",
      "word [CARSON], count[8]\n",
      "word [TWEETING], count[1]\n",
      "word [MINUTES], count[2]\n",
      "word [AGREE], count[1]\n",
      "word [WAR], count[4]\n",
      "word [USSS], count[3]\n",
      "word [YEMEN], count[1]\n",
      "word [INDV], count[1]\n",
      "word [LEAD], count[2]\n",
      "word [FCC], count[1]\n",
      "word [LOWEST], count[1]\n",
      "word [BAD], count[24]\n",
      "word [VILLA], count[1]\n",
      "word [UCLA], count[2]\n",
      "word [LOSING], count[1]\n",
      "word [VICTORY], count[5]\n",
      "word [RELATED], count[2]\n",
      "word [STUPIDITY], count[1]\n",
      "word [A-X], count[1]\n",
      "word [STRONG], count[9]\n",
      "word [HIRE], count[4]\n",
      "word [MISSION], count[2]\n",
      "word [NBC], count[155]\n",
      "word [TRIBUTE], count[1]\n",
      "word [CROOKED], count[2]\n",
      "word [ECONOMY], count[2]\n",
      "word [TURMOIL], count[1]\n",
      "word [CHANGE], count[19]\n",
      "word [TRIPLED], count[1]\n",
      "word [TAKES], count[1]\n",
      "word [VETS], count[11]\n",
      "word [IN], count[51]\n",
      "word [SUDAN], count[1]\n",
      "word [VETERANS], count[14]\n",
      "word [KAREN], count[1]\n",
      "word [STAR], count[3]\n",
      "word [RUBIOS], count[1]\n",
      "word [8T], count[2]\n",
      "word [SICK], count[2]\n",
      "word [KORUS], count[1]\n",
      "word [DOESNT], count[1]\n",
      "word [STEVE], count[1]\n",
      "word [STAY], count[3]\n",
      "word [MISSILES], count[1]\n",
      "word [SYRACUSE], count[2]\n",
      "word [GUY], count[1]\n",
      "word [RT], count[43]\n",
      "word [IRAN], count[3]\n",
      "word [SS], count[1]\n",
      "word [6T], count[13]\n",
      "word [TONIGHT], count[2]\n",
      "word [BEFORE], count[3]\n",
      "word [TRENDING], count[1]\n",
      "word [HISTORIC], count[4]\n",
      "word [EVERY], count[5]\n",
      "word [NC], count[11]\n",
      "word [PLACE], count[2]\n",
      "word [PARODY], count[1]\n",
      "word [63RD], count[1]\n",
      "word [ALIS], count[1]\n",
      "word [CUT], count[17]\n",
      "word [MODERN], count[1]\n",
      "word [COO], count[1]\n",
      "word [COMMITTEE], count[4]\n",
      "word [DETERRENT], count[1]\n",
      "word [PRAISES], count[1]\n",
      "word [HAVE], count[13]\n",
      "word [DNC], count[43]\n",
      "word [GENERALS], count[2]\n",
      "word [UT], count[1]\n",
      "word [NYT], count[1]\n",
      "word [NEVADA], count[6]\n",
      "word [GO], count[9]\n",
      "word [HIGH], count[3]\n",
      "word [WRONG], count[8]\n",
      "word [RALLIES], count[2]\n",
      "word [RELIEF], count[1]\n",
      "word [ADMINISTRATION], count[1]\n",
      "word [LIBYA], count[1]\n",
      "word [KLZA], count[1]\n",
      "word [21T], count[1]\n",
      "word [W20], count[1]\n",
      "word [CROWNED], count[1]\n",
      "word [PRIMARY], count[2]\n",
      "word [CONTROL], count[1]\n",
      "word [IMBEDDED], count[1]\n",
      "word [PROTECT], count[7]\n",
      "word [LIARS], count[1]\n",
      "word [PACK], count[1]\n",
      "word [THROUGH], count[2]\n",
      "word [1K], count[1]\n",
      "word [KKK], count[1]\n",
      "word [WEAK], count[8]\n",
      "word [505K], count[1]\n",
      "word [TWO], count[5]\n",
      "word [GET], count[14]\n",
      "word [NY], count[77]\n",
      "word [MOVING], count[1]\n",
      "word [ILLEGAL], count[4]\n",
      "word [300B], count[1]\n",
      "word [RJC], count[1]\n",
      "word [OHIO], count[6]\n",
      "word [DEFEAT], count[2]\n",
      "word [FLAG], count[3]\n",
      "word [HILLARY], count[6]\n",
      "word [T], count[78]\n",
      "word [PTS], count[1]\n",
      "word [PROBE], count[1]\n",
      "word [BUST], count[1]\n",
      "word [WORLDS], count[4]\n",
      "word [C21], count[1]\n",
      "word [ACCEPTANCE], count[2]\n",
      "word [SINGLE], count[5]\n",
      "word [NICE], count[3]\n",
      "word [D], count[147]\n",
      "word [IRAQS], count[1]\n",
      "word [VET], count[1]\n",
      "word [CLASSIFIED], count[4]\n",
      "word [BUT], count[3]\n",
      "word [SAF], count[1]\n",
      "word [IS], count[33]\n",
      "word [WGC], count[16]\n",
      "word [5T], count[4]\n",
      "word [YOUVE], count[1]\n",
      "word [AGO], count[1]\n",
      "word [X], count[13]\n",
      "word [LAW], count[5]\n",
      "word [HYPOCRITE], count[1]\n",
      "word [GRANITE], count[1]\n",
      "word [SD], count[1]\n",
      "word [PERIOD], count[1]\n",
      "word [QUICKLY], count[1]\n",
      "word [RAPIDLY], count[1]\n",
      "word [REST], count[1]\n",
      "word [OK], count[9]\n",
      "word [REFORM], count[5]\n",
      "word [HEART], count[1]\n",
      "word [M], count[306]\n",
      "word [CHIP], count[1]\n",
      "word [APPLICATIONS], count[1]\n",
      "word [OPENLY], count[1]\n",
      "word [USA🇺🇸], count[2]\n",
      "word [REASONS], count[2]\n",
      "word [HAPPY], count[22]\n",
      "word [SPIRIT], count[5]\n",
      "word [K], count[48]\n",
      "word [12K], count[1]\n",
      "word [ILL], count[1]\n",
      "word [PLA], count[1]\n",
      "word [AAA], count[11]\n",
      "word [CONFIRMED], count[2]\n",
      "word [DIDNT], count[1]\n",
      "word [BIG], count[40]\n",
      "word [WIN], count[41]\n",
      "word [SHUTDOWN], count[2]\n",
      "word [DISRESPECT], count[1]\n",
      "word [REFUGEE], count[1]\n",
      "word [VPE], count[1]\n",
      "word [CHEAPER], count[1]\n",
      "word [SAILOR], count[1]\n",
      "word [MSM], count[9]\n",
      "word [TOLD], count[6]\n",
      "word [NATIONWIDE], count[1]\n",
      "word [NIELSEN], count[1]\n",
      "word [OFFICIAL], count[3]\n",
      "word [BROKE], count[1]\n",
      "word [G500], count[1]\n",
      "word [SEE], count[5]\n",
      "word [OVER], count[4]\n",
      "word [ROMNEY], count[1]\n",
      "word [WJIM], count[1]\n",
      "word [GARY], count[2]\n",
      "word [WV], count[2]\n",
      "word [ABOUT], count[4]\n",
      "word [INFORMANT], count[2]\n",
      "word [WHOA], count[1]\n",
      "word [YOURS], count[5]\n",
      "word [SELFISH], count[1]\n",
      "word [GIRLFRIEND], count[1]\n",
      "word [NOSE], count[1]\n",
      "word [CPAC2013], count[1]\n",
      "word [ANTI], count[4]\n",
      "word [JUDICIARY], count[1]\n",
      "word [PENCE], count[2]\n",
      "word [BIRTHDAY], count[10]\n",
      "word [25M], count[1]\n",
      "word [SUCCESS], count[2]\n",
      "word [CPAC], count[41]\n",
      "word [REBUKES], count[1]\n",
      "word [R], count[62]\n",
      "word [LEAGUE], count[4]\n",
      "word [AUTISM], count[2]\n",
      "word [RECOVER], count[1]\n",
      "word [RALLY], count[3]\n",
      "word [DREAMERS], count[1]\n",
      "word [OAN], count[2]\n",
      "word [INSIDERS], count[1]\n",
      "word [DELIVERING], count[1]\n",
      "word [ITS], count[7]\n",
      "word [WPOST], count[1]\n",
      "word [SIU], count[1]\n",
      "word [PAGE], count[2]\n",
      "word [TPP], count[23]\n",
      "word [LATE], count[1]\n",
      "word [TMZ], count[1]\n",
      "word [SENATE], count[1]\n",
      "word [VOTERS], count[1]\n",
      "word [LETS], count[5]\n",
      "word [HEREOS], count[1]\n",
      "word [HILLARYS], count[3]\n",
      "word [WHAT], count[7]\n",
      "word [FRIENDS], count[1]\n",
      "word [BLANK], count[1]\n",
      "word [KS], count[1]\n",
      "word [831B], count[1]\n",
      "word [375B], count[1]\n",
      "word [HAPPEN], count[2]\n",
      "word [IMPORTANT], count[3]\n",
      "word [DEBT], count[3]\n",
      "word [HERE], count[2]\n",
      "word [HOTEL], count[2]\n",
      "word [R52], count[2]\n",
      "word [CARTELS], count[1]\n",
      "word [BUILD], count[7]\n",
      "word [JCOPE], count[4]\n",
      "word [WARM], count[1]\n",
      "word [GENIUS], count[1]\n",
      "word [WEAPONS], count[1]\n",
      "word [LAUNCH], count[1]\n",
      "word [PROBLEMS], count[1]\n",
      "word [GANGS], count[1]\n",
      "word [WM23], count[1]\n",
      "word [MCCAIN], count[1]\n",
      "word [MIRED], count[1]\n",
      "word [NASHVILLE], count[1]\n",
      "word [XI], count[6]\n",
      "word [MVP], count[5]\n",
      "word [OPEC], count[53]\n",
      "word [DOMESTIC], count[1]\n",
      "word [GEORGIA], count[1]\n",
      "word [MAYWEATHER], count[1]\n",
      "word [TRIUMPH], count[1]\n",
      "word [BALANCE], count[6]\n",
      "word [BORING], count[2]\n",
      "word [ADMITS], count[1]\n",
      "word [FOX], count[15]\n",
      "word [GCC], count[1]\n",
      "word [GROVELING], count[1]\n",
      "word [EMBEDDED], count[1]\n",
      "word [STOLEN], count[1]\n",
      "word [DISASTER], count[8]\n",
      "word [CAN], count[4]\n",
      "word [YEARS], count[3]\n",
      "word [8B], count[3]\n",
      "word [CHEMICAL], count[1]\n",
      "word [DEADLINES], count[1]\n",
      "word [PUTIN], count[1]\n",
      "word [10B], count[1]\n",
      "word [DONT], count[14]\n",
      "word [WAPO], count[1]\n",
      "word [PTSD], count[1]\n",
      "word [MSG], count[3]\n",
      "word [FISA], count[9]\n",
      "word [PLAYERS], count[2]\n",
      "word [FOOL], count[1]\n",
      "word [PARTY], count[1]\n",
      "word [LAUGHING], count[1]\n",
      "word [KGB], count[1]\n",
      "word [VT], count[2]\n",
      "word [EMAIL], count[2]\n",
      "word [ARE], count[14]\n",
      "word [FEC], count[2]\n",
      "word [EO2], count[1]\n",
      "word [STRAIGHT], count[1]\n",
      "word [IGNITE], count[1]\n",
      "word [CHINA], count[2]\n",
      "word [TRIED], count[1]\n",
      "word [DSRL], count[1]\n",
      "word [HBO], count[3]\n",
      "word [FRONT], count[1]\n",
      "word [PRESIDENT], count[9]\n",
      "word [MIA], count[1]\n",
      "word [JULY], count[1]\n",
      "word [CLOSE], count[1]\n",
      "word [K1047], count[1]\n",
      "word [C8], count[3]\n",
      "word [RIPOFF], count[1]\n",
      "word [TO], count[57]\n",
      "word [MIGHT], count[1]\n",
      "word [CUTS], count[22]\n",
      "word [TODAY], count[15]\n",
      "word [25K], count[3]\n",
      "word [SF], count[2]\n",
      "word [BILL], count[4]\n",
      "word [SEALS], count[2]\n",
      "word [ISRAEL], count[1]\n",
      "word [VOTER], count[4]\n",
      "word [CRAZY], count[6]\n",
      "word [770M], count[1]\n",
      "word [SC], count[23]\n",
      "word [SEEN], count[3]\n",
      "word [MOSCOW], count[2]\n",
      "word [YOUR], count[8]\n",
      "word [IMPACT], count[1]\n",
      "word [POINT], count[1]\n",
      "word [COUNTRY], count[27]\n",
      "word [ALSO], count[2]\n",
      "word [TESTS], count[1]\n",
      "word [OF], count[48]\n",
      "word [EVE], count[1]\n",
      "word [STUPID], count[5]\n",
      "word [NEVER], count[43]\n",
      "word [11M], count[1]\n",
      "word [DALLAS], count[1]\n",
      "word [JUDGED], count[1]\n",
      "word [MERRY], count[5]\n",
      "word [TOUCH], count[1]\n",
      "word [SHARE], count[1]\n",
      "word [AMNESTY], count[3]\n",
      "word [SHR1], count[2]\n",
      "word [DIRECTLY], count[1]\n",
      "word [TELL], count[3]\n",
      "word [WON], count[4]\n",
      "word [400M], count[5]\n",
      "word [TWITTER], count[3]\n",
      "word [NYPD], count[12]\n",
      "word [TRANSCRIPT], count[1]\n",
      "word [REPUBLICAN], count[2]\n",
      "word [TONIGHTS], count[1]\n",
      "word [VOICE], count[1]\n",
      "word [ARMY], count[1]\n",
      "word [PUNISHMENT], count[1]\n",
      "word [SCOTT], count[1]\n",
      "word [THEE], count[1]\n",
      "word [99M], count[1]\n",
      "word [NDA], count[1]\n",
      "word [PERCENT], count[1]\n",
      "word [GRAVE], count[1]\n",
      "word [ILLEGALS], count[2]\n",
      "word [WAY], count[9]\n",
      "word [CHAIN], count[2]\n",
      "word [RESET], count[1]\n",
      "word [FIX], count[4]\n",
      "word [PATRIOTISM], count[1]\n",
      "word [S544], count[1]\n",
      "word [VERY], count[23]\n",
      "word [IRONIC], count[2]\n",
      "word [JUDGEMENT], count[5]\n",
      "word [FIRING], count[1]\n",
      "word [SECURITY], count[6]\n",
      "word [TX], count[5]\n",
      "word [FREEZING], count[1]\n",
      "word [HAPPENED], count[2]\n",
      "word [SPIED], count[2]\n",
      "word [USELESS], count[1]\n",
      "word [MONSTER], count[3]\n",
      "word [FAILED], count[5]\n",
      "word [50B], count[1]\n",
      "word [30AM], count[1]\n",
      "word [MAHER], count[1]\n",
      "word [CALL], count[2]\n",
      "word [MISTAKES], count[1]\n",
      "word [PUBLIC], count[3]\n",
      "word [MAGNIFICENT], count[2]\n",
      "word [476K], count[1]\n",
      "word [SCRUTINY], count[1]\n",
      "word [EDITORIAL], count[1]\n",
      "word [OT], count[1]\n",
      "word [173K], count[1]\n",
      "word [MUSKET], count[1]\n",
      "word [COURT], count[5]\n",
      "word [39M], count[1]\n",
      "word [52M], count[1]\n",
      "word [SERVER], count[1]\n",
      "word [UNITED], count[3]\n",
      "word [WOMEN], count[3]\n",
      "word [NJ], count[16]\n",
      "word [O], count[57]\n",
      "word [COUNTY], count[2]\n",
      "word [POLICE], count[5]\n",
      "word [OPPRESSION], count[1]\n",
      "word [RNC], count[18]\n",
      "word [GM], count[2]\n",
      "word [DOD], count[2]\n",
      "word [HELP], count[2]\n",
      "word [CITIES], count[2]\n",
      "word [DEPORTATION], count[1]\n",
      "word [RADICAL], count[10]\n",
      "word [LESS], count[4]\n",
      "word [THUNDER], count[1]\n",
      "word [STARS], count[1]\n",
      "word [MONEY], count[4]\n",
      "word [CRUZ], count[7]\n",
      "word [WORK], count[9]\n",
      "word [ESCALATES], count[1]\n",
      "word [LAWFARE], count[1]\n",
      "word [UMC], count[1]\n",
      "word [KIM], count[4]\n",
      "word [PEOPLES], count[1]\n",
      "word [ROLLING], count[1]\n",
      "word [SEASON], count[1]\n",
      "word [DECREASED], count[1]\n",
      "word [FAVOR], count[1]\n",
      "word [AM], count[42]\n",
      "word [AGENT], count[1]\n",
      "word [AILES], count[1]\n",
      "word [MOINES], count[1]\n",
      "word [UP], count[10]\n",
      "word [EST], count[47]\n",
      "word [KEEP], count[2]\n",
      "word [CAUCUS], count[1]\n",
      "word [CRS], count[1]\n",
      "word [TERRORISM], count[5]\n",
      "word [FREE], count[4]\n",
      "word [WILL], count[49]\n",
      "word [SPENT], count[1]\n",
      "word [RETALIATE], count[1]\n",
      "word [96K], count[1]\n",
      "word [RUBIO], count[6]\n",
      "word [DEMS], count[2]\n",
      "word [NORTH], count[3]\n",
      "word [JP], count[3]\n",
      "word [RISES], count[1]\n",
      "word [SURGES], count[1]\n",
      "word [BATTLES], count[1]\n",
      "word [LIKE], count[7]\n",
      "word [LEADING], count[1]\n",
      "word [WT], count[1]\n",
      "word [COLLECTION™], count[1]\n",
      "word [ISLAM], count[1]\n",
      "word [SHORT], count[3]\n",
      "word [CIRCUITS], count[1]\n",
      "word [DEM], count[1]\n",
      "word [ROOM], count[1]\n",
      "word [INCR], count[1]\n",
      "word [CEA], count[1]\n",
      "word [12T], count[2]\n",
      "word [CC], count[2]\n",
      "word [THEM], count[4]\n",
      "word [REPLACED], count[1]\n",
      "word [WINNING], count[3]\n",
      "word [1BILLION], count[1]\n",
      "word [140M], count[1]\n",
      "word [POWERFUL], count[1]\n",
      "word [WALL], count[37]\n",
      "word [VOTES], count[1]\n",
      "word [EVERYTHING], count[3]\n",
      "word [NONE], count[1]\n",
      "word [FOX2], count[1]\n",
      "word [VIOLENCE], count[1]\n",
      "word [KILLER], count[1]\n",
      "word [GOOD], count[2]\n",
      "word [ADMIRALS], count[1]\n",
      "word [BLT], count[1]\n",
      "word [GROWTH], count[3]\n",
      "word [STRONGLY], count[1]\n",
      "word [CEILING], count[2]\n",
      "word [SPAN], count[2]\n",
      "word [PICKS], count[1]\n",
      "word [MIAMI], count[2]\n",
      "word [BUSH], count[9]\n",
      "word [DACA], count[58]\n",
      "word [ALWAYS], count[9]\n",
      "word [SWAMP], count[3]\n",
      "word [OHPA], count[1]\n",
      "word [SMACKS], count[1]\n",
      "word [SHOOT], count[2]\n",
      "word [URGENT], count[1]\n",
      "word [PRIDE], count[4]\n",
      "word [SARCASM], count[1]\n",
      "word [Q3], count[2]\n",
      "word [CELEBRITY], count[2]\n",
      "word [SPEAKER], count[1]\n",
      "word [ROLL], count[1]\n",
      "word [18T], count[2]\n",
      "word [PANEL], count[1]\n",
      "word [ARIZONA], count[3]\n",
      "word [Y752], count[2]\n",
      "word [LIMITED], count[5]\n",
      "word [200M], count[9]\n",
      "word [COST], count[1]\n",
      "word [BOSTON], count[1]\n",
      "word [DELUSIONAL], count[1]\n",
      "word [ASAP], count[10]\n",
      "word [MAJOR], count[3]\n",
      "word [UNDER], count[1]\n",
      "word [COMRADES], count[1]\n",
      "word [PLANS], count[1]\n",
      "word [K104], count[1]\n",
      "word [487B], count[2]\n",
      "word [SHARPTON], count[1]\n",
      "word [TRAIN], count[1]\n",
      "word [90T], count[1]\n",
      "word [GAME], count[1]\n",
      "word [VAST], count[1]\n",
      "word [ESTABLISHMENT], count[1]\n",
      "word [FORTUNE], count[1]\n",
      "word [CHAMPION], count[2]\n",
      "word [DLG], count[1]\n",
      "word [RENTA], count[1]\n",
      "word [CLUELESS], count[1]\n",
      "word [DESTROYS], count[1]\n",
      "word [AGAIN], count[260]\n",
      "word [FIORINA], count[1]\n",
      "word [POLL], count[12]\n",
      "word [AC], count[1]\n",
      "word [SI], count[1]\n",
      "word [LEADER], count[2]\n",
      "word [IMMIGRATION], count[9]\n",
      "word [TELEVISION], count[1]\n",
      "word [61RTG], count[1]\n",
      "word [FACT], count[5]\n",
      "word [COVERAGE], count[1]\n",
      "word [FORMAL], count[1]\n",
      "word [STATES], count[4]\n",
      "word [ISIL], count[2]\n",
      "word [BP], count[3]\n",
      "word [Q], count[22]\n",
      "word [PEAC], count[1]\n",
      "word [STAMINA], count[2]\n",
      "word [MUCH], count[13]\n",
      "word [MIGRATION], count[2]\n",
      "word [VIRGINIA], count[1]\n",
      "word [FFC], count[1]\n",
      "word [PROSPERITY], count[4]\n",
      "word [OPSEC], count[1]\n",
      "word [7B], count[3]\n",
      "word [FORTY], count[1]\n",
      "word [EASTERN], count[2]\n",
      "word [ABLE], count[1]\n",
      "word [MACYS], count[2]\n",
      "word [PROUD], count[2]\n",
      "word [HALF], count[1]\n",
      "word [SELECT], count[1]\n",
      "word [MS], count[18]\n",
      "word [LIFETIME], count[1]\n",
      "word [LEAST], count[1]\n",
      "word [WOW], count[17]\n",
      "word [9M], count[1]\n",
      "word [INCOMPETENCE], count[1]\n",
      "word [FORGET], count[1]\n",
      "word [POOR], count[2]\n",
      "word [SERTA], count[1]\n",
      "word [BULLS], count[1]\n",
      "word [DUMBEST], count[2]\n",
      "word [WRECK], count[1]\n",
      "word [DOW], count[3]\n",
      "word [CHECKING], count[1]\n",
      "word [EMBARRASSMENT], count[1]\n",
      "word [PAY], count[6]\n",
      "word [ROUNDS], count[1]\n",
      "word [INDIANA], count[1]\n",
      "word [DORAL], count[1]\n",
      "word [ATTACK], count[2]\n",
      "word [JOAN], count[1]\n",
      "word [BLESS], count[4]\n",
      "word [JOB], count[7]\n",
      "word [OATH], count[1]\n",
      "word [8C], count[2]\n",
      "word [HANDS], count[1]\n",
      "word [NAFW], count[2]\n",
      "word [STRICKEN], count[1]\n",
      "word [AMERICANS🇺🇸], count[1]\n",
      "word [THINGS], count[1]\n",
      "word [LOANS], count[1]\n",
      "word [120K], count[1]\n",
      "word [OTHER], count[2]\n",
      "word [FRAUDULENT], count[1]\n",
      "word [GOT], count[2]\n",
      "word [RAN], count[2]\n",
      "word [OIL], count[5]\n",
      "word [47M], count[1]\n",
      "word [BLESSINGS], count[1]\n",
      "word [HOME], count[4]\n",
      "word [FR], count[1]\n",
      "word [CBN], count[1]\n",
      "word [OJ], count[1]\n",
      "word [DREAM], count[3]\n",
      "word [8PM], count[11]\n",
      "word [CFPB], count[2]\n",
      "word [FLAILING], count[1]\n",
      "word [COMPLETE], count[2]\n",
      "word [REFORMS], count[1]\n",
      "word [AD], count[1]\n",
      "word [75X], count[1]\n",
      "word [64B], count[1]\n",
      "word [MELANIA], count[1]\n",
      "word [ZONES], count[1]\n",
      "word [SECRET], count[1]\n",
      "word [C], count[196]\n",
      "word [CRIPPLED], count[20]\n",
      "word [HE], count[6]\n",
      "word [BORROW], count[1]\n",
      "word [RAISE], count[1]\n",
      "word [1B], count[11]\n",
      "word [TAXED], count[2]\n",
      "word [BETWEEN], count[1]\n",
      "word [GROSS], count[1]\n",
      "word [FIRST], count[21]\n",
      "word [L30809], count[1]\n",
      "word [ZERO], count[27]\n",
      "word [HELL], count[3]\n",
      "word [CPW], count[1]\n",
      "word [ESTATE], count[1]\n",
      "word [DESTINY], count[1]\n",
      "word [SAY], count[2]\n",
      "word [DEFUND], count[3]\n",
      "word [AMERICAS], count[2]\n",
      "word [AGAINST], count[3]\n",
      "word [EO1], count[1]\n",
      "word [64K], count[1]\n",
      "word [MASTERMINDS], count[1]\n",
      "word [MICHIGAN], count[1]\n",
      "word [APPRENTICE], count[4]\n",
      "word [INFORMATION], count[2]\n",
      "word [MEN], count[1]\n",
      "word [DISGRACEFUL], count[1]\n",
      "word [PROBABLY], count[1]\n",
      "word [1T], count[7]\n",
      "word [3O], count[1]\n",
      "word [JOKE], count[4]\n",
      "word [FIRES], count[2]\n",
      "word [63K], count[1]\n",
      "word [LAUGHS], count[1]\n",
      "word [CANT], count[1]\n",
      "word [WINTERS], count[1]\n",
      "word [SSE], count[1]\n",
      "word [EARNED], count[2]\n",
      "word [ANNOUNCES], count[1]\n",
      "word [LANDING], count[1]\n",
      "word [AIPAC], count[1]\n",
      "word [4T], count[4]\n",
      "word [PILLARS], count[1]\n",
      "word [BOMB], count[1]\n",
      "word [ESPN], count[7]\n",
      "word [MLB], count[1]\n",
      "word [GQ], count[2]\n",
      "word [FIVE], count[1]\n",
      "word [POST], count[1]\n",
      "word [FUN], count[1]\n",
      "word [FINALE], count[3]\n",
      "word [TRUMPS], count[3]\n",
      "word [PUT], count[2]\n",
      "word [RESPONSE], count[3]\n",
      "word [92M], count[1]\n",
      "word [GOP], count[135]\n",
      "word [10T], count[1]\n",
      "word [INTERESTING], count[1]\n",
      "word [FMR], count[1]\n",
      "word [FAME], count[1]\n",
      "word [LOT], count[1]\n",
      "word [DISHONEST], count[3]\n",
      "word [SOLDIERS], count[1]\n",
      "word [RESORT], count[1]\n",
      "word [ET], count[56]\n",
      "word [FIFTY], count[1]\n",
      "word [BLOCK], count[1]\n",
      "word [SMART], count[7]\n",
      "word [TRAVEL], count[3]\n",
      "word [NEBRASKA], count[1]\n",
      "word [RECORDS], count[1]\n",
      "word [COLLEGE], count[1]\n",
      "word [TRILLION], count[4]\n",
      "word [USGA], count[1]\n",
      "word [00PM], count[1]\n",
      "word [GENERATIONS], count[1]\n",
      "word [BILLION], count[13]\n",
      "word [2A], count[3]\n",
      "word [PRES], count[2]\n",
      "word [RE], count[2]\n",
      "word [VICTORIES], count[1]\n",
      "word [FIGHTER], count[1]\n",
      "word [GRIT], count[1]\n",
      "word [FACE], count[1]\n",
      "word [COVERUP], count[1]\n",
      "word [CONSERVATIVE], count[2]\n",
      "word [EVIDENCE], count[3]\n",
      "word [RECIPROCAL], count[1]\n",
      "word [READ], count[8]\n",
      "word [SG], count[2]\n",
      "word [FEATURING], count[1]\n",
      "word [PASS], count[2]\n",
      "word [COURSE], count[1]\n",
      "word [DUMB], count[6]\n",
      "word [INCREDIBLE], count[2]\n",
      "word [BREAKING], count[2]\n",
      "word [VA], count[39]\n",
      "word [NOON], count[1]\n",
      "word [CT], count[5]\n",
      "word [DEFEND], count[1]\n",
      "word [NYTIMES], count[1]\n",
      "word [DOWNSIDE], count[1]\n",
      "word [COMPETITIVE], count[1]\n",
      "word [50K], count[2]\n",
      "word [ART], count[5]\n",
      "word [FIELD], count[2]\n",
      "word [LC], count[1]\n",
      "word [MIDNIGHT], count[2]\n",
      "word [SPLASHY], count[1]\n",
      "word [S10], count[1]\n",
      "word [BLASTS], count[2]\n",
      "word [BACK], count[13]\n",
      "word [MASSIVE], count[9]\n",
      "word [CAMPAIGNS], count[1]\n",
      "word [147M], count[1]\n",
      "word [PLAY], count[4]\n",
      "word [AVE], count[1]\n",
      "word [NARRATIVE], count[1]\n",
      "word [20K], count[2]\n",
      "word [O0], count[1]\n",
      "word [WARRIORS], count[1]\n",
      "word [CRIMINAL], count[4]\n",
      "word [FOOTBALL], count[1]\n",
      "word [CNMI], count[1]\n",
      "word [TD], count[1]\n",
      "word [CONCACAF], count[1]\n",
      "word [PALMETTO], count[1]\n",
      "word [UPSIDE], count[1]\n",
      "word [SWEAT], count[1]\n",
      "word [NCAA], count[4]\n",
      "word [557B], count[1]\n",
      "word [MINNESOTA], count[1]\n",
      "word [AROUND], count[1]\n",
      "word [ENJOY], count[27]\n",
      "word [ACCUMULATE], count[1]\n",
      "word [GOING], count[1]\n",
      "word [HHS], count[2]\n",
      "word [ZTE], count[3]\n",
      "word [UBS], count[1]\n",
      "word [HEALTHY], count[1]\n",
      "word [AIG], count[3]\n",
      "word [CARD], count[2]\n",
      "word [GROWING], count[1]\n",
      "word [TBI], count[1]\n",
      "word [APP], count[1]\n",
      "word [TOUGH], count[3]\n",
      "word [MERCY], count[1]\n",
      "word [LEAK], count[1]\n",
      "word [EMERGENCY], count[1]\n",
      "word [BEAUTY], count[1]\n",
      "word [USS], count[3]\n",
      "word [10PM], count[12]\n",
      "word [TV1], count[1]\n",
      "word [700B], count[1]\n",
      "word [REGISTRATION], count[1]\n",
      "word [LP], count[1]\n",
      "word [2T], count[2]\n",
      "word [SHOW], count[5]\n",
      "word [FIND], count[2]\n",
      "word [WEEKLY], count[4]\n",
      "word [CHECK], count[1]\n",
      "word [STEVENS], count[1]\n",
      "word [L], count[25]\n",
      "word [BETTER], count[4]\n",
      "word [WATCH], count[7]\n",
      "word [LINE], count[2]\n",
      "word [YORK], count[5]\n",
      "word [IF], count[8]\n",
      "word [TRY], count[2]\n",
      "word [SILENT], count[1]\n",
      "word [MD], count[2]\n",
      "word [OH], count[12]\n",
      "word [TRUMP], count[94]\n",
      "word [J], count[226]\n",
      "word [OCC], count[1]\n",
      "word [430M], count[1]\n",
      "word [MAKING], count[15]\n",
      "word [CHAMP], count[1]\n",
      "word [DEMOCRACY], count[1]\n",
      "word [GROWNG], count[1]\n",
      "word [THEY], count[10]\n",
      "word [XL], count[11]\n",
      "word [BORDER], count[15]\n",
      "word [SPEAK], count[3]\n",
      "word [NYS], count[3]\n",
      "word [HANDEL], count[1]\n",
      "word [RUN], count[3]\n",
      "word [POLLING], count[1]\n",
      "word [US], count[746]\n",
      "word [POLAR], count[3]\n",
      "word [DE], count[1]\n",
      "word [CR], count[2]\n",
      "word [WEBSITE], count[1]\n",
      "word [LOST], count[1]\n",
      "word [FALSE], count[1]\n",
      "word [SUPPORT], count[4]\n",
      "word [WALK], count[1]\n",
      "word [LEAKERS], count[1]\n",
      "word [K720], count[1]\n",
      "word [CEO], count[19]\n",
      "word [SCANDAL], count[1]\n",
      "word [BEACH], count[1]\n",
      "word [AMERICA], count[312]\n",
      "word [DECISION], count[1]\n",
      "word [WHISTLEBLOWER], count[1]\n",
      "word [GREATEST], count[5]\n",
      "word [HER], count[2]\n",
      "word [TANKED], count[1]\n",
      "word [REALLY], count[5]\n",
      "word [BLOOD], count[1]\n",
      "word [INVESTIGATION], count[3]\n",
      "word [PROGRESS], count[1]\n",
      "word [CARDS], count[1]\n",
      "word [TRULY], count[2]\n",
      "word [TRAITOR], count[1]\n",
      "word [JONG], count[3]\n",
      "word [SOLVE], count[1]\n",
      "word [UNLEASH], count[1]\n",
      "word [DROP], count[2]\n",
      "word [MOGUL], count[2]\n",
      "word [DES], count[1]\n",
      "word [PM], count[77]\n",
      "word [CIRCUITED], count[2]\n",
      "word [NH], count[32]\n",
      "word [ACCEPTABLE], count[1]\n",
      "word [AG], count[17]\n",
      "word [MONTHS], count[1]\n",
      "word [FROM], count[8]\n",
      "word [TAXES], count[4]\n",
      "word [TSA], count[1]\n",
      "word [MST], count[1]\n",
      "word [DESPERATE], count[1]\n",
      "word [EPA], count[11]\n",
      "word [EPISODE], count[2]\n",
      "word [BOTH], count[1]\n",
      "word [EXEC], count[1]\n",
      "word [THANKS], count[5]\n",
      "word [CHOICES], count[1]\n",
      "word [HOUSE], count[4]\n",
      "word [FIRED], count[7]\n",
      "word [30B], count[1]\n",
      "word [UNIVERSITY], count[1]\n",
      "word [ELIMINATED], count[1]\n",
      "word [LOVING], count[1]\n",
      "word [VETTING], count[2]\n",
      "word [CHRISTIAN], count[2]\n",
      "word [HISTORY], count[3]\n",
      "word [OUT], count[9]\n",
      "word [CBO], count[12]\n",
      "word [DID], count[3]\n",
      "word [FANTASTIC], count[7]\n",
      "word [FACTS], count[1]\n",
      "word [GOD], count[10]\n",
      "word [ATTACKS], count[2]\n",
      "word [EVEN], count[1]\n",
      "word [HEADING], count[1]\n",
      "word [23B], count[1]\n",
      "word [REFUGEES], count[1]\n",
      "word [SHAKING], count[1]\n",
      "word [7AM], count[1]\n",
      "word [VISAS], count[2]\n",
      "word [GMT], count[1]\n",
      "word [635M], count[4]\n",
      "word [HC], count[1]\n",
      "word [DJT], count[11]\n",
      "word [H], count[56]\n",
      "word [SENT], count[1]\n",
      "word [DRAIN], count[3]\n",
      "word [COME], count[6]\n",
      "word [PC], count[1]\n",
      "word [MONDAY], count[2]\n",
      "word [REPEAL], count[14]\n",
      "word [WELCOME], count[2]\n",
      "word [288M], count[1]\n",
      "word [STAKE], count[1]\n",
      "word [ASA], count[1]\n",
      "word [DOMINATING], count[1]\n",
      "word [NIGHT], count[2]\n",
      "word [SEVEN], count[1]\n",
      "word [RAIDS], count[1]\n",
      "word [11AM], count[6]\n",
      "word [ARRIVAL], count[1]\n",
      "word [AL], count[4]\n",
      "word [GROW], count[2]\n",
      "word [LEAKS], count[2]\n",
      "word [DESPERATION], count[1]\n",
      "word [PRODUCT], count[1]\n",
      "word [WSJ], count[10]\n",
      "word [APPROVAL], count[1]\n",
      "word [STORY], count[1]\n",
      "word [150K], count[1]\n",
      "word [WAKE], count[1]\n",
      "word [500M], count[2]\n",
      "word [POLITICAL], count[1]\n",
      "word [10X], count[1]\n",
      "word [PRO], count[3]\n",
      "word [COAL], count[1]\n",
      "word [13B], count[1]\n",
      "word [5PM], count[4]\n",
      "word [TOUGHER], count[1]\n",
      "word [III], count[4]\n",
      "word [TERROR], count[2]\n",
      "word [SET], count[1]\n",
      "word [MAKE], count[256]\n",
      "word [TOWER], count[1]\n",
      "word [OBSTRUCTION], count[1]\n",
      "word [NYC], count[90]\n",
      "word [PAID], count[1]\n",
      "word [ROK], count[1]\n",
      "word [NRO], count[1]\n",
      "word [RANT], count[1]\n",
      "word [DISGRACE], count[1]\n",
      "word [FUNDS], count[1]\n",
      "word [NEWS], count[55]\n",
      "word [7T], count[2]\n",
      "word [105M], count[1]\n",
      "word [CAMPAIGN], count[5]\n",
      "word [GRAMMY], count[1]\n",
      "word [QVC], count[11]\n",
      "word [600M], count[1]\n",
      "word [EBOLA], count[4]\n",
      "word [AP], count[3]\n",
      "word [INFORM], count[1]\n",
      "word [THAT], count[11]\n",
      "word [OPENING], count[1]\n",
      "word [UNLOADS], count[1]\n",
      "word [DOES], count[1]\n",
      "word [CLINTONS], count[3]\n",
      "word [1M], count[12]\n",
      "word [ARTICLE], count[1]\n",
      "word [15K], count[1]\n",
      "word [70B], count[1]\n",
      "word [POINTS], count[1]\n",
      "word [PAC], count[7]\n",
      "word [UMASS], count[1]\n",
      "word [REBUILD], count[6]\n",
      "word [44M], count[1]\n",
      "word [BEG], count[1]\n",
      "word [ORGS], count[1]\n",
      "word [FOUR], count[3]\n",
      "word [FOR], count[26]\n",
      "word [BILLY], count[1]\n",
      "word [SANCHEZ], count[1]\n",
      "word [NFL], count[38]\n",
      "word [MIDWEST], count[1]\n",
      "word [MEDAL], count[1]\n",
      "word [NOT], count[41]\n",
      "word [ARG], count[1]\n",
      "word [D49], count[1]\n",
      "word [SELDOM], count[1]\n",
      "word [RIDICULOUS], count[1]\n",
      "word [STUMPS], count[1]\n",
      "word [PHOENIX], count[1]\n",
      "word [MILLIONS], count[3]\n",
      "word [BQ], count[1]\n",
      "word [ALMOST], count[1]\n",
      "word [SEXISM], count[1]\n",
      "word [REFUSE], count[1]\n",
      "word [INNER], count[1]\n",
      "word [REAL], count[6]\n",
      "word [FOUNDERS], count[1]\n",
      "word [WTO], count[2]\n",
      "word [70M], count[2]\n",
      "word [HEROES], count[18]\n",
      "word [ON], count[28]\n",
      "word [AND], count[62]\n",
      "word [PASSION], count[1]\n",
      "word [PLUNDER], count[1]\n",
      "word [BRING], count[3]\n",
      "word [IT], count[21]\n",
      "word [GRETA], count[1]\n",
      "word [STAND], count[3]\n",
      "word [FOOLISH], count[1]\n",
      "word [ME], count[14]\n",
      "word [3B], count[1]\n",
      "word [COUP], count[1]\n",
      "word [SEPTEMBER], count[1]\n",
      "word [2M], count[4]\n",
      "word [SUMMIT], count[1]\n",
      "word [TEXTS], count[1]\n",
      "word [DRAFT], count[1]\n",
      "word [ASIA], count[1]\n",
      "word [CHOICE], count[2]\n",
      "word [INCREASE], count[3]\n",
      "word [ENEMIES], count[1]\n",
      "word [MYRTLE], count[1]\n",
      "word [THEN], count[1]\n",
      "word [BAN], count[6]\n",
      "word [NIA], count[1]\n",
      "word [CRIME], count[2]\n",
      "word [TY], count[1]\n",
      "word [MEDIA], count[4]\n",
      "word [IDF], count[2]\n",
      "word [’THE], count[1]\n",
      "word [SAFE], count[47]\n",
      "word [ENFORCE], count[1]\n",
      "word [ISLAMIC], count[9]\n",
      "word [POLITICS], count[1]\n",
      "word [FLORIDA], count[9]\n",
      "word [LOBBYIST], count[1]\n",
      "word [LAST], count[4]\n",
      "word [CARE], count[4]\n",
      "word [WAS], count[5]\n",
      "word [LONG], count[5]\n",
      "word [FEDERAL], count[1]\n",
      "word [GDP], count[22]\n",
      "word [S1964], count[1]\n",
      "word [BILATERAL], count[1]\n",
      "word [VIOLATION], count[1]\n",
      "word [OC], count[4]\n",
      "word [WALKERS], count[1]\n",
      "word [MN], count[4]\n",
      "word [❌LIBYA❌SYRIA❌IRAN❌IRAQ❌ASIA], count[2]\n",
      "word [LIVE], count[36]\n",
      "word [PPP], count[7]\n",
      "word [DAY], count[11]\n",
      "word [RTV], count[1]\n",
      "word [FLIGHTS], count[6]\n",
      "word [ROLLOUT], count[1]\n",
      "word [BREAKDOWN], count[1]\n",
      "word [FOREIGN], count[1]\n",
      "word [SORRY], count[1]\n",
      "word [TAX], count[35]\n",
      "word [MUST], count[34]\n",
      "word [DREAMS], count[1]\n",
      "word [LGBT], count[2]\n",
      "word [DEMONSTRATED], count[1]\n",
      "word [COSTLY], count[1]\n",
      "word [DONE], count[1]\n",
      "word [PUMMEL], count[1]\n",
      "word [FIRE], count[2]\n",
      "word [TOTAL], count[16]\n",
      "word [TOGETHER], count[7]\n",
      "word [FAMILIES], count[2]\n",
      "word [CITY], count[2]\n",
      "word [AFTER], count[8]\n",
      "word [SPECIAL], count[2]\n",
      "word [20PM], count[1]\n",
      "word [WITH], count[7]\n",
      "word [BEAR], count[1]\n",
      "word [TPA], count[1]\n",
      "word [GOVERNMENT], count[4]\n",
      "word [GUN], count[1]\n",
      "word [APOLOGIZE], count[1]\n",
      "word [ONES], count[2]\n",
      "word [DAN], count[1]\n",
      "word [SHOULD], count[5]\n",
      "word [LEE], count[1]\n",
      "word [DECEPTION], count[1]\n",
      "word [PLAIN], count[1]\n",
      "word [NYE], count[2]\n",
      "word [USA], count[90]\n",
      "word [CONGRESS], count[3]\n",
      "word [MELTDOWN], count[1]\n",
      "word [29T], count[1]\n",
      "word [JV], count[1]\n",
      "word [AWOL], count[1]\n",
      "word [LEAKING], count[1]\n",
      "word [HERO], count[1]\n",
      "word [WAVE], count[1]\n",
      "word [FOUND], count[2]\n",
      "word [RESTORE], count[2]\n",
      "word [HMX], count[1]\n",
      "word [FLIGHT], count[1]\n",
      "word [CUTCAP], count[1]\n",
      "word [50OOO], count[1]\n",
      "word [CH], count[1]\n",
      "word [TRILLIONS], count[2]\n",
      "word [PREVAIL], count[1]\n",
      "word [COULD], count[1]\n",
      "word [HARBORING], count[1]\n",
      "word [SIGNUPS], count[1]\n",
      "word [JERSEY], count[1]\n",
      "word [MANIFESTO], count[1]\n",
      "word [COLLECTION], count[1]\n",
      "word [WORST], count[3]\n",
      "word [CHRIS], count[1]\n",
      "word [COUNTRIES], count[1]\n",
      "word [OUR], count[43]\n",
      "word [IRAQ], count[1]\n",
      "word [BET], count[1]\n",
      "word [MEXICO], count[1]\n",
      "word [BH], count[2]\n",
      "word [GAINS], count[1]\n",
      "word [SOUTHCOM], count[1]\n",
      "word [PR], count[9]\n",
      "word [DISGUSTING], count[1]\n",
      "word [STRATEGIC], count[1]\n",
      "word [OPO], count[4]\n",
      "word [LOW], count[1]\n",
      "word [FINDS], count[1]\n",
      "word [ORC], count[1]\n",
      "word [GIFT], count[1]\n",
      "word [7A], count[1]\n",
      "word [WINS], count[3]\n",
      "word [BEAUTIFUL], count[4]\n",
      "word [TERM], count[1]\n",
      "word [MUSLIMS], count[1]\n",
      "word [TARP], count[1]\n",
      "word [LOWER], count[4]\n",
      "word [30M], count[3]\n",
      "word [AWFUL], count[1]\n",
      "word [50M], count[2]\n",
      "word [WEEK], count[1]\n",
      "word [DOLLARS💰RICHER], count[1]\n",
      "word [WORSHIP], count[2]\n",
      "word [NATIONAL], count[10]\n",
      "word [UL], count[1]\n",
      "word [ORANGE], count[1]\n",
      "word [17T], count[7]\n",
      "word [DUMP], count[1]\n",
      "word [LOVE], count[9]\n",
      "word [NEED], count[7]\n",
      "word [FLASHBACK], count[7]\n",
      "word [68M], count[1]\n",
      "word [NHL], count[1]\n",
      "word [OWNED], count[1]\n",
      "word [FAST], count[13]\n",
      "word [P], count[159]\n",
      "word [PATRIOT], count[1]\n",
      "word [EXCLUSIVE], count[10]\n",
      "word [460M], count[1]\n",
      "word [FABRICATE], count[1]\n",
      "word [TIES], count[1]\n",
      "word [ALGAE], count[1]\n",
      "word [LINKS], count[1]\n",
      "word [EVER], count[14]\n",
      "word [SHOULDNT], count[1]\n",
      "word [FDA], count[2]\n",
      "word [BASE], count[1]\n",
      "word [UNTRUE], count[1]\n",
      "word [G20], count[4]\n",
      "word [CHRISTMAS], count[5]\n",
      "word [JUDGES], count[1]\n",
      "word [N8], count[1]\n",
      "word [TOP], count[4]\n",
      "word [TRADE], count[7]\n",
      "word [CANTORS], count[1]\n",
      "word [WE], count[51]\n",
      "word [IPSOS], count[1]\n",
      "word [SPINNING], count[1]\n",
      "word [NASCAR], count[3]\n",
      "word [WANTS], count[1]\n",
      "word [GREAT], count[384]\n",
      "word [CAP], count[5]\n",
      "word [TEAMWORK], count[1]\n",
      "word [CHELSEA], count[1]\n",
      "word [8M], count[2]\n",
      "word [DOWN], count[8]\n",
      "word [CLINTON], count[10]\n",
      "word [9PM], count[23]\n",
      "word [FOCUS], count[3]\n",
      "word [BOMBING], count[1]\n",
      "word [REUTERS], count[1]\n",
      "word [CATHOLIC], count[1]\n",
      "word [FEMA], count[11]\n",
      "word [CM], count[2]\n",
      "word [MATHEMATICALLY], count[1]\n",
      "word [MANS], count[1]\n",
      "word [MAJORS], count[2]\n",
      "word [S1], count[2]\n",
      "word [VARIOUS], count[1]\n",
      "word [MONUMENT], count[2]\n",
      "word [BOOMING], count[2]\n",
      "word [LET], count[4]\n",
      "word [EVERYONE], count[5]\n",
      "word [ENDS], count[1]\n",
      "word [SERIOUS], count[1]\n",
      "word [IPO], count[1]\n",
      "word [I], count[4267]\n",
      "word [KOREAN], count[1]\n",
      "word [TNGC], count[2]\n",
      "word [SUPREME], count[1]\n",
      "word [SAW], count[2]\n",
      "word [FUTURES], count[1]\n",
      "word [FABRICATION], count[2]\n",
      "word [YOU], count[141]\n",
      "word [SHELL], count[1]\n",
      "word [GOTV], count[4]\n",
      "word [HURRY], count[1]\n",
      "word [ACU], count[3]\n",
      "word [Z], count[14]\n",
      "word [VALOR], count[1]\n",
      "word [250M], count[6]\n",
      "word [FIGHT], count[6]\n",
      "word [BATTLE], count[1]\n",
      "word [MOST], count[1]\n",
      "word [BEAT], count[3]\n",
      "word [TROUBLE], count[1]\n",
      "word [ENTIRE], count[1]\n",
      "word [NEEDS], count[2]\n",
      "word [AS], count[7]\n",
      "word [FRIDAY], count[1]\n",
      "word [DOSSIER], count[2]\n",
      "word [00A], count[1]\n",
      "word [BILLS], count[1]\n",
      "word [HARD], count[1]\n",
      "word [TUESDAY], count[1]\n",
      "word [PASTORS], count[2]\n",
      "word [DC], count[50]\n",
      "word [IA], count[9]\n",
      "word [HUNT], count[7]\n",
      "word [DRUG], count[3]\n",
      "word [DO], count[9]\n",
      "word [TOMORROW], count[4]\n",
      "word [PRIZE], count[1]\n",
      "word [POLLS], count[2]\n",
      "word [KNOW], count[2]\n",
      "word [SAFETY], count[1]\n",
      "word [CNNS], count[1]\n",
      "word [HAUNT], count[1]\n",
      "word [LEGAL], count[2]\n",
      "word [75M], count[1]\n",
      "word [17M], count[1]\n",
      "word [WATERTOWN], count[1]\n",
      "word [RED], count[5]\n",
      "word [DESERVE], count[1]\n",
      "word [VIDEO], count[4]\n",
      "word [NUCLEAR], count[2]\n",
      "word [Y], count[44]\n",
      "word [NOTHING], count[30]\n",
      "word [OBAMATRADE], count[1]\n",
      "word [ERO], count[1]\n",
      "word [TERRORIST], count[1]\n",
      "word [GAMES], count[1]\n",
      "word [DEPTH], count[1]\n",
      "word [HEALTHCARE], count[2]\n",
      "word [OFF], count[2]\n",
      "word [WSOC], count[2]\n",
      "word [MILITARY], count[4]\n",
      "word [BEEN], count[3]\n",
      "word [GE], count[1]\n",
      "word [WA], count[1]\n",
      "word [KAINE], count[1]\n",
      "word [DEAD], count[5]\n",
      "word [LADY], count[1]\n",
      "word [POTENTIAL], count[2]\n",
      "word [AR], count[1]\n",
      "word [WH], count[43]\n",
      "word [ARRESTED], count[1]\n",
      "word [FERRY], count[1]\n",
      "word [CORRUPT], count[3]\n",
      "word [9B], count[1]\n",
      "word [FRAUD], count[5]\n",
      "word [CONFIDENCE], count[1]\n",
      "word [VIP], count[3]\n",
      "word [BAILOUTS], count[2]\n",
      "word [HOURS], count[1]\n",
      "word [SCHEME], count[1]\n",
      "word [MO], count[1]\n",
      "word [SOME], count[1]\n"
     ]
    }
   ],
   "source": [
    "for word in words_uppercase:\n",
    "    \n",
    "    print( \"word [%s], count[%d]\" % ( word, word_counts[ word ] ) ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len( tokens_unique ) == len( tokens_unique_lowercase ) True\n",
      "Words not in embeddings_index: 2058\n",
      "len( tokenizer.word_index ) 24497\n",
      "len( embeddings_index ) 22439\n",
      "vocab_size 24498\n"
     ]
    }
   ],
   "source": [
    "print( \"len( tokens_unique ) == len( tokens_unique_lowercase )\", len( tokens_unique ) == len( tokens_unique_lowercase ) )\n",
    "print( \"Words not in embeddings_index:\", len( tokens_unique_lowercase ) - len( embeddings_index ) )\n",
    "#print( embeddings_index[ \"the\" ] )\n",
    "print( \"len( tokenizer.word_index )\", len( tokenizer.word_index ) )\n",
    "print( \"len( embeddings_index )\", len( embeddings_index ) )\n",
    "print( \"vocab_size\", vocab_size )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transform into Matrix That Maps Coefs by Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Missing words: 2058\n",
      "embedding_matrix_words 24498\n",
      "embedding_matrix.shape (24498, 300)\n",
      "embedding_matrix[ 0 ] [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      "embedding_matrix_words[ 0 ] DUMMY_KLUDGE_DUMMY_KLUDGE_DUMMY_KLUDGE\n"
     ]
    }
   ],
   "source": [
    "# create a weight matrix for words in training docs\n",
    "embedding_matrix = np.zeros( ( vocab_size, embeddings_dimension ) )\n",
    "embedding_matrix_words = [ \"DUMMY_KLUDGE_DUMMY_KLUDGE_DUMMY_KLUDGE\" ] # first word in list should be dummy\n",
    "missing_words = []\n",
    "\n",
    "# we need this to create empty coefficients array\n",
    "dummy_shape = embeddings_index[ \"the\" ].shape\n",
    "\n",
    "# use the mixed case list: tokens_unique\n",
    "#for i, word in enumerate( tokens_unique ):\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    \n",
    "    embedding_vector = embeddings_index.get( word )\n",
    "    \n",
    "    # not all words in our token list are in the wikipedia 400K set!\n",
    "    if embedding_vector is None:\n",
    "        \n",
    "        # 1st time, get the lowercase vector\n",
    "        embedding_vector = embeddings_index.get( word.lower() )\n",
    "        \n",
    "    # 2nd test: If not found as original or lower case, then assign it an empty vector\n",
    "    if embedding_vector is None:  \n",
    "        \n",
    "        # report and create empty coefficients array\n",
    "        missing_words.append( word )\n",
    "        embedding_vector = np.zeros( dummy_shape )\n",
    "     \n",
    "    #print( \"i\", i, \"word\", word )\n",
    "    embedding_matrix[ i ] = embedding_vector\n",
    "    embedding_matrix_words.append( word )\n",
    "    \n",
    "print( \"Missing words:\", len( missing_words ) )\n",
    "print( \"embedding_matrix_words\", len( embedding_matrix_words ) )\n",
    "print( \"embedding_matrix.shape\", embedding_matrix.shape )\n",
    "print( \"embedding_matrix[ 0 ]\", embedding_matrix[ 0 ] )\n",
    "print( \"embedding_matrix_words[ 0 ]\", embedding_matrix_words[ 0 ] )\n",
    "\n",
    "# before hashtag segmentation Missing words: 5640\n",
    "# after hashtag segmentation: 5060"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write Tokenizer and Embeddings Matrix to Local Storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the tokenizer\n",
    "dump( tokenizer, open( tokenizer_path, 'wb' ) )\n",
    "\n",
    "# save embedding_matrix based on wiki embeddings, complete w/ missing coefficients array dummies\n",
    "dump( embedding_matrix, open( embedding_path, 'wb' ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['0017',\n",
       " '00A',\n",
       " '00AM',\n",
       " '00PM',\n",
       " '00am',\n",
       " '00amE',\n",
       " '00pm',\n",
       " '00pmE',\n",
       " '00pmEST',\n",
       " '01AM',\n",
       " '068',\n",
       " '071067',\n",
       " '1000000',\n",
       " '10000000',\n",
       " '100yrs',\n",
       " '100′',\n",
       " '10543',\n",
       " '109B',\n",
       " '10T',\n",
       " '10aj',\n",
       " '10pE',\n",
       " '10pmE',\n",
       " '113826',\n",
       " '118M',\n",
       " '119000',\n",
       " '11am6',\n",
       " '11pmE',\n",
       " '120000',\n",
       " '12000000',\n",
       " '120K',\n",
       " '12400000',\n",
       " '1250000',\n",
       " '12T',\n",
       " '12months',\n",
       " '130000',\n",
       " '13164',\n",
       " '132000',\n",
       " '13375',\n",
       " '136260',\n",
       " '13K',\n",
       " '14076',\n",
       " '14789',\n",
       " '147M',\n",
       " '150000',\n",
       " '150000000',\n",
       " '15T',\n",
       " '15Trillion',\n",
       " '16500',\n",
       " '16977',\n",
       " '169B',\n",
       " '16T',\n",
       " '16’s',\n",
       " '173K',\n",
       " '17500',\n",
       " '175000',\n",
       " '179249',\n",
       " '17T',\n",
       " '180000',\n",
       " '18000000',\n",
       " '18142',\n",
       " '18290',\n",
       " '185000',\n",
       " '18T',\n",
       " '19000',\n",
       " '190000',\n",
       " '19017',\n",
       " '1950’s',\n",
       " '1967porky',\n",
       " '1970’s',\n",
       " '197M',\n",
       " '1986fed',\n",
       " '1Trillion',\n",
       " '1davidkim',\n",
       " '1fares',\n",
       " '1fos1',\n",
       " '1mcd',\n",
       " '1pmE',\n",
       " '200000',\n",
       " '2000000',\n",
       " '200000000',\n",
       " '20000📈21000📈22000📈',\n",
       " '2004📈',\n",
       " '2013juniorpga',\n",
       " '2016Scranton',\n",
       " '2017jambo',\n",
       " '20PM',\n",
       " '20T',\n",
       " '21456',\n",
       " '217Billion',\n",
       " '21T',\n",
       " '21points',\n",
       " '2200000',\n",
       " '225000',\n",
       " '227000',\n",
       " '23000',\n",
       " '23000📈this',\n",
       " '23116928',\n",
       " '24000',\n",
       " '242K',\n",
       " '242navybday',\n",
       " '250000',\n",
       " '250g',\n",
       " '2537',\n",
       " '269710',\n",
       " '2764',\n",
       " '2783',\n",
       " '283938',\n",
       " '2853',\n",
       " '288M',\n",
       " '295M',\n",
       " '296000',\n",
       " '29T',\n",
       " '2Trillion',\n",
       " '2lloyd',\n",
       " '2years',\n",
       " '2yrs',\n",
       " '300000',\n",
       " '3000000',\n",
       " '300B',\n",
       " '300ft',\n",
       " '300g',\n",
       " '3011',\n",
       " '30AM',\n",
       " '30PM',\n",
       " '30am',\n",
       " '30pm',\n",
       " '30pmE',\n",
       " '30’s',\n",
       " '31000',\n",
       " '31318',\n",
       " '31334',\n",
       " '315000',\n",
       " '31951',\n",
       " '32000',\n",
       " '3203',\n",
       " '32417',\n",
       " '3249',\n",
       " '325000',\n",
       " '3263',\n",
       " '32º',\n",
       " '33000',\n",
       " '33331',\n",
       " '33428',\n",
       " '33701',\n",
       " '350000',\n",
       " '35000000',\n",
       " '3557',\n",
       " '3590',\n",
       " '35pmE',\n",
       " '36489',\n",
       " '365000',\n",
       " '375B',\n",
       " '38000',\n",
       " '388000',\n",
       " '39M',\n",
       " '3Billion',\n",
       " '3girlsandd',\n",
       " '3nv',\n",
       " '3pm6',\n",
       " '3ssaf',\n",
       " '400000',\n",
       " '40000000',\n",
       " '40mph',\n",
       " '4155',\n",
       " '41to14',\n",
       " '42k',\n",
       " '430M',\n",
       " '432000',\n",
       " '4331',\n",
       " '4466',\n",
       " '4485',\n",
       " '45AM',\n",
       " '45pm',\n",
       " '45pmE',\n",
       " '46000',\n",
       " '460M',\n",
       " '4676',\n",
       " '46º',\n",
       " '476K',\n",
       " '4787',\n",
       " '47M',\n",
       " '48000',\n",
       " '48708',\n",
       " '487B',\n",
       " '487K',\n",
       " '49500000',\n",
       " '4992343',\n",
       " '4Trillion',\n",
       " '4geiger',\n",
       " '4pmE',\n",
       " '4randr',\n",
       " '4real',\n",
       " '4yrs',\n",
       " '500000',\n",
       " '5000000',\n",
       " '500B',\n",
       " '500Billion',\n",
       " '505K',\n",
       " '50678',\n",
       " '50OOO',\n",
       " '51152',\n",
       " '513pango',\n",
       " '51yrs',\n",
       " '522000',\n",
       " '52500',\n",
       " '53000',\n",
       " '530000',\n",
       " '55000',\n",
       " '55198',\n",
       " '557B',\n",
       " '5600000',\n",
       " '5630',\n",
       " '566000',\n",
       " '57728',\n",
       " '5783',\n",
       " '5841',\n",
       " '59101',\n",
       " '5thmcfly',\n",
       " '600000',\n",
       " '6000000',\n",
       " '6125',\n",
       " '61RTG',\n",
       " '620000',\n",
       " '63000000',\n",
       " '635000000',\n",
       " '635M',\n",
       " '63K',\n",
       " '64B',\n",
       " '650000',\n",
       " '6700000',\n",
       " '68M',\n",
       " '6T',\n",
       " '6abc',\n",
       " '700000',\n",
       " '70B',\n",
       " '7205',\n",
       " '7242',\n",
       " '7291',\n",
       " '7310',\n",
       " '7326',\n",
       " '74085825',\n",
       " '7450',\n",
       " '75000',\n",
       " '750000',\n",
       " '7531',\n",
       " '7533692',\n",
       " '7540',\n",
       " '75X',\n",
       " '766000',\n",
       " '7693',\n",
       " '770M',\n",
       " '7777',\n",
       " '77777',\n",
       " '77wabc',\n",
       " '7884',\n",
       " '790k',\n",
       " '79352',\n",
       " '7943',\n",
       " '79M',\n",
       " '79rose79',\n",
       " '7T',\n",
       " '7pmE',\n",
       " '7yr',\n",
       " '7yrs',\n",
       " '800B',\n",
       " '80’s',\n",
       " '81000',\n",
       " '82800',\n",
       " '831B',\n",
       " '8332000',\n",
       " '8344',\n",
       " '8597',\n",
       " '85B',\n",
       " '8646551',\n",
       " '88000',\n",
       " '8838',\n",
       " '88419000',\n",
       " '88921000',\n",
       " '88fan',\n",
       " '88pkane',\n",
       " '894520',\n",
       " '8T',\n",
       " '8amE',\n",
       " '8pm6',\n",
       " '8pmE',\n",
       " '8zz8',\n",
       " '900Billion',\n",
       " '905k9',\n",
       " '90T',\n",
       " '910000',\n",
       " '91RTG',\n",
       " '92M',\n",
       " '93516',\n",
       " '94000',\n",
       " '951957',\n",
       " '9658',\n",
       " '96K',\n",
       " '971fmtalk',\n",
       " '972000',\n",
       " '97m',\n",
       " '995mu',\n",
       " '99M',\n",
       " '99or1',\n",
       " '9amE',\n",
       " '9pmE',\n",
       " '9trillion',\n",
       " '9’s',\n",
       " 'A-X',\n",
       " 'A10shun',\n",
       " 'A123meoli',\n",
       " 'A123r7o',\n",
       " 'ABCCBS',\n",
       " 'ABCWashington',\n",
       " 'ADDRESS🇺🇸',\n",
       " 'AEIs',\n",
       " 'AGAIN🇺🇸',\n",
       " 'AJPerez',\n",
       " 'AMERICANS🇺🇸',\n",
       " 'Aahs5',\n",
       " 'Abc2020',\n",
       " 'Abcnews',\n",
       " 'Abedinis',\n",
       " 'Ac360',\n",
       " 'Acman84',\n",
       " 'Adam4',\n",
       " 'Adb1707',\n",
       " 'AddressJoin',\n",
       " 'Address🇺🇸',\n",
       " 'Aeroleila7',\n",
       " 'Afghanistant',\n",
       " 'Africahas',\n",
       " 'Ahm1038',\n",
       " 'Ahmadinejads',\n",
       " 'Ahmedmallick2',\n",
       " 'Ajb073',\n",
       " 'Ajdelgado13',\n",
       " 'Ajg',\n",
       " 'Ajshaw1003',\n",
       " 'AlabamaTickets',\n",
       " 'Alau2',\n",
       " 'Alconv2017',\n",
       " 'Alexleiser33',\n",
       " 'Alexs',\n",
       " 'Algemeiner',\n",
       " 'Algore',\n",
       " 'Alisyn',\n",
       " 'Alweaver22',\n",
       " 'Alxg21',\n",
       " 'Alyx518',\n",
       " 'Amauryd3',\n",
       " 'AmazonWashingtonPost',\n",
       " 'Ambs',\n",
       " 'Amelionaire69',\n",
       " 'Amercia',\n",
       " 'Americ',\n",
       " 'AmericaExecutive',\n",
       " 'Americans🇺🇸',\n",
       " 'America🇺🇸',\n",
       " 'America🇺🇸FOUR',\n",
       " 'Amieburton2010',\n",
       " 'Amishflyers66',\n",
       " 'Amonix',\n",
       " 'AmyMek',\n",
       " 'Amyg2200',\n",
       " 'An3t',\n",
       " 'Andeavor',\n",
       " 'Andey',\n",
       " 'AngieApon',\n",
       " 'Anishabhasin1',\n",
       " 'Annika59',\n",
       " 'Anniv',\n",
       " 'Apageor2',\n",
       " 'Apec2017',\n",
       " 'Apka',\n",
       " 'Arid93',\n",
       " 'Arronj1',\n",
       " 'Asean50',\n",
       " 'Ash1560',\n",
       " 'Ashonair101',\n",
       " 'Asmik',\n",
       " 'Ast99',\n",
       " 'Atfd17',\n",
       " 'Austile147',\n",
       " 'Avant24',\n",
       " 'Average223',\n",
       " 'Awh',\n",
       " 'Awr',\n",
       " 'Axlrose1996',\n",
       " 'Az08',\n",
       " 'Azcentral',\n",
       " 'B205',\n",
       " 'B3kz0r',\n",
       " 'B4bed',\n",
       " 'B550',\n",
       " 'BHOs',\n",
       " 'BREXIT',\n",
       " 'BSer',\n",
       " 'Ba11er',\n",
       " 'Babe8',\n",
       " 'Baeten',\n",
       " 'Bagans',\n",
       " 'Bahia6085',\n",
       " 'BarackObama',\n",
       " 'BarkleyFrance',\n",
       " 'Barky',\n",
       " 'BarnesandNoble',\n",
       " 'Barnini',\n",
       " 'Bats67',\n",
       " 'Bburt2367',\n",
       " 'Bday',\n",
       " 'Beachs',\n",
       " 'Beck17',\n",
       " 'Belichicks',\n",
       " 'Belloved21',\n",
       " 'Bergdahls',\n",
       " 'Bergdhal',\n",
       " 'Berghdal',\n",
       " 'Bernies',\n",
       " 'Berry87',\n",
       " 'Bertoldo1269',\n",
       " 'Bf2088',\n",
       " 'Bfree007',\n",
       " 'Bighank500',\n",
       " 'Bighurt',\n",
       " 'Bigpoppa181',\n",
       " 'Bigstack19',\n",
       " 'Billaget1',\n",
       " 'Billdaley1',\n",
       " 'Bizbash',\n",
       " 'Bizjournals',\n",
       " 'Bkfviking123',\n",
       " 'Bklynborn71',\n",
       " 'Blackdog',\n",
       " 'Blairwsoc9',\n",
       " 'Blake1',\n",
       " 'Blazey53',\n",
       " 'Bn40',\n",
       " 'Bobturner9th',\n",
       " 'Bonbon823',\n",
       " 'Bongsi21',\n",
       " 'Bostonherald',\n",
       " 'Brager',\n",
       " 'BreitbartNews',\n",
       " 'Brets',\n",
       " 'Brettnpu32',\n",
       " 'Bretty1973',\n",
       " 'Brexit',\n",
       " 'Brn2shpfrcd2wrk',\n",
       " 'Broadcoms',\n",
       " 'Broadwells',\n",
       " 'Bronxs',\n",
       " 'Brose783',\n",
       " 'Brwn',\n",
       " 'Bshapiro91',\n",
       " 'Bulgers',\n",
       " 'Bullie',\n",
       " 'Bullsh',\n",
       " 'Bun31',\n",
       " 'BuseyLand',\n",
       " 'Buseyisms',\n",
       " 'Businesswire',\n",
       " 'Bvogel56',\n",
       " 'C45',\n",
       " 'CASSmoney',\n",
       " 'CBSWashDC',\n",
       " 'CNBCs',\n",
       " 'CNNS',\n",
       " 'CNNs',\n",
       " 'COLLECTION™',\n",
       " 'CPAC2013',\n",
       " 'CUTCAP',\n",
       " 'Caar',\n",
       " 'Cam2883',\n",
       " 'Canada🇨🇦and',\n",
       " 'Careful’',\n",
       " 'Carl7',\n",
       " 'CarolinaTed',\n",
       " 'Carolina✅Ohio',\n",
       " 'Cat41',\n",
       " 'Cathco9',\n",
       " 'Cbmc',\n",
       " 'Cbs19',\n",
       " 'Ccade937',\n",
       " 'Ccalder0811',\n",
       " 'Cccabrera10',\n",
       " 'Ccv77',\n",
       " 'Cg227',\n",
       " 'Cgroll33',\n",
       " 'Chadfisher3',\n",
       " 'Chadillac107',\n",
       " 'Chalian',\n",
       " 'Champions⚾️',\n",
       " 'Chance’',\n",
       " 'Chapos',\n",
       " 'CharlottesvilleVirginia',\n",
       " 'Che85',\n",
       " 'Chef2234',\n",
       " 'Chers',\n",
       " 'Chiamboy8',\n",
       " 'Chicagos',\n",
       " 'Chicagotribune',\n",
       " 'Chid',\n",
       " 'Chididdy57',\n",
       " 'Chozick',\n",
       " 'ChrisCassidy',\n",
       " 'Cjdew67',\n",
       " 'Cjf',\n",
       " 'Ckmagic32',\n",
       " 'Clar68',\n",
       " 'Clary77',\n",
       " 'Cletendre21',\n",
       " 'Clownstick',\n",
       " 'Cmo626',\n",
       " 'Cmte',\n",
       " 'Cmtoms101',\n",
       " 'Cnnmoney',\n",
       " 'Coco471031',\n",
       " 'Codyaa722',\n",
       " 'Coffey1',\n",
       " 'Cohen1',\n",
       " 'Cokehead',\n",
       " 'Colenechia116',\n",
       " 'Colinth21',\n",
       " 'Colmery',\n",
       " 'Colvio117',\n",
       " 'Comeys',\n",
       " 'Comfynumb2012',\n",
       " 'Concentrationassessment',\n",
       " 'Conf➡️',\n",
       " 'Congressioal',\n",
       " 'Connecticuts',\n",
       " 'Conneticuts',\n",
       " 'Conserv',\n",
       " 'Core15',\n",
       " 'Corene7',\n",
       " 'Corsoc311',\n",
       " 'Corte74',\n",
       " 'Coulters',\n",
       " 'Councel',\n",
       " 'Councilhe',\n",
       " 'Country’',\n",
       " 'Courtmarshal',\n",
       " 'Cous13',\n",
       " 'Cpac13',\n",
       " 'Cpac2013',\n",
       " 'Cpac2014',\n",
       " 'Cpac2015',\n",
       " 'Cpac2018',\n",
       " 'Crainey777',\n",
       " 'Crookeds',\n",
       " 'Cruzhad',\n",
       " 'Cruzs',\n",
       " 'Crysta',\n",
       " 'CubaMemorandum',\n",
       " 'CubaVideo',\n",
       " 'CutsSecurity',\n",
       " 'Cuty',\n",
       " 'D49',\n",
       " 'D4du',\n",
       " 'DACA',\n",
       " 'DEPLORABLES',\n",
       " 'DJT',\n",
       " 'DOLLARS💰RICHER',\n",
       " 'DSRL',\n",
       " 'Da1nonlyasulony',\n",
       " 'Dabg3241',\n",
       " 'Dafny01',\n",
       " 'DailyCaller',\n",
       " 'Dannydamico67',\n",
       " 'Danshaw2012',\n",
       " 'Danusia234',\n",
       " 'Danyella',\n",
       " 'Dareing',\n",
       " 'Dasboot96',\n",
       " 'Dattan78',\n",
       " 'Dave89',\n",
       " 'Davebratva7th',\n",
       " 'Daveyd147',\n",
       " 'Daviss',\n",
       " 'Davos2018',\n",
       " 'Dawnie',\n",
       " 'Daytrader',\n",
       " 'Dblsolo700',\n",
       " 'Dday70',\n",
       " 'Ddfouche1',\n",
       " 'DePrimo',\n",
       " 'Dee16',\n",
       " 'Deelaney31',\n",
       " 'Dems4trump',\n",
       " 'Derekj3031',\n",
       " 'Desmoines',\n",
       " 'Detroits',\n",
       " 'Dette12',\n",
       " 'Dgibson120',\n",
       " 'Dhfr64',\n",
       " 'Di2577',\n",
       " 'DillonCJTF',\n",
       " 'Dj84',\n",
       " 'Djcoles1',\n",
       " 'Djkelly78',\n",
       " 'Djp3tiesmallzz',\n",
       " 'Djt',\n",
       " 'Dkc628',\n",
       " 'Dnewman83',\n",
       " 'Dogman',\n",
       " 'Dogs4',\n",
       " 'Doll82',\n",
       " 'Dolla1125',\n",
       " 'Dommarino21',\n",
       " 'DonaldTrump',\n",
       " 'Dongibson12',\n",
       " 'Donhall5',\n",
       " 'Donmarie16',\n",
       " 'Doody3',\n",
       " 'Dorals',\n",
       " 'Dornsife',\n",
       " 'Dorya',\n",
       " 'Dougtammy14',\n",
       " 'Dow23k',\n",
       " 'Dow24k',\n",
       " 'Down📉31',\n",
       " 'Doyle5smom',\n",
       " 'Dpatten32',\n",
       " 'Dpmurphy93',\n",
       " 'Drake4444444',\n",
       " 'Drako2',\n",
       " 'Dreadphil1',\n",
       " 'Drehm010',\n",
       " 'Drew4',\n",
       " 'DrudgeTime',\n",
       " 'Drudgereport',\n",
       " 'Drzuhdijasser',\n",
       " 'Dubaiexpo2020',\n",
       " 'Dumbass',\n",
       " 'Dummythanks',\n",
       " 'Dursts',\n",
       " 'Duthie1',\n",
       " 'Dwatts3',\n",
       " 'Dxer',\n",
       " 'Dyl',\n",
       " 'Dylawn420',\n",
       " 'Dynomike3',\n",
       " 'Dzero',\n",
       " 'EO1',\n",
       " 'EO2',\n",
       " 'EWErickson',\n",
       " 'EXPERIENCEVideo',\n",
       " 'Ebolas',\n",
       " 'Ebolathen',\n",
       " 'Eboy12318',\n",
       " 'Ecat',\n",
       " 'Echen',\n",
       " 'Edddie24',\n",
       " 'Eddiecash23',\n",
       " 'Eecatt21',\n",
       " 'Egypts',\n",
       " 'Ehlen',\n",
       " 'Eliots',\n",
       " 'Elkay',\n",
       " 'Ellis38671w',\n",
       " 'Emeka89',\n",
       " 'Emmyb96',\n",
       " 'Emom23',\n",
       " 'Enough’',\n",
       " 'Enq',\n",
       " 'Entrepeneurs',\n",
       " 'Eow',\n",
       " 'Er57',\n",
       " 'Erickingnbc5',\n",
       " 'Ericlahti929',\n",
       " 'Erics',\n",
       " 'Erizzle05',\n",
       " 'Euan888',\n",
       " 'Evanchos',\n",
       " 'Everybodys',\n",
       " 'Everyones',\n",
       " 'Ewilli1204',\n",
       " 'Ewim55',\n",
       " 'Ex369ppd',\n",
       " 'Exider2010',\n",
       " 'Expo2020',\n",
       " 'Eyesa',\n",
       " 'FFCs',\n",
       " 'FOX2',\n",
       " 'FailingNewYorkTimes',\n",
       " 'FakeNews',\n",
       " 'Fan24',\n",
       " 'Fan3000',\n",
       " 'Ferr',\n",
       " 'Ff3476',\n",
       " 'Fheile',\n",
       " 'FighterJet',\n",
       " 'Fishbowldc',\n",
       " 'Flashbk',\n",
       " 'Fleur498',\n",
       " 'Flyar09',\n",
       " 'Fnh',\n",
       " 'Foerderer',\n",
       " 'Foldo',\n",
       " 'Fox02',\n",
       " 'Fox5newsdc',\n",
       " 'FoxBusiness',\n",
       " 'FoxNewsInsider',\n",
       " 'Foxsports',\n",
       " 'Foyt4',\n",
       " 'FrankG',\n",
       " 'Frankenstien',\n",
       " 'Frisina',\n",
       " 'Fud31',\n",
       " 'FundAnything',\n",
       " 'G1959',\n",
       " 'GOPLegacy',\n",
       " 'GROWNG',\n",
       " 'GROWTH📈that',\n",
       " 'Ga06',\n",
       " 'Gadhafis',\n",
       " 'Gallenofmilk13',\n",
       " 'Garnetts',\n",
       " 'Gary4205',\n",
       " 'Garys',\n",
       " 'Gasparinos',\n",
       " 'Gauravk725',\n",
       " 'Gcj',\n",
       " 'Gd61',\n",
       " 'Gem3wood',\n",
       " 'Genall',\n",
       " 'Geomac24',\n",
       " 'GeorgesNYC',\n",
       " 'Geronimos',\n",
       " 'Ges2017',\n",
       " 'Gfg',\n",
       " 'Ghn',\n",
       " 'Giaccio',\n",
       " 'Giaritelli',\n",
       " 'Gil1fresnoca',\n",
       " 'Ginac',\n",
       " 'Gingram66',\n",
       " 'Ginny14974',\n",
       " 'Giovannis',\n",
       " 'Girl75',\n",
       " 'Glennaon10',\n",
       " 'Gllleymon900000',\n",
       " 'GoAngelo',\n",
       " 'GoldmanVice',\n",
       " 'GolfSwag',\n",
       " 'Golfmagic',\n",
       " 'Goodnews',\n",
       " 'Goonerdad52',\n",
       " 'Gop2014ftw',\n",
       " 'Gp237',\n",
       " 'Gphoto',\n",
       " 'Gramma421',\n",
       " 'Grandstanders',\n",
       " 'Gray0622',\n",
       " 'Greeces',\n",
       " 'Gretas',\n",
       " 'Griffith1',\n",
       " 'Grimey',\n",
       " 'Grn777',\n",
       " 'Growthwhich',\n",
       " 'Gruters',\n",
       " 'Guts666',\n",
       " 'HCare',\n",
       " 'HEREOS',\n",
       " 'HER🇺🇸Good',\n",
       " 'HOAXSTERS',\n",
       " 'Habberman',\n",
       " 'Hagels',\n",
       " 'Hallmar',\n",
       " 'Hamilton69',\n",
       " 'Hammerle',\n",
       " 'Hantas88',\n",
       " 'Hart4',\n",
       " 'Harveymy',\n",
       " 'Hawaiis',\n",
       " 'Hayjayp26',\n",
       " 'Hcare',\n",
       " 'HealthcareTax',\n",
       " 'Heilemanns',\n",
       " 'Hellsbells116',\n",
       " 'Hensch',\n",
       " 'Hhsc',\n",
       " 'HillaryClinton',\n",
       " 'Hispanically',\n",
       " 'Hldrpiper73',\n",
       " 'Hmac',\n",
       " 'Hodnett',\n",
       " 'Hoft',\n",
       " 'Hogan4',\n",
       " 'Hoganseaisle129',\n",
       " 'HollyRod',\n",
       " 'Honolulus',\n",
       " 'Hpca',\n",
       " 'Hr27',\n",
       " 'Hr873',\n",
       " 'Hrtbrknomore74',\n",
       " 'Huckebee',\n",
       " 'Huffingtonpost',\n",
       " 'Humas',\n",
       " 'Huntleys',\n",
       " 'Hvidston',\n",
       " 'Hvr',\n",
       " 'ICYMI',\n",
       " 'INCR',\n",
       " 'INDV',\n",
       " 'ISIL',\n",
       " 'Ianb',\n",
       " 'Icer100',\n",
       " 'Igo4par',\n",
       " 'Ihave',\n",
       " 'Impd',\n",
       " 'Imper',\n",
       " 'In5',\n",
       " 'Indiv',\n",
       " 'Investopedia',\n",
       " 'Iowas',\n",
       " 'Iraelis',\n",
       " 'IraniansSyrians',\n",
       " 'Ire1',\n",
       " 'Irene1969',\n",
       " 'Is3',\n",
       " 'IstanbulTurkey',\n",
       " 'Ivankas',\n",
       " 'J0y',\n",
       " 'J4k9flatley',\n",
       " 'JCOPE',\n",
       " 'JCena',\n",
       " 'JVF',\n",
       " 'Jackies',\n",
       " 'Jagadis63',\n",
       " 'Jaine004',\n",
       " 'Jake0',\n",
       " 'Jamess',\n",
       " 'Janeiros',\n",
       " 'Janise',\n",
       " 'Japotter81',\n",
       " 'Jarboni72',\n",
       " 'Jared808',\n",
       " 'Jareds',\n",
       " 'Javi8104',\n",
       " 'Jaw44444',\n",
       " 'JayZ',\n",
       " 'Jayknowles1978',\n",
       " 'Jaymz',\n",
       " 'Jayni',\n",
       " 'Jaynie',\n",
       " 'Jb19',\n",
       " 'Jbailey1987',\n",
       " 'Jbluvx2',\n",
       " 'Jboyle007',\n",
       " 'Jcarp61',\n",
       " 'Jcfields13',\n",
       " 'Jchorvath11',\n",
       " 'Jcriv305',\n",
       " 'Jdaddy98',\n",
       " 'Jdiamond1',\n",
       " 'Jebs',\n",
       " 'JeffH',\n",
       " 'Jeffjlpa1',\n",
       " 'Jemele',\n",
       " 'Jenke2501',\n",
       " 'Jepsens',\n",
       " 'Jerad',\n",
       " 'Jessilin02',\n",
       " 'Jeters',\n",
       " 'Jgy777ph',\n",
       " 'Jhayman2',\n",
       " 'Jibz',\n",
       " 'Jimbrownnfl32',\n",
       " 'Jkells2005',\n",
       " 'Jlin7',\n",
       " 'Jma187',\n",
       " 'Jmackd10',\n",
       " 'Jmonaski1',\n",
       " 'Jmull71',\n",
       " 'Jnorr11',\n",
       " 'Jns',\n",
       " 'Joannah',\n",
       " 'Joe49',\n",
       " 'Joebaker1234',\n",
       " 'Joequinn6',\n",
       " 'JohnDaly',\n",
       " 'Johnson68',\n",
       " 'Jonareeves6127',\n",
       " 'Jonzy1523',\n",
       " 'Josh8J4',\n",
       " 'Journos',\n",
       " 'Jr37',\n",
       " 'Jreid1973',\n",
       " 'Jrg710',\n",
       " 'Jrkirk22',\n",
       " 'Jschwab2',\n",
       " 'Jshue20',\n",
       " 'Jsunday72',\n",
       " 'Jtbone97',\n",
       " 'Jtholder2010',\n",
       " 'Jtracy08',\n",
       " 'Jumpman23',\n",
       " 'Jurciuoli19',\n",
       " 'Jusr',\n",
       " 'JusticeDepartment',\n",
       " 'Jwizzle03',\n",
       " 'K104',\n",
       " 'K1047',\n",
       " 'K720',\n",
       " 'K9koda',\n",
       " 'K9s',\n",
       " 'KLZA',\n",
       " 'Kabarzombie69',\n",
       " 'Karidaway2388',\n",
       " 'Karllewis01',\n",
       " 'Kasie',\n",
       " 'Kat3500',\n",
       " 'Katek104',\n",
       " 'KatherineWebb',\n",
       " 'KatherineWebbs',\n",
       " 'Kathomas212',\n",
       " 'Katt8808',\n",
       " 'Katwatson59',\n",
       " 'Kaufstache',\n",
       " 'Kaybee0h',\n",
       " 'KayyArrDee',\n",
       " 'Kbetter52',\n",
       " 'Kc5yvv',\n",
       " 'Kcrg',\n",
       " 'Keign',\n",
       " 'Kelbug99',\n",
       " 'Kelly1',\n",
       " 'KenStarr',\n",
       " 'Kenkelly08',\n",
       " 'Kentoidi8',\n",
       " 'Kenzig',\n",
       " 'Kesslers',\n",
       " 'Keysyone',\n",
       " 'Kharmdd59',\n",
       " 'Khloekardashian',\n",
       " 'Kimfeather177',\n",
       " 'Kiss925',\n",
       " 'Kjclt1',\n",
       " 'Kjl',\n",
       " 'Klr',\n",
       " 'Kmt9999',\n",
       " 'Kodi1',\n",
       " 'Konichiwa',\n",
       " 'Konstant',\n",
       " 'Korea🇺🇸🇰🇷',\n",
       " 'Koster4',\n",
       " 'Kredo0',\n",
       " 'Krn',\n",
       " 'Kryssitoocool18',\n",
       " 'Krzanich',\n",
       " 'Kscj1360',\n",
       " 'Kstan500',\n",
       " 'Kuhn5',\n",
       " 'Kumite67',\n",
       " 'Kurse',\n",
       " 'Kwr',\n",
       " 'Kwwl',\n",
       " 'Kylemccall4',\n",
       " 'L100ran',\n",
       " 'L30809',\n",
       " 'LaVars',\n",
       " 'Lanah11',\n",
       " 'Lanele123',\n",
       " 'Lanes08',\n",
       " 'Lara1961',\n",
       " 'Lassner',\n",
       " 'Law360',\n",
       " 'Lay2013',\n",
       " 'Lbenson3',\n",
       " 'Lcamp34',\n",
       " 'Lcf42',\n",
       " 'Ld06981p',\n",
       " 'Leachs',\n",
       " 'Leaguewhose',\n",
       " 'Lebrons',\n",
       " 'Leesport',\n",
       " 'Legit73791728',\n",
       " 'Leigh26',\n",
       " 'Leighann',\n",
       " 'Leightweight',\n",
       " 'Leisures',\n",
       " 'Lettermans',\n",
       " 'Lexiperez96',\n",
       " 'Lexxay',\n",
       " 'LiAngelo',\n",
       " 'Liabil',\n",
       " 'Libertys',\n",
       " 'Libyas',\n",
       " 'Limka11',\n",
       " 'Lin0729',\n",
       " 'Linkins',\n",
       " 'Lism',\n",
       " 'Littl',\n",
       " 'Lkr',\n",
       " 'Lmetzger',\n",
       " 'Lmward7',\n",
       " 'Loads29',\n",
       " 'Loispope1',\n",
       " 'Lood800',\n",
       " ...]"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missing_words.sort()\n",
    "missing_words#[ :20 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(24498, 300)\n",
      "(541841, 25)\n",
      "(541841,)\n"
     ]
    }
   ],
   "source": [
    "print( embedding_matrix.shape )\n",
    "print( X.shape )\n",
    "print( y.shape )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write missing words to file\n",
    "with open( \"output/missing-words-tweets.txt\", \"w\" ) as out_file:\n",
    "    \n",
    "    for word in missing_words:\n",
    "\n",
    "        out_file.write( \"%s\\n\" % word )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300\n",
      "0.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2059"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# confirm visually that \n",
    "print( len( embedding_matrix[ 0 ] ) )\n",
    "print( sum( embedding_matrix[ 0 ] ) )\n",
    "empty_coefficients_count = 0\n",
    "\n",
    "for i in range( len( embedding_matrix ) ):\n",
    "    if sum( embedding_matrix[ i ] ) == 0:\n",
    "        empty_coefficients_count += 1\n",
    "        \n",
    "empty_coefficients_count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.3\n",
      "1.4.1\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "print( keras.__version__ )\n",
    "\n",
    "import tensorflow as tf\n",
    "print( tf.__version__ )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 25, 300)           7349400   \n",
      "_________________________________________________________________\n",
      "bidirectional_3 (Bidirection (None, 25, 100)           140400    \n",
      "_________________________________________________________________\n",
      "bidirectional_4 (Bidirection (None, 100)               60400     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 50)                5050      \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 24498)             1249398   \n",
      "=================================================================\n",
      "Total params: 8,804,648\n",
      "Trainable params: 8,804,648\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# define model\n",
    "model = Sequential()\n",
    "\n",
    "# now using a pre-trained, non-trainable embedding from glove's wiki analysis\n",
    "model.add( Embedding( vocab_size, embeddings_dimension, weights=[embedding_matrix], input_length=seq_length, trainable=True ) )\n",
    "model.add( Bidirectional( LSTM( seq_length * 2, return_sequences=True ) ) )\n",
    "#model.add( Dropout( 0.90 ) )\n",
    "model.add( Bidirectional( LSTM( seq_length * 2 ) ) )\n",
    "#model.add( Dropout( 0.90 ) )\n",
    "model.add( Dense( seq_length * 2, activation='relu' ) )\n",
    "\n",
    "# fixed TypeError below, downgraded keras from 2.1.5 to 2.1.3: https://github.com/keras-team/keras/issues/9621\n",
    "# TypeError: softmax() got an unexpected keyword argument 'axis'\n",
    "model.add( Dense( vocab_size, activation='softmax' ) )\n",
    "\n",
    "print( model.summary() )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fit the Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Model, Tokenizer, Training Data & Embeddings?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load model, tokenizer & embeddings? [y/n]y\n",
      "Loading model models/trump-tweets-w-links-n-ats-take-III.h5\n",
      "Loading tokenizer tokenizers/trump-tweets-w-links-n-ats-take-III.dump\n",
      "Loading embeddings embeddings/trump-tweats-w-links-n-ats-take-III.glove\n",
      "Loading training data X's data/training-X-trump-tweets-w-links-n-ats-take-III.dump\n",
      "Loading training data y's data/training-y-trump-tweets-w-links-n-ats-take-III.dump\n"
     ]
    }
   ],
   "source": [
    "load = input( \"Load model, tokenizer & embeddings? [y/n]\" )\n",
    "\n",
    "if load == \"y\":\n",
    "    \n",
    "    #model_name = \"models/trump-tweets-w-links-n-ats-take-III.h5\"\n",
    "    print( \"Loading model %s\" % model_path )\n",
    "    model = load_model( model_path )\n",
    "    \n",
    "    print( \"Loading tokenizer %s\" % tokenizer_path )\n",
    "    tokenizer = pickle.load( open( tokenizer_path, \"rb\" ) )\n",
    "    \n",
    "    print( \"Loading embeddings %s\" % embedding_path )\n",
    "    embedding_matrix = pickle.load( open( embedding_path, \"rb\" ) )\n",
    "    \n",
    "    print( \"Loading training data X's %s\" % training_X_path )\n",
    "    X = pickle.load( open( training_X_path, \"rb\" ) )\n",
    "    \n",
    "    print( \"Loading training data y's %s\" % training_y_path )\n",
    "    y = pickle.load( open( training_y_path, \"rb\" ) )\n",
    "    \n",
    "    seq_length = len( X[ 0 ] )\n",
    "    \n",
    "else:\n",
    "    \n",
    "    print( \"NOT loading model, tokenizer, training data & embeddings, using defaults\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 1024\n",
    "# can't remember where I read that batch sizes larger than 512 cause erratic convergence patterns.\n",
    "# TODO: find that article!\n",
    "#batch_size = 512\n",
    "\n",
    "start_time = get_time()\n",
    "\n",
    "# Per comment here: https://stackoverflow.com/questions/46293734/memoryerror-in-keras-utils-np-utils-to-categoricalhttps://stackoverflow.com/questions/46293734/memoryerror-in-keras-utils-np-utils-to-categorical\n",
    "model.compile( loss='sparse_categorical_crossentropy', optimizer='adam', metrics=[ 'accuracy' ] )\n",
    "# model.compile( loss='categorical_crossentropy', optimizer='adam', metrics=[ 'accuracy' ] )\n",
    "\n",
    "model.fit( X, y, batch_size=batch_size, epochs=10 )\n",
    "end_time = get_time()\n",
    "print_time( start_time, end_time, interval=\"hours\" )\n",
    "\n",
    "# 2018.06.06 19:52\n",
    "# 100 epochs\n",
    "# Time to process: [1.6088602582613627] hours\n",
    "# 2018.06.06 21:29\n",
    "\n",
    "# 2018.06.06 22:28\n",
    "# 50 epochs\n",
    "# Time to process: [0.8057563017474281] hours @ 80% accuracy\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the whole model to file\n",
    "model.save( model_path )\n",
    "\n",
    "# save the tokenizer\n",
    "dump( tokenizer, open( tokenizer_path, 'wb' ) )\n",
    "\n",
    "# save embedding_matrix based on wiki embeddings, complete w/ missing coefficients array dummies\n",
    "dump( embedding_matrix, open( embedding_path, 'wb' ) )\n",
    "\n",
    "# write keys to embedding_matrix\n",
    "out_file = open( embedding_keys_path, 'w', encoding='utf-8' ) \n",
    "for word in embedding_matrix_words:\n",
    "    out_file.write( \"%s\\n\" % word )\n",
    "out_file.close()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use The Model to Generate Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seq_length = len( lines[ 0 ].split() ) - 1\n",
    "seq_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'”'"
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "punctuation_dict.get( \"smartquoteclose\", \"bar\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # From: https://medium.com/@david.campion/text-generation-using-bidirectional-lstm-and-doc2vec-models-1-3-8979eb65cb3a\n",
    "# if temperature = 1.0:\n",
    "#     the probability for a word to be drawn is similar to the probability for the word to be the next one in the \n",
    "#     sequence (the output of the word prediction model), compared to other words in the dictionary,\n",
    "# if temperature is big (much bigger than 1):\n",
    "#     the range of probabilities is shorten: the probabilities for all words to be the next one will increase. More \n",
    "#         variety of words will be picked-up from the vocabulary, because more words will have high probabilities.\n",
    "# if temperature is small (close to 0):\n",
    "#     small probabilities will be avoided (they will be set to a value closed to 0). Less words will be picked-up \n",
    "#     from the vocabulary.\n",
    "\n",
    "def sample_yhats( preds, temperature=1.0 ):\n",
    "    \n",
    "    # helper function to sample an index from a probability array\n",
    "    preds = np.asarray( preds ).astype( 'float64' )\n",
    "    preds = np.log( preds ) / temperature\n",
    "    exp_preds = np.exp( preds )\n",
    "    preds = exp_preds / np.sum( exp_preds )\n",
    "    probas = np.random.multinomial( 1, preds, 1 )\n",
    "    return np.argmax( probas )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_seq( model, tokenizer, seq_length, seed_text, n_words, random=False, temperature=1.0 ):\n",
    "    \n",
    "    result = list()\n",
    "    result_literal = list()\n",
    "    in_text = seed_text\n",
    "    yhat = [ 0.1 ]\n",
    "    \n",
    "    # generate a fixed number of words\n",
    "    for idx in range( n_words ):\n",
    "        \n",
    "        # encode the text as integer\n",
    "        encoded = tokenizer.texts_to_sequences( [ in_text ] )[ 0 ] \n",
    "        \n",
    "        # truncate sequences to a fixed length\n",
    "        encoded = pad_sequences( [ encoded ], maxlen=seq_length, truncating='pre' ) \n",
    "        \n",
    "        # predict probabilities for each word\n",
    "        #yhat = model.predict_classes( encoded, verbose=0 )\n",
    "        # this returns list of predictions\n",
    "        #yhats = model.predict( encoded, verbose=0 )[ 0 ]\n",
    "        \n",
    "        # map predicted word index to word\n",
    "        # print( \"yhat\", yhat, words_by_id[ yhat[ 0 ] ] )\n",
    "        #print( \"len( yhats )\", len( yhats ) )\n",
    "        #print( \"type( yhats )\", type( yhats ) )\n",
    "        #print( \"argmax( yhats )\", np.argmax( yhats ) )\n",
    "        #print( \"yhats\", yhats )\n",
    "        \n",
    "        if random == True:\n",
    "            # predict returns multiple candidates\n",
    "            yhats = model.predict( encoded, verbose=0 )[ 0 ]\n",
    "            out_word_id = sample_yhats( yhats, temperature )\n",
    "            out_word = words_by_id[ out_word_id ]\n",
    "        else:\n",
    "            # predict_classes returns one highest probability\n",
    "            yhat = model.predict_classes( encoded, verbose=0 )[ 0 ]\n",
    "            out_word = words_by_id[ yhat ]\n",
    "        \n",
    "        \n",
    "        # out_word = words_by_id[ yhat[ 0 ] ]\n",
    "                \n",
    "        # append to input\n",
    "        in_text += ' ' + out_word\n",
    "        \n",
    "        #result.append( out_word )\n",
    "        # substitute punctuation tags for actual punctuation\n",
    "        result.append( punctuation_dict.get( out_word, out_word ) )\n",
    "        \n",
    "        # only break after creating some output!  sometimes it wants to bail because it encounders \"closetweetclose\"\n",
    "        # on the first try!\n",
    "        if idx > 0 and out_word == \"closetweetclose\":\n",
    "            #print( \"Tweet end detected\" )\n",
    "            break\n",
    "            \n",
    "    return ' '.join( result )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type( words_by_id[ 1 ] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reformat_punctuation( doc ):\n",
    "    \n",
    "    doc = doc.replace( ' . ', '. ' )\n",
    "    doc = doc.replace( ' ! ', '! ' )\n",
    "    doc = doc.replace( ' ? ', '? ' )\n",
    "    doc = doc.replace( ' , ', ', ' )\n",
    "    doc = doc.replace( ' : ', ': ' )\n",
    "    doc = doc.replace( ' ; ', '; ' )\n",
    "    \n",
    "    doc = doc.replace( '“ ', '“' )\n",
    "    doc = doc.replace( ' ”', '”' )\n",
    "    doc = doc.replace( \"attweetat\", '@' )\n",
    "    #doc = doc.replace( \"hashtweethash\", '#' )\n",
    "    doc = doc.replace( \" amp; \", '&' )\n",
    "    \n",
    "    return doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "541841\n",
      "22321\n"
     ]
    }
   ],
   "source": [
    "print( len( lines ) )\n",
    "# iterate and grab sequences that begin w/ \"opentweetopen\"\n",
    "full_tweets = []\n",
    "for line in lines:\n",
    "    \n",
    "    if line.startswith( \"opentweetopen\" ):\n",
    "        full_tweets.append( line )\n",
    "print( len( full_tweets ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "opentweetopen So many people who know nothing about me are commenting all over T V endperiod and the media as though they have great D J \n",
      "\n",
      "... . P . is unable to sell us that once again get aid is breaking its narrative to them . Must read [end]\n",
      "\n",
      "\n",
      "\n",
      "... . P. is unable to sell us that once again get aid is breaking its narrative to them. Must read [end]\n"
     ]
    }
   ],
   "source": [
    "# select a seed text\n",
    "#seed_text = lines[ randint( 0, len( lines ) ) ]\n",
    "seed_text = full_tweets[ randint( 0, len( full_tweets ) ) ]\n",
    "\n",
    "print( seed_text, \"\\n\" )\n",
    "\n",
    "# # substitute the seed words\n",
    "# raw_text = seed_text.split( \" \" )\n",
    "\n",
    "# clean_text = [ punctuation_dict.get( word, word ) for word in raw_text ]\n",
    "# clean_text = ' '.join( clean_text )\n",
    "\n",
    "# print( reformat_punctuation( clean_text ) + '... \\n' )\n",
    "# #print( len( seed_text.split( \" \" ) ) )\n",
    "\n",
    "# generate new text\n",
    "generated = generate_seq( model, tokenizer, seq_length, seed_text, 50, random=False, temperature=1.0 )\n",
    "\n",
    "print( \"... \" + generated )\n",
    "print()\n",
    "print( \"\\n\\n... \" + reformat_punctuation( generated ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "So many people who know nothing about me are commenting all over T V endperiod and the media as though they have great D J\n",
      "... . P. is unable to sell us that once again get aid is breaking its narrative to them. Must read [end]\n"
     ]
    }
   ],
   "source": [
    "my_input = input()\n",
    "\"opentweetopen \" + my_input + \" closetweetclose\"\n",
    "generated = generate_seq( model, tokenizer, seq_length, my_input, 50, random=False, temperature=0.25 )\n",
    "print( \"... \" + reformat_punctuation( generated ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating text with the following seed: \"a a a a a a a a a a a a a a a Nobody has better respect for intelligence than Donald Trump .\"\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#initiate sentences\n",
    "seed_sentences = \"Nobody has better respect for intelligence than Donald Trump .\"\n",
    "generated = ''\n",
    "sentence = []\n",
    "\n",
    "for i in range( seq_length ):\n",
    "    sentence.append( \"a\" )\n",
    "\n",
    "seed = seed_sentences.split()\n",
    "\n",
    "for i in range( len( seed ) ):\n",
    "    sentence[ seq_length - i - 1 ]= seed[ len( seed ) - i - 1 ]\n",
    "\n",
    "generated += ' '.join(sentence)\n",
    "print('Generating text with the following seed: \"' + ' '.join(sentence) + '\"')\n",
    "\n",
    "print ()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_number = 100\n",
    "#generate the text\n",
    "for i in range(words_number):\n",
    "    #create the vector\n",
    "    x = np.zeros((1, seq_length, vocab_size))\n",
    "    for t, word in enumerate(sentence):\n",
    "        x[0, t, vocab[word]] = 1.\n",
    "    #print(x.shape)\n",
    "\n",
    "    #calculate next word\n",
    "    preds = model.predict(x, verbose=0)[0]\n",
    "    next_index = sample(preds, 0.34)\n",
    "    next_word = vocabulary_inv[next_index]\n",
    "\n",
    "    #add the next word to the text\n",
    "    generated += \" \" + next_word\n",
    "    # shift the sentence by one, and and the next word at its end\n",
    "    sentence = sentence[1:] + [next_word]\n",
    "\n",
    "print(generated)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}