Neuroscope2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neuroscope\n",
    "Generating horoscope-like tweets with different implementations of RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import twitter as twt\n",
    "import seaborn as sns\n",
    "import codecs as cdc\n",
    "import pickle\n",
    "import nltk\n",
    "import pymorphy2\n",
    "import itertools\n",
    "import string\n",
    "import os\n",
    "import re\n",
    "import random as rnd\n",
    "from collections import Counter, deque\n",
    "from sklearn.manifold import TSNE, MDS\n",
    "\n",
    "sns.set_style('whitegrid')\n",
    "sns.despine()\n",
    "\n",
    "%pylab inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Acquire corpus\n",
    "---\n",
    "Initialize Twitter client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "API_KEY = os.environ.get('TWITTER_API_KEY', '')\n",
    "API_SECRET = os.environ.get('TWITTER_API_SECRET', '')\n",
    "BEARER_TOKEN = twt.oauth2_dance(API_KEY, API_SECRET)\n",
    "t = twt.Twitter(auth=twt.OAuth2(bearer_token=BEARER_TOKEN))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Download data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "accounts = np.array(['gor_aries', 'gor_taurus', 'gor_gemini', 'gor_cancer', 'gor_leo', 'gor_virgo', \\\n",
    "                     'gor_libra', 'gor_scorpio', 'gor_sagittarius', 'gor_aquarius', \\\n",
    "                     'gor_pisces', 'aries_astro7', 'taurus_astro7', 'gemini_astro7', 'cancer_astro7',\n",
    "                     'leo_astro7', 'virgo_astro7', 'libra_astro7', 'scorpio_astro7', 'kozerog_astro7',\n",
    "                     'aquaris_astro7', 'pisces_astro7'])\n",
    "months = np.array([u'января', u'февраля', u'марта', u'апреля', u'мая', u'июня', u'июля', u'августа', u'сентрября', \\\n",
    "                   u'октября', u'ноября', u'декабря'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def download_data(accounts, filename='raw'):\n",
    "    print 'Downloading data'\n",
    "    raw = []\n",
    "\n",
    "    for account in accounts:\n",
    "        print '-> Processing account %s' % account\n",
    "        max_id = 0\n",
    "        tweets = []\n",
    "\n",
    "        if not os.path.isfile('./raw_%s' % account):\n",
    "            print '--> Receiving tweets'\n",
    "            part = t.statuses.user_timeline(screen_name=account, include_rts='false', exclude_replies='true')\n",
    "            for tweet in part:\n",
    "                tweets.append(tweet)\n",
    "            old_max_id = max_id\n",
    "            max_id = tweets[len(tweets)-1]['id']\n",
    "\n",
    "            while old_max_id != max_id:\n",
    "                part = t.statuses.user_timeline(screen_name=account, \n",
    "                                                include_rts='false', exclude_replies='true', max_id=max_id, timeout=1)\n",
    "                for tweet in part:\n",
    "                    tweets.append(tweet)\n",
    "                old_max_id = max_id\n",
    "                max_id = tweets[len(tweets)-1]['id']\n",
    "            print '--> Received %d tweets' % len(tweets)\n",
    "\n",
    "            with open('raw_%s' % account, 'w') as fout:\n",
    "                pickle.dump(tweets, fout)\n",
    "            print '--> Dumped tweets'\n",
    "        else:\n",
    "            with open('raw_%s' % account, 'r') as fin:\n",
    "                tweets = pickle.load(fin)\n",
    "            print '--> Loaded previously dumped %d tweets' % len(tweets)\n",
    "        \n",
    "        tweets = [tweet['text'] for tweet in tweets]\n",
    "        for tweet in tweets:\n",
    "            raw.append(tweet)\n",
    "        del tweets\n",
    "\n",
    "    with open(filename, 'w') as fout:\n",
    "        pickle.dump(raw, fout)\n",
    "    del raw\n",
    "    \n",
    "def pad_data(data, pad_to=None, pad_with='<p>'):\n",
    "    if pad_to is None:\n",
    "        pad_to = max([len(item) for item in data])\n",
    "    \n",
    "    padded = []\n",
    "    for item in data:\n",
    "        amount = pad_to - len(item)\n",
    "        if amount < 0:\n",
    "            padded.append(item[:amount])\n",
    "        else:\n",
    "            padded.append(item + [pad_with] * amount)\n",
    "            \n",
    "    return padded\n",
    "\n",
    "def array_equals(a, b):\n",
    "    if len(a) != len(b):\n",
    "        return False\n",
    "    else:\n",
    "        length = len(a)\n",
    "        for i in range(length):\n",
    "            if a[i] != b[i]:\n",
    "                return False\n",
    "            \n",
    "    return True\n",
    "\n",
    "def clean_up(tokenized):\n",
    "    print '-> Cleaning up data'\n",
    "    \n",
    "    cleaned = []\n",
    "    \n",
    "    cleaned.append(tokenized[0])\n",
    "    length = 1\n",
    "    \n",
    "    for i in range(1, len(tokenized)):\n",
    "        j = 0\n",
    "        \n",
    "        while j < length:\n",
    "            if array_equals(tokenized[i], cleaned[j]):\n",
    "                break\n",
    "            j += 1\n",
    "        \n",
    "        if j == length:\n",
    "            cleaned.append(tokenized[i])\n",
    "            length += 1\n",
    "                \n",
    "    return cleaned\n",
    "\n",
    "def load_data(filename='./raw', load=False):\n",
    "    print 'Reading data'\n",
    "    \n",
    "    tokenized = []\n",
    "    words = []\n",
    "    \n",
    "    if not load:\n",
    "        print '-> With preprocesing'\n",
    "        with open(filename, 'r') as fin:\n",
    "            tweets = pickle.load(fin)\n",
    "\n",
    "        morph = pymorphy2.MorphAnalyzer()\n",
    "\n",
    "        for tweet in tweets:\n",
    "            result = [u'<$>']\n",
    "            tweet = nltk.word_tokenize(tweet.lower())\n",
    "            tweet = [w for w in tweet if w not in string.punctuation]\n",
    "            tweet = [w for w in tweet if re.compile(u'[а-я]*').match(w).end() == len(w)]\n",
    "            tweet = [w for w in tweet if w not in months]\n",
    "            tweet = [morph.parse(w)[0].normal_form for w in tweet]\n",
    "            tweet.append(u'<#>')\n",
    "            result.extend(tweet)\n",
    "            tokenized.append(result)\n",
    "            \n",
    "        print '--> Before cleanup %d tweets' % len(tokenized)\n",
    "        tokenized = clean_up(tokenized)\n",
    "        tokenized = pad_data(tokenized)\n",
    "        print '--> After cleanup %d tweets' % len(tokenized)\n",
    "        words = np.concatenate(tokenized)\n",
    "\n",
    "        with open('tokenized', 'w') as fout:\n",
    "            pickle.dump(tokenized, fout)\n",
    "            \n",
    "        with open('words', 'w') as fout:\n",
    "            pickle.dump(words, fout)\n",
    "    else:\n",
    "        print '-> Previously processed'\n",
    "        with open('tokenized', 'r') as fin:\n",
    "            tokenized = pickle.load(fin)\n",
    "            \n",
    "        with open('words', 'r') as fin:\n",
    "            words = pickle.load(fin)\n",
    "            \n",
    "        print '--> Found %d tweets, %d tokens' % (len(tokenized), len(words))\n",
    "    \n",
    "    return tokenized, words\n",
    "\n",
    "def build_dataset(dataset, words, vocabulary_size=None):\n",
    "    word_to_index = {}\n",
    "    index_to_word = {}\n",
    "    word_count = []\n",
    "    tokenized = []\n",
    "    data = []\n",
    "    \n",
    "    word_count.extend(Counter(words).most_common(vocabulary_size))\n",
    "    print 'Most common word is %s with %d times' % (word_count[0][0], word_count[0][1])\n",
    "    print 'Least common word is %s with %d times' % (word_count[len(word_count)-1][0],\n",
    "                                                     word_count[len(word_count)-1][1])\n",
    "    \n",
    "    for word, _ in word_count:\n",
    "        word_to_index[word] = len(word_to_index)\n",
    "    \n",
    "    for item in dataset:\n",
    "        tmp = []\n",
    "        for word in item:\n",
    "            index = word_to_index[word]\n",
    "            tmp.append(index)\n",
    "            data.append(index)\n",
    "        tokenized.append(tmp)\n",
    "    \n",
    "    index_to_word = dict(zip(word_to_index.values(), word_to_index.keys()))\n",
    "    data = [word_to_index[w] for w in words]\n",
    "    \n",
    "    return tokenized, data, word_count, word_to_index, index_to_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading data\n",
      "-> Previously processed\n",
      "--> Found 10641 tweets, 308589 tokens\n"
     ]
    }
   ],
   "source": [
    "# download_data(accounts)\n",
    "tweets, words = load_data(load=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most common word is <p> with 140492 times\n",
      "Least common word is выступить with 1 times\n",
      "Using vocabulary size 7945\n"
     ]
    }
   ],
   "source": [
    "tokenized, data, word_count, word_to_index, index_to_word = build_dataset(tweets, words)\n",
    "vocabulary_size = len(word_count)\n",
    "print 'Using vocabulary size %d' % vocabulary_size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LSTM\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class DataLoader(object):\n",
    "    def __init__(self, tweets, batch_size, horizon):\n",
    "        self._tweets = tweets\n",
    "        self._num_tweets = len(tweets)\n",
    "        self._tweet_len = len(tweets[0])\n",
    "        self._batch_size = batch_size\n",
    "        self._horizon = horizon\n",
    "        self._tweets_idx = self._get_tweets_idx(self._batch_size)\n",
    "        self._cursor = 0\n",
    "        self._last_batch = self._next_batch()\n",
    "        self._cursor = 1\n",
    "        \n",
    "    def _get_tweets_idx(self, count):\n",
    "        idx = np.random.choice(self._num_tweets, count)\n",
    "        return idx\n",
    "  \n",
    "    def _next_batch(self):\n",
    "        \"\"\"Generate a single batch from the current cursor position in the data.\"\"\"\n",
    "        batch = np.zeros(shape=(self._batch_size), dtype=np.int32)\n",
    "        for b in range(self._batch_size):\n",
    "            batch[b] = self._tweets[self._tweets_idx[b]][self._cursor]\n",
    "        return batch\n",
    "  \n",
    "    def next(self):\n",
    "        \"\"\"Generate the next array of batches from the data. The array consists of\n",
    "        the last batch of the previous array, followed by num_unrollings new ones.\n",
    "        \"\"\"\n",
    "        batches = np.zeros(shape=(self._horizon + 1, self._batch_size), dtype=np.int32)\n",
    "        batches[0] = self._last_batch\n",
    "        for step in range(self._horizon):\n",
    "            batches[step + 1] = self._next_batch()\n",
    "            self._cursor += 1\n",
    "            if self._cursor % self._tweet_len == 0:\n",
    "                self._cursor = 0\n",
    "                self._tweets_idx = self._get_tweets_idx(self._batch_size)\n",
    "                break\n",
    "        self._last_batch = batches[-1]\n",
    "        return batches\n",
    "\n",
    "def logprob(predictions, labels):\n",
    "    true = np.zeros_like(predictions, dtype=np.float)\n",
    "    for i in range(len(labels)):\n",
    "        true[i, labels[i]] = 1.0\n",
    "    predictions[predictions < 1e-10] = 1e-10\n",
    "    return np.sum(np.multiply(true, -np.log(predictions))) / true.shape[0]\n",
    "\n",
    "def perplexity(predictions, labels):\n",
    "    return np.exp(logprob(predictions, labels))\n",
    "\n",
    "def sample(distribution):\n",
    "    r = rnd.uniform(0, 1)\n",
    "    s = 0\n",
    "    \n",
    "    for i in range(len(distribution)):\n",
    "        s += distribution[i]\n",
    "        if s >= r:\n",
    "            return i\n",
    "    \n",
    "    return len(distribution) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "valid_pct = 0.1\n",
    "valid_size = int(len(tokenized) * valid_pct)\n",
    "trunc_by = 30\n",
    "batch_size = 128\n",
    "num_nodes = 256\n",
    "embedding_size = 256\n",
    "valid_dataset = tokenized[:valid_size]\n",
    "train_dataset = tokenized[valid_size:]\n",
    "\n",
    "train_batches = DataLoader(train_dataset, batch_size, trunc_by)\n",
    "valid_batches = DataLoader(valid_dataset, 1, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loss function:\n",
    "$$\\mathcal{L} = - \\sum\\limits_{k = 1}^N y_k \\log o_k$$\n",
    "\n",
    "Output at step $k$:\n",
    "$$o_k = softmax(H_k \\ast W_{(sm)}^k + b_{(sm)}^k) = \\frac{\\exp \\{H_k \\ast W_{(sm)}^k + b_{(sm)}^k\\}}{\\sum\\limits_{n=1}^N \\exp \\{H_n \\ast W_{(sm)}^n + b_{(sm)}^n\\}}$$\n",
    "\n",
    "Hidden state at step $k$:\n",
    "$$H_k = output\\_gate(w_k, H_{k-1}) \\cdot \\tanh (state_k \\ast W_{(h)} + b_{(h)})$$\n",
    "\n",
    "Cell state at step $k$:\n",
    "$$state_k = forget\\_gate \\cdot state_{k-1} + input\\_gate \\cdot \\tanh ([w_k, h_{k-1}] \\ast W_{(c)} + b_{(c)})$$\n",
    "\n",
    "Output gate function for step $k$:\n",
    "$$output\\_gate(w_k, H_{k-1}) = \\sigma([w_k, h_{k-1}] W_{(o)} + b_{(o)})$$\n",
    "\n",
    "Forget gate function for step $k$:\n",
    "$$forget\\_gate(w_k, H_{k-1}) = \\sigma([w_k, h_{k-1}] W_{(f)} + b_{(f)})$$\n",
    "\n",
    "Input gate function for step $k$:\n",
    "$$output\\_gate(w_k, H_{k-1}) = \\sigma([w_k, h_{k-1}] W_{(i)} + b_{(i)})$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "graph = tf.Graph()\n",
    "\n",
    "with graph.as_default(), tf.device('/cpu:0'):\n",
    "    # Input data\n",
    "    train_data = [tf.placeholder(tf.int32, shape=[batch_size]) for i in range(trunc_by+1)]\n",
    "    train_inputs = train_data[:-1]\n",
    "    train_labels = train_data[1:]\n",
    "\n",
    "    # Variables\n",
    "    # Embedding\n",
    "    embeddings = tf.Variable(tf.truncated_normal([vocabulary_size, embedding_size], -0.1, 0.1), name='embeddings')\n",
    "    # The big matrix\n",
    "    Tw = tf.Variable(tf.truncated_normal([embedding_size + num_nodes, 4 * num_nodes], -0.1, 0.1))\n",
    "    Tb = tf.Variable(tf.zeros([1, 4 * num_nodes]))\n",
    "    # Connections\n",
    "    saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]),\n",
    "                              name='saved_output')\n",
    "    saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]),\n",
    "                          name='saved_state')\n",
    "    # Softmax\n",
    "    softmax_weights = tf.Variable(tf.truncated_normal([num_nodes, vocabulary_size]),\n",
    "                                  name='softmax_weights')\n",
    "    softmax_biases = tf.Variable(tf.zeros([vocabulary_size]),\n",
    "                                 name='softmax_biases')\n",
    "    \n",
    "    # Cell\n",
    "    def lstm_cell(i, state, cell):\n",
    "        i = tf.concat([i, state], 1)\n",
    "        tmp = tf.matmul(i, Tw) + Tb\n",
    "\n",
    "        inputs_gate = tf.sigmoid(tmp[:, :num_nodes])\n",
    "        forget_gate = tf.sigmoid(tmp[:, num_nodes:2*num_nodes])\n",
    "        output_gate = tf.sigmoid(tmp[:, 2*num_nodes:3*num_nodes])\n",
    "        update_gate = tf.tanh(tmp[:, 3*num_nodes:])\n",
    "\n",
    "        cell = forget_gate * cell + inputs_gate * update_gate\n",
    "        state = output_gate * tf.tanh(cell)\n",
    "        return state, cell\n",
    "\n",
    "    # Model\n",
    "    outputs = []\n",
    "    output = saved_output\n",
    "    state = saved_state\n",
    "    for w in train_inputs:\n",
    "        e = tf.nn.embedding_lookup(embeddings, w)\n",
    "        output, state = lstm_cell(e, output, state)\n",
    "        outputs.append(output)\n",
    "        \n",
    "    with tf.control_dependencies([saved_output.assign(output), saved_state.assign(state)]):\n",
    "        sparse_outputs = tf.nn.dropout(tf.concat(outputs, 0), keep_prob=0.5)\n",
    "        logits = tf.nn.xw_plus_b(sparse_outputs, softmax_weights, softmax_biases)\n",
    "        true = tf.one_hot(indices=tf.concat(train_labels, 0), depth=vocabulary_size)\n",
    "        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=tf.concat(true, 0), logits=logits)\n",
    "                              + 0.01 * tf.nn.l2_loss(Tw) + 0.01 * tf.nn.l2_loss(Tb))\n",
    "    # State resetting\n",
    "    reset_state = tf.group(saved_state.assign(tf.zeros([batch_size, num_nodes])))\n",
    "    tf.add_to_collection('ops', reset_state)\n",
    "    \n",
    "    # Optimizer\n",
    "    gs = tf.Variable(0)\n",
    "    learning_rate = tf.train.exponential_decay(1e-2, gs, 3000, 0.1, staircase=True)\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "    \n",
    "    # Gradients\n",
    "    gradients, v = zip(*optimizer.compute_gradients(loss))\n",
    "    gradients, _ = tf.clip_by_global_norm(gradients, 1.25)\n",
    "    optimizer = optimizer.apply_gradients(zip(gradients, v), global_step=gs)\n",
    "\n",
    "    # Prediction\n",
    "    train_prediction = tf.nn.softmax(tf.nn.xw_plus_b(tf.nn.dropout(tf.concat(outputs, 0), keep_prob=0.5),\n",
    "                                                     softmax_weights, softmax_biases))\n",
    "    \n",
    "    # Evaluation\n",
    "    sample_input = tf.placeholder(tf.int32, [1])\n",
    "    saved_sample_output = tf.Variable(tf.zeros([1, num_nodes]), name='saved_sample_output')\n",
    "    saved_sample_state = tf.Variable(tf.zeros([1, num_nodes]), name='saved_sample_state')\n",
    "    \n",
    "    sample_embedding = tf.nn.embedding_lookup(embeddings, sample_input)\n",
    "    reset_sample_state = tf.group(saved_sample_output.assign(tf.zeros([1, num_nodes])),\n",
    "                                  saved_sample_state.assign(tf.zeros([1, num_nodes])))\n",
    "    \n",
    "    tf.add_to_collection('ops', reset_sample_state)\n",
    "    \n",
    "    sample_output, sample_state = lstm_cell(sample_embedding, saved_sample_output, saved_sample_state)\n",
    "    with tf.control_dependencies([saved_sample_output.assign(sample_output), saved_sample_state.assign(sample_state)]):\n",
    "        sample_prediction = tf.nn.softmax(tf.nn.xw_plus_b(sample_output, softmax_weights, softmax_biases))\n",
    "        tf.add_to_collection('ops', sample_prediction)\n",
    "        \n",
    "    # Saving\n",
    "    to_save = {'Tw': Tw, 'Tb': Tb, 'saved_state': saved_state, 'saved_output': saved_output,\n",
    "               'softmax_weights': softmax_weights, 'softmax_biases': softmax_biases,\n",
    "               'saved_sample_output': saved_sample_output, 'saved_sample_state': saved_sample_state,\n",
    "               'embeddings': embeddings}\n",
    "    \n",
    "    saver = tf.train.Saver(to_save)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Average loss at step 0: 77.90 learning rate 1.000000e-02\n",
      "Minibatch perplexity: 8.697712e+06\n",
      "==================================================================================================\n",
      "значит болезненно значит защитник значит приземлиться значит подлизываться значит увезти значит запрещать значит помечтать значит потеплеть значит выдохнуть значит сообразовываться значит соваться значит нигде значит умалчивать значит естественность значит поселиться значит утренний значит простота значит сжечь значит курс значит\n",
      "==================================================================================================\n",
      "Validation set perplexity: 6.985936e+06\n",
      "\n",
      "Average loss at step 500: 4.10 learning rate 1.000000e-02\n",
      "Minibatch perplexity: 1.135128e+01\n",
      "Validation set perplexity: 4.030215e+01\n",
      "\n",
      "Average loss at step 1000: 2.53 learning rate 1.000000e-02\n",
      "Minibatch perplexity: 8.020035e+00\n",
      "Validation set perplexity: 3.035326e+01\n",
      "\n",
      "Average loss at step 1500: 2.27 learning rate 1.000000e-02\n",
      "Minibatch perplexity: 5.417690e+00\n",
      "Validation set perplexity: 3.244524e+01\n",
      "\n",
      "Average loss at step 2000: 2.14 learning rate 1.000000e-02\n",
      "Minibatch perplexity: 6.760558e+00\n",
      "Validation set perplexity: 3.723173e+01\n",
      "\n",
      "Average loss at step 2500: 2.08 learning rate 1.000000e-02\n",
      "Minibatch perplexity: 5.224605e+00\n",
      "==================================================================================================\n",
      "девиз львов если конечно уметь приходиться но к он вы намерить предложить\n",
      "==================================================================================================\n",
      "Validation set perplexity: 4.850565e+01\n",
      "\n",
      "Average loss at step 3000: 2.04 learning rate 1.000000e-03\n",
      "Minibatch perplexity: 4.557762e+00\n",
      "Validation set perplexity: 4.293648e+01\n",
      "\n",
      "Average loss at step 3500: 1.70 learning rate 1.000000e-03\n",
      "Minibatch perplexity: 4.012845e+00\n",
      "Validation set perplexity: 3.135741e+01\n",
      "\n",
      "Average loss at step 4000: 1.66 learning rate 1.000000e-03\n",
      "Minibatch perplexity: 4.877762e+00\n",
      "Validation set perplexity: 6.166612e+01\n",
      "\n",
      "Average loss at step 4500: 1.64 learning rate 1.000000e-03\n",
      "Minibatch perplexity: 4.549268e+00\n",
      "Validation set perplexity: 3.429857e+01\n",
      "\n",
      "Average loss at step 5000: 1.63 learning rate 1.000000e-03\n",
      "Minibatch perplexity: 4.191356e+00\n",
      "==================================================================================================\n",
      "водолей переходить к долгий отличный день стоить ебанить похищение даже с нетерпение весь мечта и наконец сдуреть то ли просто\n",
      "==================================================================================================\n",
      "Validation set perplexity: 5.593049e+01\n",
      "\n",
      "Average loss at step 5500: 1.61 learning rate 1.000000e-03\n",
      "Minibatch perplexity: 4.099434e+00\n",
      "Validation set perplexity: 3.855091e+01\n",
      "\n",
      "Average loss at step 6000: 1.60 learning rate 1.000000e-04\n",
      "Minibatch perplexity: 4.396310e+00\n",
      "Validation set perplexity: 8.209823e+01\n",
      "\n",
      "Average loss at step 6500: 1.56 learning rate 1.000000e-04\n",
      "Minibatch perplexity: 4.048745e+00\n",
      "Validation set perplexity: 7.050976e+01\n",
      "\n",
      "Average loss at step 7000: 1.54 learning rate 1.000000e-04\n",
      "Minibatch perplexity: 4.091596e+00\n",
      "Validation set perplexity: 5.263522e+01\n",
      "\n",
      "Average loss at step 7500: 1.54 learning rate 1.000000e-04\n",
      "Minibatch perplexity: 4.249883e+00\n",
      "==================================================================================================\n",
      "каждый человек да рак всегда слушать мир они\n",
      "==================================================================================================\n",
      "Validation set perplexity: 7.108649e+01\n",
      "\n",
      "Converged\n"
     ]
    }
   ],
   "source": [
    "num_steps = 20001\n",
    "print_every = 500\n",
    "losses = []\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    tf.global_variables_initializer().run()\n",
    "    print('Initialized')\n",
    "    \n",
    "    average_loss = 0\n",
    "    for step in range(num_steps):\n",
    "        batches = train_batches.next()\n",
    "        labels = np.concatenate(batches[1:])\n",
    "        feed_dict = {}\n",
    "        for i in range(trunc_by + 1):\n",
    "            feed_dict[train_data[i]] = batches[i]\n",
    "\n",
    "        opt, l, pred, lr = session.run([optimizer, loss, train_prediction, learning_rate], feed_dict=feed_dict)\n",
    "        average_loss += l\n",
    "        if step % print_every == 0:\n",
    "            if step > 0:\n",
    "                average_loss = average_loss / print_every\n",
    "            losses.append(average_loss)\n",
    "            print 'Average loss at step %d: %.2f learning rate %e' % (step, average_loss, lr)\n",
    "            average_loss = 0\n",
    "            labels = np.concatenate(batches[1:])\n",
    "            print 'Minibatch perplexity: %e' % perplexity(pred, labels)\n",
    "                  \n",
    "            if (step % (print_every * 5)) == 0:\n",
    "                sym = word_to_index['<$>']\n",
    "                reset_sample_state.run()\n",
    "                iter_num = 0\n",
    "                tweet = []\n",
    "                while sym != word_to_index['<#>'] and iter_num < 40:\n",
    "                    prediction = sample_prediction.eval({sample_input: np.array([sym])})[0]\n",
    "                    prediction = sample(prediction)\n",
    "                    if prediction == word_to_index['<$>']:\n",
    "                        break\n",
    "                    tweet.append(index_to_word[prediction])\n",
    "                    sym = prediction\n",
    "                    iter_num += 1\n",
    "                print '=' * 98\n",
    "                print ' '.join(tweet[:-1])\n",
    "                print '=' * 98\n",
    "                \n",
    "                saver.save(session, './Model2/model')\n",
    "            \n",
    "            reset_sample_state.run()\n",
    "            valid_logprob = 0\n",
    "            for _ in range(valid_size):\n",
    "                b = valid_batches.next()\n",
    "                predictions = sample_prediction.eval({sample_input: b[0]})\n",
    "                valid_logprob = valid_logprob + logprob(predictions, b[1])\n",
    "            print 'Validation set perplexity: %e\\n' % np.exp(valid_logprob / valid_size)\n",
    "            if len(losses) > 2 and abs(losses[-2] - losses[-1]) < 1e-3:\n",
    "                print 'Converged'\n",
    "                break\n",
    "        if array_equals(batches[-1], np.zeros_like(batches[0])):\n",
    "            reset_state.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x111adedd0>]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABIgAAAJTCAYAAABjBS0VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xl05fdd5+n3vVdbqbTUXqUlLlfJsRK7HMdxEi+JzdKE\nBrqhYaAbmGbYGybDkj4dwjpNNw3NmnBOWELTAZrpYeDANENYeiCQoRMbk3JIZbOdWEmttlS7q0qq\nRbs0f+iWquzEdi2Sfnd5nnPqSLq5kj/OUU7s1/l+P7/S4uJiAAAAAGhe5aIHAAAAAKBYAhEAAABA\nkxOIAAAAAJqcQAQAAADQ5AQiAAAAgCYnEAEAAAA0OYEIAAAAoMkJRAAAAABNrmUlf9jw8PCXJvmm\nJOuS/OLIyMgTK/nzAQAAAFh5K32CaN3IyMj3JHlXki9f4Z8NAAAAwCooLS4uXtMbh4eH70vy8yMj\nI18yPDxcSvKeJHcnmUry3SMjIwer7+tM8qtJfmRkZOT06owNAAAAwEq5phNEw8PD70jy3iTt1Ze+\nNkn7yMjIg0l+LMkvV9+3JUtx6CfFIQAAAID6cK1XzPYn+bqrvn5zkr9KkpGRkceT3Ft9/V1JdiT5\nueHh4f9ppYYEAAAAYPVc05LqkZGRPxkeHt551Us9Scav+np+eHi4PDIy8m3X8xfft2/ftd1vAwAA\nAOCa3XvvvaXref+NPsVsIkn3VV+XR0ZGFm7kB917770v/yZIsm/fPr8vXBO/K1wPvy9cK78rXA+/\nL1wrvytcD78vXKt9+/Zd9/fc6FPMHkvyVUkyPDx8fxKPswcAAACoUzd6guhPkrxleHj4serX37FC\n8wAAAACwxq45EI2MjBxJ8mD188Ukb12toQAAAABYOzd6xQwAAACABiEQAQAAADQ5gQgAAACgyQlE\nAAAAAE1OIAIAAABocgIRAAAAQJMTiAAAAACanEAEAAAA0OQEIgAAAIAmJxABAAAANDmBCAAAAKDJ\nCUQAAAAATU4gAgAAAGhyAhEAAABAkxOIAAAAAJqcQAQAAADQ5AQiAAAAgCYnEAEAAAA0OYEIAAAA\noMkJRAAAAABNTiACAAAAaHICEQAAAECTE4gAAAAAmpxABAAAANDkCg9EFyZnix4BAAAAoKkVHog+\nPnKy6BEAAAAAmlrhgejA6LmiRwAAAABoasUHorHxokcAAAAAaGrFB6LR8SwuLhY9BgAAAEDTKjwQ\nnb80k1PnJoseAwAAAKBpFR6IkqVTRAAAAAAUozYC0ZhF1QAAAABFqY1A5AQRAAAAQGEKD0RbNqzL\nQSeIAAAAAApTeCAaGujNmYnpnJmYKnoUAAAAgKZUfCAa3JAkOTjmmhkAAABAEWogEPUmSQ6MumYG\nAAAAUITiA9FANRA5QQQAAABQiMID0aaejmzobs9+J4gAAAAAClF4ICqVShka6M2ps5MZvzBd9DgA\nAAAATafwQJQkt1lUDQAAAFCYmghEy4uqBSIAAACANVcbgWhg6QSRJ5kBAAAArL2aCERbN65Ld2er\nE0QAAAAABaiJQLS0qHpDjp2+mIuTs0WPAwAAANBUaiIQJVf2EB086hQRAAAAwFqqnUC0vIdIIAIA\nAABYS7UTiJafZGZRNQAAAMBaqplAtGPz+qxrb3GCCAAAAGCN1UwgKpdL2T3Qm7GT5zM1PVf0OAAA\nAABNo2YCUbJ0zWxhMTl8bKLoUQAAAACaRm0FouVF1fYQAQAAAKyV2gpEy4uq7SECAAAAWCs1FYgG\nt3alrbViUTUAAADAGqqpQFSplLOrvydHjk9kdm6+6HEAAAAAmkJNBaIkGRrozfzCYo4cO1/0KAAA\nAABNofYC0eDSour9FlUDAAAArInaC0QDFlUDAAAArKWaC0S37OhJS6XsUfcAAAAAa6TmAlFrSzm3\n9nXn8LGJzM0vFD0OAAAAQMOruUCULO0hmp1byLMnLKoGAAAAWG21GYgu7yEatYcIAAAAYLXVZiCq\nPsnswJg9RAAAAACrrSYD0c6+npTLJSeIAAAAANZATQai9tZKbtnenUNHxzO/sFj0OAAAAAANrSYD\nUZLsHujN1Mx8jp66UPQoAAAAAA2tZgPR0GB1UfWYa2YAAAAAq6l2A9FAdVH1qEXVAAAAAKupZgPR\nrv6elErJQSeIAAAAAFZVzQaizo7W9G/pyoHRc1lctKgaAAAAYLXUbCBKlvYQXZyay4kzl4oeBQAA\nAKBh1XYgWt5D5JoZAAAAwGqp7UC0/CQzi6oBAAAAVkttB6KBaiBygggAAABg1dR0IOrqbMv2TZ3Z\nb1E1AAAAwKqp6UCULF0zm7g4k9PnpooeBQAAAKAh1Xwgum2wuqjaHiIAAACAVVHzgciTzAAAAABW\nV80Hot0DnmQGAAAAsJpqPhBt6G7Plt4OJ4gAAAAAVknNB6IkGRrckDMTUzk7YVE1AAAAwEqrj0C0\nfM3MKSIAAACAlVYfgciTzAAAAABWTZ0EouoJInuIAAAAAFZcXQSiTT0d2dDV7ooZAAAAwCqoi0BU\nKpWye7A3J89cyvlLM0WPAwAAANBQ6iIQJVcWVR90zQwAAABgRdVPILKoGgAAAGBV1E8gGrCoGgAA\nAGA11E0g2r6pM+vXtTpBBAAAALDC6iYQlUqlDA30ZuzUxVyami16HAAAAICGUTeBKLmyh+jQ0YmC\nJwEAAABoHPUViKp7iPaPumYGAAAAsFLqKxANXl5ULRABAAAArJS6CkT9W7qyrr2SA2OeZAYAAACw\nUuoqEJXLpewe2JDRE+czNTNX9DgAAAAADaGuAlGytIdoYTE5fMyiagAAAICVUH+BaHkPkWtmAAAA\nACuh/gLRwNKj7i2qBgAAAFgZdReIBrd1pa2lbFE1AAAAwAqpu0BUqZSzq783zxyfyOzcfNHjAAAA\nANS9ugtESbJ7sDdz84s5cvx80aMAAAAA1L26DERX9hC5ZgYAAABws+ozEF1+ktmYRdUAAAAAN6su\nA9HOHd1pqZRy0AkiAAAAgJtWl4GotaWSW3b05NDR8czPLxQ9DgAAAEBdq8tAlCRDA72ZmVvI6MkL\nRY8CAAAAUNfqNxANVhdV20MEAAAAcFPqOBBVF1XbQwQAAABwU+o2EN3a15NyKTkwJhABAAAA3Iy6\nDUQdbS0Z3N6dg2PnsrCwWPQ4AAAAAHWrbgNRsrSoenJ6PkdPW1QNAAAAcKPqOhDddnlRtT1EAAAA\nADesrgPRlSeZCUQAAAAAN6quA9Gu/p4kyYFRj7oHAAAAuFF1HYg6O1ozsHV9DoyNZ3HRomoAAACA\nG1HXgShJhgY25OLkbE6cuVT0KAAAAAB1qf4D0WBvEnuIAAAAAG5U/QeigctPMrOHCAAAAOBG1H0g\n2u0EEQAAAMBNqftA1N3Zlm2bOnNg9JxF1QAAAAA3oO4DUZIMDfRm/MJMzkxMFT0KAAAAQN1pjEB0\n+ZrZqGtmAAAAANerMQKRRdUAAAAAN6wxApFF1QAAAAA3rCEC0cbujmzq6XCCCAAAAOAGNEQgSpZO\nEZ0en8q589NFjwIAAABQVxonEFX3EB10zQwAAADgujROIKruIdrvmhkAAADAdWmcQHT5SWZjAhEA\nAADA9WiYQLRlQ0d6u9pyYNQVMwAAAIDr0TCBqFQqZWhgQ06cuZQLl2aKHgcAAACgbjRMIEqu7CE6\nYFE1AAAAwDVrrEB0eQ+Ra2YAAAAA16yxAtHyCSKLqgEAAACuVUMFou2bOrO+o8UJIgAAAIDr0FCB\nqFQqZWhwQ46evpBLU7NFjwMAAABQFxoqECXJ7oHeLC4mh45OFD0KAAAAQF1ouEA0NFhdVG0PEQAA\nAMA1abxANFBdVG0PEQAAAMA1abhA1L+1Kx1tlRwcE4gAAAAArkXDBaJKuZRd/b155sT5TM/OFz0O\nAAAAQM1ruECUJEODvVlYWMyRYxZVAwAAALycxgxEA9VF1aMWVQMAAAC8nMYMRIPVRdX2EAEAAAC8\nrIYMRK/Y3p3WlrITRAAAAADXoCEDUUulnFv7enL42PnMzi0UPQ4AAABATWvIQJQktw1uyNz8Qp45\nblE1AAAAwEtp2EBkDxEAAADAtWncQORJZgAAAADXpGED0c6+7lTKJSeIAAAAAF5Gwwai1pZKdu7o\nyaGjE5mft6gaAAAA4MU0bCBKlvYQzczOZ/TUhaJHAQAAAKhZjR2IBqqLqkddMwMAAAB4MY0diAar\ni6rHLKoGAAAAeDENHYhu7etJueQEEQAAAMBLaehA1NHekoFt3Tk4Np6FhcWixwEAAACoSQ0diJKl\nRdWT03M5/tzFokcBAAAAqEmNH4gGqnuIXDMDAAAA+IIaPxANVp9kZlE1AAAAwBfU8IFod79H3QMA\nAAC8lIYPROvXtaZvy/ocGDuXxUWLqgEAAABeqOEDUZIMDfTm/KXZnDo7WfQoAAAAADWnOQLRYHVR\ntT1EAAAAAJ+nOQLRwNIeov32EAEAAAB8nuYIRJdPEI06QQQAAADwQk0RiHrWt2XbxnU5MDpuUTUA\nAADACzRFIEqWThGduzCdMxNTRY8CAAAAUFOaJxBV9xAdGLOHCAAAAOBqzROIlvcQCUQAAAAAV2ue\nQHT5BJFF1QAAAADP0zSBaGNPRzb1tLtiBgAAAPACTROIkmT3wIacPjeZ8QvTRY8CAAAAUDOaKhAN\nDVpUDQAAAPBCzRWIBi4vqraHCAAAAOCy5gpEThABAAAAfJ6mCkRbN6xLd2dbDnrUPQAAAMCypgpE\npVIpQ4O9OfbcxVyYnC16HAAAAICa0FSBKEmGBpaumR1yzQwAAAAgSTMGosHqouoxi6oBAAAAkqYM\nRNVF1fYQAQAAACRpwkC0Y9P6dHa0OEEEAAAAUNV0gahcLmX3QG9GT17I5PRc0eMAAAAAFK7pAlGS\n3Da4IYuLyaGjrpkBAAAANGUguvwkM3uIAAAAAJo1EHmSGQAAAMCypgxE/Vu70t5WcYIIAAAAIE0a\niCrlUnb39+aZE+czMztf9DgAAAAAhWrKQJQs7SFaWFjM4WMTRY8CAAAAUKjmDUSD1UXVY66ZAQAA\nAM2tiQNRdVH1qEXVAAAAQHNr2kD0iu3daamUnSACAAAAml7TBqKWSjm39vfk8NGJzM0vFD0OAAAA\nQGGaNhAlS4uq5+YX8uyJ80WPAgAAAFCY5g5E9hABAAAANHkgGqg+yWzUHiIAAACgeTV1ILq1ryfl\ncsmiagAAAKCpNXUgamut5Jbt3Tl4dDzzC4tFjwMAAABQiKYOREkyNNib6Zn5HD11oehRAAAAAAoh\nEA1YVA0AAAA0N4FocGlR9X6LqgEAAIAm1fSBaHd/b0ql5MCYE0QAAABAc2r6QNTR3pLBbV05ODae\nBYuqAQAAgCbU9IEoWdpDdGlqLsfPXCx6FAAAAIA1JxDlyh6iA/YQAQAAAE1IIIonmQEAAADNTSBK\nsmugeoJozAkiAAAAoPkIREm61rWmb/P6HBgdz+KiRdUAAABAcxGIqnYP9ub8pZmcOjdZ9CgAAAAA\na0ogqhoasKgaAAAAaE4CUdXQYHVR9ZhF1QAAAEBzEYiqnCACAAAAmpVAVNXb1Z4tG9bloBNEAAAA\nQJMRiK4yNNCbMxPTOTMxVfQoAAAAAGtGILrK5T1EB8dcMwMAAACah0B0laHBy3uIXDMDAAAAmodA\ndJXlRdVOEAEAAABNRCC6yqaejmzobneCCAAAAGgqAtFVSqVShgZ6c/LsZCYuzhQ9DgAAAMCaEIhe\n4LbqomqniAAAAIBmIRC9wPKianuIAAAAgCYhEL3A0IATRAAAAEBzEYheYOvGdenubHWCCAAAAGga\nAtELLC2q3pBjpy/m4uRs0eMAAAAArDqB6Au4vIfo4FGniAAAAIDGJxB9AVf2EAlEAAAAQOMTiL6A\nK08ys6gaAAAAaHwC0RewY/P6rGtvcYIIAAAAaAoC0RdQLpeye6A3YyfPZ2p6ruhxAAAAAFaVQPQi\nhgZ7s7CYHD42UfQoAAAAAKtKIHoRVxZV20MEAAAANDaB6EVcWVRtDxEAAADQ2ASiFzG4tSttrRWL\nqgEAAICGJxC9iEqlnF39PTlyfCKzc/NFjwMAAACwagSilzA00Jv5hcUcOXa+6FEAAAAAVo1A9BKG\nBquLqscsqgYAAAAal0D0EoYGlhZV77eHCAAAAGhgAtFLuGVHT1oqZY+6BwAAABqaQPQSWlvKubWv\nO4ePTWRufqHocQAAAABWhUD0MoYGN2R2biHPnrCoGgAAAGhMAtHLuLyH6IA9RAAAAECDEohehieZ\nAQAAAI1OIHoZO/t6Ui6XnCACAAAAGpZA9DLaWyu5ZXt3Dh0dz/zCYtHjAAAAAKw4gega7B7ozdTM\nfI6eulD0KAAAAAArTiC6BkOD1UXVY66ZAQAAAI1HILoGQwPVRdWjFlUDAAAAjUcguga7+ntSKiUH\nnSACAAAAGpBAdA06O1rTv6UrB0bPZXHRomoAAACgsQhE12hosDcXp+Zy4sylokcBAAAAWFEC0TW6\nsofINTMAAACgsQhE1+jKk8wsqgYAAAAai0B0jYYGqoHICSIAAACgwQhE16irsy3bN3XmwJhF1QAA\nAEBjEYiuw9Bgb8YvzOS58amiRwEAAABYMQLRdbhtcGlR9f5Re4gAAACAxiEQXQdPMgMAAAAakUB0\nHXYPeJIZAAAA0HgEouuwobs9W3o7nCACAAAAGopAdJ2GBjfkzMRUzk5YVA0AAAA0BoHoOg0tXzNz\niggAAABoDALRdRqqPsnMHiIAAACgUQhE12losHqCyB4iAAAAoEEIRNdpU09HNnS1u2IGAAAANAyB\n6DqVSqXsHuzNyTOXcv7STNHjAAAAANw0gegGXF5UfdA1MwAAAKABCEQ3wKJqAAAAoJEIRDdg+VH3\nThABAAAADUAgugHbN3Vm/bpWJ4gAAACAhrAqgWh4ePhLhoeH37saP7sWlEqlDA30ZuzUxVyami16\nHAAAAICbsuKBaHh4eCjJ65K0r/TPriWX9xAdOjpR8CQAAAAAN6flet48PDx8X5KfHxkZ+ZLh4eFS\nkvckuTvJVJLvHhkZOTgyMnIgybuGh4f/68qPWzuu7CE6lzt3by54GgAAAIAbd80niIaHh9+R5L25\ncjLoa5O0j4yMPJjkx5L88gu+pbQiE9aoocFqIBqzqBoAAACob9dzxWx/kq+76us3J/mrJBkZGXk8\nyetf8P7FmxuttvVv6cq69kr2j1pUDQAAANS30uLitXec4eHhnUn+YGRk5MHqEur/NjIy8v7qf3Y4\nye6RkZGFa/15+/btq+uI9Dt/czLPnp7Jj/3z/rS1eCAcAAAAUBvuvffe67rZdV07iF5gIkn3VV+X\nrycOXXbvvffexAjF+tizT+SZRw9m446hvGrnpqLHaXj79u2r698X1o7fFa6H3xeuld8VroffF66V\n3xWuh98XrtW+ffuu+3tu5tjLY0m+KkmGh4fvT/LETfysurS8h2jUHiIAAACgft3MCaI/SfKW4eHh\nx6pff8cKzFNXhgaWHnV/wB4iAAAAoI5dVyAaGRk5kuTB6ueLSd66GkPVi8FtXWlrKXuSGQAAAFDX\nbFa+CZVKObv6e/PM8YnMzs0XPQ4AAADADRGIbtLuwd7MzS/myPHzRY8CAAAAcEMEopt0ZQ+Ra2YA\nAABAfRKIbtLyk8zGLKoGAAAA6pNAdJN27uhOS6WUg04QAQAAAHVKILpJrS2V3LKjJ4eOjmd+fqHo\ncQAAAACum0C0AoYGejMzt5DRkxeKHgUAAADguglEK2BosLqo2h4iAAAAoA4JRCtgeVG1PUQAAABA\nHRKIVsCtfT0pl5IDYwIRAAAAUH8EohXQ0daSwe3dOTh2LgsLi0WPAwAAAHBdBKIVctvghkxOz+fY\ncxeLHgUAAADgughEK2RoYGkP0f5nLaoGAAAA6otAtEKuPMnMHiIAAACgvghEK2RXf0+S5MCoE0QA\nAABAfRGIVkhnR2sGtq7PgbHxLC5aVA0AAADUD4FoBQ0NbMjFydmcOHOp6FEAAAAArplAtIKGBpcW\nVdtDBAAAANQTgWgFDQ1UF1XbQwQAAADUEYFoBe12gggAAACoQwLRCurubMu2TZ05MHrOomoAAACg\nbghEK2xooDfjF2ZyZmKq6FEAAAAArolAtMKWF1WPumYGAAAA1AeBaIVZVA0AAADUG4FohXnUPQAA\nAFBvBKIVtrG7I5t6OpwgAgAAAOqGQLQKhgZ7c3p8KufOTxc9CgAAAMDLEohWweU9RAddMwMAAADq\ngEC0Cq7sIXLNDAAAAKh9AtEquG3w8pPMnCACAAAAap9AtAo293akt6st+y2qBgAAAOqAQLQKSqVS\nhgY25MSZS7lwaabocQAAAABekkC0Sq7sIXLNDAAAAKhtAtEqufwkM3uIAAAAgFonEK0STzIDAAAA\n6oVAtEq2b+rM+o4WJ4gAAACAmicQrZJSqZShwQ05evpCLk3NFj0OAAAAwIsSiFbR7oHeLC4mh45O\nFD0KAAAAwIsSiFbR0GB1UbU9RAAAAEANE4hW0dBAdVG1PUQAAABADROIVlH/1q50tFVycEwgAgAA\nAGqXQLSKKuVSdvX35pkT5zM9O1/0OAAAAABfkEC0yoYGe7OwsJgjxyyqBgAAAGqTQLTKhgaqi6pH\nLaoGAAAAapNAtMqGBquLqu0hAgAAAGqUQLTKXrG9O60tZSeIAAAAgJolEK2ylko5t/b15PCx85md\nWyh6HAAAAIDPIxCtgdsGN2RufiHPnjhf9CgAAAAAn0cgWgOX9xDtd80MAAAAqEEC0RrwJDMAAACg\nlglEa2BnX3cq5ZInmQEAAAA1SSBaA60tlezc0ZNDRycyP29RNQAAAFBbBKI1MjTYm5nZ+YyeulD0\nKAAAAADPIxCtkaGBpUXVB0ZdMwMAAABqi0C0RoYGq4uqxyyqBgAAAGqLQLRGbu3rSbnkBBEAAABQ\newSiNdLR3pKBbd05ODaehYXFoscBAAAAWCYQraGhwd5MTs/l+HMXix4FAAAAYJlAtIaGBqp7iFwz\nAwAAAGqIQLSGhgarTzKzqBoAAACoIQLRGtrd71H3AAAAQO0RiNbQ+nWt6duyPgfGzmVx0aJqAAAA\noDYIRGtsaKA35y/N5tTZyaJHAQAAAEgiEK25ocHqomp7iAAAAIAaIRCtsaEBe4gAAACA2iIQrbEr\nJ4gEIgAAAKA2CERrrGd9W7ZtXJf9oxZVAwAAALVBICrA0OCGnDs/nTMTU0WPAgAAACAQFWF5D5Fr\nZgAAAEANEIgKsLyHyKJqAAAAoAYIRAW48iQzj7oHAAAAiicQFWBjT0c29bS7YgYAAADUBIGoILsH\nNuT0ucmMX5guehQAAACgyQlEBRkatKgaAAAAqA0CUUGGBi4vqraHCAAAACiWQFQQJ4gAAACAWiEQ\nFWTrhnXp7mzLQY+6BwAAAAomEBWkVCplaLA3x567mAuTs0WPAwAAADQxgahAQwNL18wOuWYGAAAA\nFEggKtDQYHVR9ZhF1QAAAEBxBKICLS+qtocIAAAAKJBAVKAdm9ans6PFCSIAAACgUAJRgcrlUnYP\n9Gb05IVMTc8VPQ4AAADQpASigt02uCGLi8mhoxNFjwIAAAA0KYGoYJefZLZ/1DUzAAAAoBgCUcE8\nyQwAAAAomkBUsP6tXWlvq3iSGQAAAFAYgahglXIpu/t788yJ85mZnS96HAAAAKAJCUQ1YGigNwsL\nizl8zKJqAAAAYO0JRDVgaHBpUfWBMdfMAAAAgLUnENWA5UXVnmQGAAAAFEAgqgGv2N6dlkrZCSIA\nAACgEAJRDWiplHNrf08OH53I3PxC0eMAAAAATUYgqhFDA72Zm1/IsyfOFz0KAAAA0GQEohphDxEA\nAABQFIGoRgwNVJ9kNmoPEQAAALC2BKIacWtfT8rlkkXVAAAAwJoTiGpEW2slt2zvzsGj45lfWCx6\nHAAAAKCJCEQ1ZGiwN9Mz8zl66kLRowAAAABNRCCqIUMDFlUDAAAAa08gqiFDg9VF1fYQAQAAAGtI\nIKohu/t7Uyp5khkAAACwtgSiGtLR3pLBbV0ZeeZsnj5ypuhxAAAAgCYhENWYf/GPbs/c3Hx+/D2P\n5ZGPjxY9DgAAANAEBKIa88X3viI/+d33p7WlnF/6vX35/fc/ncVFj70HAAAAVo9AVIPufdX2/NIP\nPJTtmzrzB389knf+3r5Mz84XPRYAAADQoASiGnXLjp68620P59W3bsojnxjLT7znsZydmCp6LAAA\nAKABCUQ1rLerPf/xrQ/mS1//iow8czb/5t2P5NBRTzgDAAAAVpZAVONaWyr51990T771q16d0+cm\n8yO/9mg+8unjRY8FAAAANBCBqA6USqX88390e370296Q+YXkZ37n8bzvQ/strwYAAABWhEBUR970\nmv78wve9ORu7O/Lbf/ZUfv2/fTJz8wtFjwUAAADUOYGoztz2ig355X/9cHYP9Ob9e4/k3/3nD+f8\npZmixwIAAADqmEBUhzb3rssvfN+bc/+eHfnU/tP5oXc/krFTF4oeCwAAAKhTAlGd6mhvyY992xvz\nDV/6yhw9fTE/9O5H8qn9p4oeCwAAAKhDAlEdK5dL+bZ/ckfe9o33ZGpmLj/5mx/O+/ceKXosAAAA\noM4IRA3gy954S376ex9MZ0drfu3//kR++8+ezPyCJ5wBAAAA10YgahB7hrbkXW97OIPbuvK+Dx3I\nz/6Xj2Ryeq7osQAAAIA6IBA1kL4t6/NLP/hwXnv71nzk08fzI7/2aE6evVT0WAAAAECNE4gaTNe6\n1vz7774/X/XgrTl0dCJvf/cjGTlypuixAAAAgBomEDWgSqWct3793fmer70rExem8+PveSyPfnys\n6LEAAACAGiUQNbCvfmh3/u133Z9KpZxf/L2P5g/+eiSLi5ZXAwAAAM8nEDW41796e37pBx7Ktk2d\n+f33P513/l/7MjM7X/RYAAAAQA0RiJrAzr6evOsHH86rb92URz4+lh//jcdy9vxU0WMBAAAANUIg\nahIbutvzM//rg/niewczcuRs3v7uR3L42ETRYwEAAAA1QCBqIm2tlfybb35dvuUrX5VTZyfzw7/6\nSP7h08cabQ9iAAAfSklEQVSLHgsAAAAomEDUZEqlUr7xy4bzo9/6hswvJD/zO4/nTx85YHk1AAAA\nNDGBqEm96e7+/Pz3vSkbutvzW3/6ZN7zx5/K3PxC0WMBAAAABRCImtgrX7Ex73rbF2V3f2/+6sOH\n8+/f++FcuDRT9FgAAADAGhOImtyWDevy89//5tx354588nOn80O/8kiOnrpQ9FgAAADAGhKIyLr2\nlvz4t78xX/8lt2Xs1MW8/d2P5In9p4seCwAAAFgjAhFJknK5lG//p3fmbd/42kzNzOXf/ubf568f\nP1L0WAAAAMAaEIh4ni974878h+99MJ0dLfnVP/pEfufPn8r8giecAQAAQCMTiPg8dw1tyTvf9nAG\ntnblTz64Pz/3ux/J5PRc0WMBAAAAq0Qg4gvq39KVd/7gQ3ntK7fm8aeO50d+7dGcOjtZ9FgAAADA\nKhCIeFFdnW35d//q/nzlA7fm0NGJvP3dH8pnnzlb9FgAAADAChOIeEktlXLe+vWvyb/62j0ZvzCd\nH/v1v8ujnxgreiwAAABgBQlEvKxSqZSveWgo//a77k+lUs4v/p8fzR/+zUgWFy2vBgAAgEYgEHHN\nXv/q7fnFH3go2zauy+/91dP55d//WGZm54seCwAAALhJAhHX5da+nrzrbV+UV+3cmA9+bDQ/8RuP\n5dz56aLHAgAAAG6CQMR129Ddnv/41jfli+4ZzNNHzubt7/5QjhybKHosAAAA4AYJRNyQttZK3v4v\nX5dv+YpX5eTZybzjVx/NRz9zouixAAAAgBsgEHHDSqVSvvEtw/nh/+X1mZ9fyE//9t782aMHLK8G\nAACAOiMQcdMeeu1Afu773pyerva8931P5jf++FOZm18oeiwAAADgGglErIjbb9mYd73t4ezq78lf\nfvhwfuq9e3NhcrbosQAAAIBrIBCxYrZt7MwvfP9Due/OHfnE507lHb/ySI6evlD0WAAAAMDLEIhY\nUevaW/Jj3/7GfN0X35bRkxfyQ+9+JE8cOF30WAAAAMBLEIhYcZVyKd/51XfmB/7Fa3Npai4/+Zt/\nnw985EjRYwEAAAAvQiBi1Xz5fTvz09/7YNa1t+Tdf/iJ/O5fPJWFBU84AwAAgFojELGq7rptS975\ngw9nYOv6/PH/2J+f/d2PZHJ6ruixAAAAgKsIRKy6/q1deecPPpzX3LYljz91PD/6a3+X0+cmix4L\nAAAAqBKIWBNdnW35qe95IP/4/p05eHQ8b3/3h/K5Z88WPRYAAAAQgYg11FIp5/u+4e5819fsydnz\n0/nRX38sj33yaNFjAQAAQNMTiFhTpVIpX/tFQ/nfv/O+VMrJz//Xf8gffmAki4uWVwMAAEBRBCIK\n8cY7duQXvv+hbN24Lr/3l0/nl//gY5mdmy96LAAAAGhKAhGF2dXfm3e97eEM79yYD+4bzU/8xt/n\n3PnposcCAACApiMQUaiN3R352be+KQ/fM5DPHD6Tt//KIzlyfKLosQAAAKCpCEQUrq21kh/6l/fm\nf/7Hr8rJM5fyjl95NPuePlH0WAAAANA0BCJqQqlUyjd/+XB++Ften/n5hfyH39qbP3/0oOXVAAAA\nsAYEImrKQ/cM5Gf/tzelp6s9//l9T+Q//T+fyvz8QtFjAQAAQEMTiKg5wzs35V1vezi39vXk//37\nw/mp39qbC5OzRY8FAAAADUsgoiZt29iZX/j+N+cNd2zPxz97Ku/4lUdy+OR0pqbnih4NAAAAGk5L\n0QPAi+nsaM1PfMd9+d2/eCrv+9CB/O4HLuT/+P/+e/q3dGX3QG929fdk90Bvdvf3ZmNPR9HjAgAA\nQN0SiKhplXIp3/U1e3L3K7fm/Y8+mYtzHTl4dDyPfmIsj35ibPl9G7rbs7v/qmg00Ju+LV2plEsF\nTg8AAAD1QSCiLrz+1dtTujSae++9N4uLizl1djIHxsZz6Oh4DlY/fmzkZD42cnL5e9rbKrm1r2cp\nHA30Znd/T3b29aSjza89AAAAXM2/KVN3SqVStm3qzLZNnXngrr7l1y9cmsmhoxM5eFU02v/suYwc\nObv8nnIp6d/ald39S6eMdlWvqG3obi/ibwUAAABqgkBEw+jqbMtdt23JXbdtWX5tdm4+zxw/v3TS\n6OjEcjgaPXkhj1x1RW1TT3t2XY5G1Y99m9en7IoaAAAATUAgoqG1tlQyNLghQ4Mbll9bXFzMiTOX\nqtfTJqrxaDz7nj6ZfU9fuaLWcfmK2sCVcLSzryftrZUi/lYAAABg1QhENJ1SqZQdm9dnx+b1eeCu\n/uXXJy7O5NDRq/caTeSzz57L0y+4ojawrbt6Ra1n+bRRb5cragAAANQvgQiqeta35e5Xbs3dr9y6\n/NrM7HyeOXE+h8aWThkdql5Te/bE+Xzo41e+d1NPx/JJo6Wl2D3ZsckVNQAAAOqDQAQvoa21ktsG\nN+S2q66oLSwsXVE7eHT8SjgaG89HP3MiH/3MieX3rWuv5Na+q/ca9WTnjp60uaIGAABAjRGI4DqV\ny6X0bVmfvi3r86bXXLmiNn5hOoeveorawaPjGXnmbD5z+Mzzvndw25WnqO3u782t/T2uqAEAAFAo\ngQhWSG9Xe+6+fWvuvv3KFbXp2fk8c3ziyjLssfEcPjaeZ46fzwc/Nrr8vi29Hdm1fD1t6eP2TZ2u\nqAEAALAmBCJYRe2tlbzyFRvzyldsXH5tYWExx89cXDplNHZlr9E/fPpE/uHTV19Ra8mu/p6r9hr1\nZueO7rS2uKIGAADAyhKIYI2Vy6X0b+lK/5auvPnugeXXz52fvuopaktX1Z4+fCafPnTlilqlXMor\ntncvh6Ndfb3ZtqkzG3va09Hmf84AAADcGP9GCTViQ3d77hnelnuGty2/NjUzl2eOn1/eaXRobDyH\njk3k8LGJ/I99o8/7/vUdLdnU25FNPR3Z2NORzT1XPt/U05HNvUuft1uSDQAAwAsIRFDDOtpacvst\nG3P7LVeuqM0vLOb4cxer+4wmcvrcZM5OTOVM9c+zJy685M9cv641m3o6sqmnvfqx+qe3Ixu7hSQA\nAIBmJBBBnamUSxnY2pWBrV156LUDn/efz8zO5+z56ZwZn8qZ81NLHyee/+fsxFSePXH+Jf86l0PS\n5p6ObLwck3o7nh+VejrSJiQBAADUPYEIGkxbayXbN3Vm+6bOl3zfzOx8NRZNf15AWv4z/vIhqWtd\n61I46u646opbezb3rFv+XEgCAACobQIRNKm21kp2bF6fHZvXv+T7pmfnl6+wnZ2YznMTkzkzPrV8\nSum5akh65vhLh6TuztblfUgvvNp2JS61e0obAABAAQQi4CW130BIunz66Pknkqbz3DWGpM9brl0N\nSJuXXxeSAAAAVpJABKyI6w1Jz41P5eyL7Eg6PT6VIy8bktquLNp+wW6kE6ems+noeNa1t6S9rZJ1\nbUsfS6XSSv4tAwAANAyBCFhT1xqSpmbmPm8/0tmJK1fazp6fyulzky8ekv7mg8/7slRKOtoqaW9r\nybq2lnS0V9LR1pKOtko62lueF5M62q96vfrxytfV97Uvva+tpSw8AQAAdU8gAmpSR1tL+ra0pG/L\ndYSk6pPbnv7ckWzYuCVTM/OZnJnL1PRcpmbmM1X9fHJ6PuMXpzM1PZeFxZubs1zKUnS6HJxeLCZV\nPz7vvVfFqY62K9+/rr2SlorwBAAArB2BCKhrXygkDa4/l3vvfc3Lfu/i4mJm5xYyOT2X6atj0vRS\nTJqcmf8Ccenqr+ef972T03M5e346UzNzWbzZ8FQuXXV66cqJp+efdKq8xMmnq09JLUWn9raWtFRK\nwhMAAPB5BCKgaZVKpbS1VtLWurILrxcXFzM9O78cmqaqoWnyBbFp+fXlELX02nJ0qr52cXI2z41P\nZmpm/qZnK5dLS1ftWitpr55auvx5+4u8fvX726unotpbK8tX9q58vvR6pVJegf8WAQCAtSQQAayw\nUqm0fHInaV+xn7uwcDk8XQlNk1eFpqWv5zNd/fj5cWouM7MLmZpZClDTs/M5f2ky0zPzmZtfWLE5\nWyrlF0Sky5+3PC9CLb/e/sIY1fKiwaqjbSnolctOQQEAwEoSiADqRLlcyrrqzqKVNj+/kOnZ+eVw\nNDWzFJqufH75z9zy+6aq752uxqrl73/B6+cuTGdqZj4LN7vw6SptLeXnn2Zqr8akFwSnlz4VtfT9\nF6du/mQWAADUO4EIgFQq5XRWyunsaF21v8bc/MLzwtP0VeFp6gbC0+XPL07O5szE0hW8G9n9VCol\nf/nJv8sDe/py356+bN/UufJ/8wAAUOMEIgDWREulnK515XStW50IdXnp+PMi0vNORF0OTFdevzQ1\nl72fOpKnDj6XJw88l/f+6ZPZPdCb+/f05YG7+rJzR7el3gAANAWBCICGcPXS8e7rOAS0Z8dkdr3y\nznzkqeP58JPH8qnPncrBsfH8/vufzo7Nnbl/T1/u39OXV926KRW7jwAAaFACEQBNb1NPR77igVvz\nFQ/cmktTs/noZ05k75PH89HPnMj7PnQg7/vQgWzoas8b79yRB+7qy2tu27LiT78DAIAiCUQAcJXO\njtY8fM9gHr5nMLNz8/nk505n75PH8viTx/PXjx/JXz9+JOvaK7n3Vdtz/56+vP7V27N+la7NAQDA\nWhGIAOBFtLZU8vpXb8/rX709b/36xYwcOZO9Tx7P3ieO5e8+eTR/98mjaamU8prbtub+u/py3507\nsqmno+ixAQDguglEAHANKuVS7ti1OXfs2pzv+Kd35Jnj5/PhJ49l75PH8rGRk/nYyMn8xh9/MsO3\nbFxect2/tavosQEA4JoIRABwnUqlUnb29WRnX0++6S3DOXnmUvY+dSx7nziepw6eztNHzuZ3//un\n84rt3Xngrr7cv2dHbhvc4IloAADULIEIAG7Stk2d+ZqHhvI1Dw1l/MJ0/uHTJ7L3yWP5+MjJ/NEH\nPps/+sBns6W3Y+mJaHf15c7dm9NSKRc9NgAALBOIAGAF9Xa158veeEu+7I23ZGp6Lh8bOZm9Tx7L\nRz59In/x2KH8xWOH0rWuNW+8c0fu37Mj9wxvS0eb/zsGAKBY/okUAFZJR3tLHnxNfx58TX/m5hfy\n1IHnlvcW/e1Hn83ffvTZtLVWcs/tW/PAXX15wx070rO+reixAQBoQgIRAKyBlko5d9++NXffvjXf\n+3V35XPPnsveaix6/Knjefyp4ymXS9mze3Pu27Mj9+/py7aNnUWPDQBAkxCIAGCNlUql3H7Lxtx+\ny8Z861fdkdGT57P3yePZ++SxfGr/6Xxq/+m8931PZmiwNw/s6cv9e/pyy45uS64BAFg1AhEAFGxw\nW3e+4Uu78w1f+so8Nz6Zjzx1PB9+YikWHRgdz+/91dPp27I+9+/pywN7+jK8c2PKZbEIAICVIxAB\nQA3Z3LsuX/ngrnzlg7tyYXI2H/3M0hPR9n3mRP7kg/vzJx/cnw3d7bnvzqVraHe/cktaWypFjw0A\nQJ0TiACgRnWta80Xv24wX/y6wczMzueTnzuVDz9xLB/59PG8f++RvH/vkaxrb8nrX709D+zpy72v\n3pbOjtaixwYAoA4JRABQB9paK3nDHTvyhjt2ZH5hMU8fPpO9Tx7Lh584lkc/MZZHPzG2tAj7lVty\n/56+3LdnRzZ2dxQ9NgAAdUIgAoA6UymXcufuzblz9+Z851ffmcPHJpaWXD9xLPuePpl9T5/Me/74\nk3nVzk25f09f7r9rR/q3dBU9NgAANUwgAoA6ViqVsqu/N7v6e/PNXz6c489dzOPVJdefOfRcPnP4\nTP7LXzyVnTu6q7GoL0MDvZ6IBgDA8whEANBAdmxen3/28FD+2cNDGb8wnY88dTx7nzyej3/2ZP7w\nA5/NH37gs9m6cd1SLNqzI3fu2pxKpVz02AAAFEwgAoAG1dvVnrfctzNvuW9nJqfn8rGRk9n7xLH8\nw6eP588fPZg/f/Rgujtb84Y7duSBu/ry2tu3pqPNPxoAADQj/xQIAE1gXXtL3vSa/rzpNf2Zm1/I\nE/tPZ++Tx7L3yeP5248+m7/96LNpb6vkdcPb8sY7dmTbpnXpaGtJe1sl7a2VdLS1pKOtkrbWSspl\n19MAABqNQAQATaalUs49w9tyz/C2fO/XvSb7R8/lw08cW34q2oefOPaS39/WWklH29Kf9mpEWvr6\n6qB0JSq1V9/3vPd8oe9pb0lbS9l+JACAAghEANDEyuVSbr9lY26/ZWO+7Z/ckWdPnM/HP3syFy7N\nZmpmPtMzc9WP85ma+f/bu/8QW/O6DuDv85yZO+7WapC5/XDNLPgmEYoTqav5C5dy/yiFQFlEMqww\nAzE03MIgopAIswhL1ixFklIyssjF6JctoXgj0H58RawlNV3bdb16d2fmzjmnP845M8+cOffeuePc\nec7s83rB5Xme7/Prc+99mDPznu/z/e5m+9LoQPuDX93O9qVRdi6NTqaeQWaB0ZLgqR06rc+2N9rb\nVw6hHnVumLWhAAoAYBkBEQCw55abb8otN990zeeNx5NZeLSb7VagtLUzyvalUba3D27Pj9vaucw5\nsxDq/q9Mg6rd0fhE/n5NM9jv/XS1EGph+3OfvZiHms9lOBxkbdhMl810OZwt99qHTYbNtH1tOMhw\nuL+ctg8EVQDAShEQAQBft6YZ5IaNtdywcX2+tRiNxnu9l5YFSu1QaavV22l7Z5St7YXtVgh14eJ2\ntnZGGY0nRyvknz9+Yn+nYdMKjubrzZIwabG9ORhCzUOppe0L57Wvuxdc7YVcB+/dbj8YfC20N4M0\n8z+DgTGqAOCMEhABACtvOGxy47DJjY9avy7Xv7Q7znbrFbqt7Vbvp1mg9KlPfyaP/45bMhpPsjua\nZDQeT5ej8ayttZztH42Wt8/P2x3vnz8a7bdvX2pfd3/9rGiHRcMmGQz2t9v7mmaQ4WCQpsmh9sFg\nvq99Tq5ynUEGg8PXOnydhfUl5yyuDxdqHAza+xZqbAa5977tfMO9DxwK65aFd02rTc8yALoiIAIA\nem99rcn62rl84xWOeXTuy+bmk06tpkWTyWQaJC2ESfOwqh1CHQqpxofbjxJy7Y7GGS8Nvw5eazye\nZDyZZDzObDn9M5qtT+Zte/uyt293PD5wzt76JAe2z6S/+dI1n9I0g70eYwd7iC3pRdY0++HSQm+v\n9jHt85f3Ltvvqda0eqwd5f7t+zYL9/daJcDZIiACADgDBoPpa2FrwyTrw67LOXUHA6b99dFC8DSZ\nXG7fweMOtR86LgdCroPh1SSTvX05tG88nuSzn/t8Hve4m5cGa+3QbT/Aa20v9CqbB2nbO7t5aKFX\n2Wg0zlnJz+Y9rZYuB9Nwqt3b62rHD4fNbLnQk6tZvj5smjSDXPa8yy2bZlbX/Pirntdcpt52z7r9\n+s5S70DgkU1ABADAymuaQZoMkjOSjZ0/fzGbm993Kveah2GLry0eCKOW9PzaC6favdIOBFOtHmqz\nY8ejyaF7LL32+HDQNQ2zWuHd4nIyyfbu9JjRaB62jTMaJ+O9nmqn8k966tbe9/nWTIzzGRoPDqR/\nYDmfwbHdvnFwkP0bNtaycW4tQ+OCAUckIAIAgDNs3hNlfa3pupTrbrIsYGoFSnvLeaB0hWPHo4M9\nxKbnZO/co90jS++12PvscCC2f+6XH/xKzm3cuDd4/sWt3TxwYStbO6NMTiAQW19rLh8qXSl02mgF\nVufWWvtnMz+uDw1KD48wAiIAAOBMGMxe1RqekZ5kR3H+/Plsbm4eap9MJtnZHe8Nmj+fwfHh7d2D\nszhedv/8mP3ZHi9c3MnWlx/O9s7oRGqf93Cah0o37PV6WgydDvdyam8vnnNurTFmFXRAQAQAALBi\nBoNBNtaH2Vgf5jEnfO3xeDKbtfFqodN0JsfDodP+cj7T45cvbOV/d0a5tDv+uutrBsn6+nA2wPl8\nUPT2IOqXGax9Yd+hAdRb+xdnEBwu2X+g/YoDvl/tnvPjZ2Na6XnFihIQAQAA9EjTDHLDxlpu2Dj5\nHwdHo3G2Ly3pybR9MFR6eHs/XDocOo2yfWk0G/Pq4NhWWzuj2RhX7XGyztZsh4NBloZPRwnBvva1\nr+bPPnbPse55rFpz7BNP7bRz68O84vYn5wnf+ujj3ZQ9AiIAAABOxHDY5MZhkxsftX6q953PIrg/\nOPv+AOvzwdIXB0+fzxw4GrdmDFzYvxhEjQ7tn20vXHN/UPfDNSyvY/9+W7ujpffby8C+uH2q/7ar\nbtgM8sIffIKA6AQIiAAAADjT5jMdrg0fuYO1j8eTnD9/Pk9bMmbVFR1ztPPj9sk6/uDqxztxMHhk\n/7+fJgERAAAArLj5jIXXPoaRMY84GjEbAAAAQM8JiAAAAAB6TkAEAAAA0HMCIgAAAICeExABAAAA\n9JyACAAAAKDnBEQAAAAAPScgAgAAAOg5AREAAABAzwmIAAAAAHpOQAQAAADQcwIiAAAAgJ4TEAEA\nAAD0nIAIAAAAoOcERAAAAAA9JyACAAAA6DkBEQAAAEDPCYgAAAAAek5ABAAAANBzAiIAAACAnhMQ\nAQAAAPScgAgAAACg5wREAAAAAD0nIAIAAADoOQERAAAAQM8JiAAAAAB6TkAEAAAA0HMCIgAAAICe\nExABAAAA9JyACAAAAKDnBEQAAAAAPScgAgAAAOg5AREAAABAzwmIAAAAAHpOQAQAAADQcwIiAAAA\ngJ4TEAEAAAD0nIAIAAAAoOcERAAAAAA9JyACAAAA6DkBEQAAAEDPCYgAAAAAek5ABAAAANBzAiIA\nAACAnhMQAQAAAPTc2klerJTyzCQ/k2SS5LW11gsneX0AAAAATt5J9yD66dmfP0jyshO+NgAAAADX\nwZF7EJVSnp7kzbXW55dSBkneluQpSbaSvKrW+pkkTa11p5TyhSQvuC4VAwAAAHCijtSDqJTyhiR3\nJdmYNb04yUat9dYkdyZ5y6z9oVLKuSTfluQLJ1wrAAAAANfBUV8x+3SSl7S2n53kQ0lSa/1oks1Z\n+11J3p7pa2bvOaEaAQAAALiOBpPJ5EgHllK+M8l7a623llLuSvL+Wuvds33/neRJtdbxtdz8/Pnz\nR7s5AAAAAEe2ubk5uJbjjzuL2YUkN7W2m2sNh5JrLxYAAACAk3fcWczuSXJ7kpRSnpHkEydWEQAA\nAACn6rg9iD6Q5LZSyj2z7VeeUD0AAAAAnLIjj0EEAAAAwCPTcV8xAwAAAOARQkAEAAAA0HPHHYPo\n61JKGSR5W5KnJNlK8qpa62e6qIXVVkpZS/LOJE9Mci7Jr9VaP9hpUay8Usrjknw8yQtrrZ/quh5W\nUynljUl+NMl6krfVWv+w45JYUbPPondl+lm0m+SnfG1hUSnl6UneXGt9finlu5P8UZJxkk/WWl/T\naXGsnIXn5alJfifTry/bSV5Ra/1SpwWyMtrPSqvtjiQ/V2u9tbvKWEULX1u+JcldSb4pyTDTry3/\ndaXzu+pB9OIkG7MH+s4kb+moDlbfy5P8X631OZnOnPe7HdfDipv9IPf7SR7quhZWVynluUmeOfsc\nel6SW7qtiBV3e5JhrfVZSX41ya93XA8rppTyhky/Cd+YNb0lyS/WWp+bpCml/FhnxbFyljwvb03y\nmlrrCzKdDOiNXdXGalnyrGQWKP5kZ0WxspY8L7+R5D211ucleVOS773aNboKiJ6d5ENJUmv9aJIf\n6KgOVt+fZvowJ8kgyaUOa+Fs+M0kv5fk810Xwkr74SSfLKX8eZK/SPKXHdfDavtUkrVZD+jHJNnp\nuB5Wz6eTvKS1vVlr/chs/a+TvPD0S2KFLT4vL621fmK2vpbk4dMviRV14FkppXxzpr+keG1nFbHK\nFr+2PCvJ40spH05yR5K/v9oFugqIHp3kK63t3VKK8ZA4pNb6UK31YinlpiTvS/JLXdfE6iql/ESS\n+2qtH840UITLeWySzSQ/nuTVSf6423JYcV9L8l1J/jPJ2zN9FQT21Fo/kOnrQXPtz6CvZhosQpLD\nz0ut9YtJUkq5NclrkvxWR6WxYtrPyuzn5XckeV2Si/G9LguWfBY9MckDtdbbkvxPjtA7satQ5kKS\nm9p11FrHHdXCiiul3JLkb5O8q9b6J13Xw0p7ZZLbSil/l+SpSd49G48IFt2f5O5a6+5sLJmtUspj\nuy6KlfW6JB+qtZZMx098dynlXMc1sdra39felOTBrgrhbCilvDTTMVpvr7Xe33U9rKSnJfmeTHvK\nvzfJk0sphmrhSu5PMh+/94OZ/nL0iroKiO7J9H3+lFKekeQTVz6cviql3Jzk7iS/UGt9V9f1sNpq\nrc+ttT5/Nojfv2Y6ENt9XdfFSvqnJD+SJKWUb09yY6YforDMA9nv+fxgpq+ADLsrhzPgX0opz5mt\nvyjJR650MP1WSnl5pj2HnldrvbfrelhJg1rrx2ut3z8bq+plSf691vrzXRfGSvtIZrlLkuck+ber\nndDJLGaZDr52Wynlntn2Kzuqg9V3Z6ajrr+plPLLSSZJXlRr3e62LM6ASdcFsLpqrX9VSvmhUsrH\nMu2i/bO1Vs8Ml/PWJO8spfxjprPe3VlrNUYIV/L6JHeVUtaT/EeS93dcDytq9trQbye5N8kHSimT\nJP9Qa/2VbitjxfgeheN4fZJ3lFJenekvuu642gmDycSzBgAAANBnBoYGAAAA6DkBEQAAAEDPCYgA\nAAAAek5ABAAAANBzAiIAAACAnhMQAQAAAPScgAgAAACg5wREAAAAAD33/7VZWDvcdkIeAAAAAElF\nTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11f7f5590>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(20, 10))\n",
    "plt.semilogy(losses)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Restore model to make new predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--> сентябрь вы сегодня прекрасный новое сходу осознать что жизнь мочь не быть так что не хотеть\n",
      "--> готовиться к о это очень высокий позитивный и эмоция в нежный\n",
      "--> вчера спокойный пятница не далеко не держаться нахрен выступить соло подобрать место и проблема\n",
      "--> дева недовольный пока не сдохнуть предыдущий отдых как выделываться и\n",
      "--> весы большой не хватать машина деньга на близнец для львов если это человек где ручка и быть\n"
     ]
    }
   ],
   "source": [
    "with tf.Session(graph=graph) as session:\n",
    "    loader = tf.train.import_meta_graph('./Model2/model.meta')\n",
    "    loader.restore(session, './Model2/model')\n",
    "    \n",
    "    ops = tf.get_collection('ops')\n",
    "    reset_state = ops[0]\n",
    "    reset_sample_state = ops[1]\n",
    "    sample_prediction = ops[2]\n",
    "    \n",
    "    for _ in range(5):\n",
    "        sym = word_to_index['<$>']\n",
    "        reset_sample_state.run(session=session)\n",
    "        iter_num = 0\n",
    "        tweet = []\n",
    "        while sym != word_to_index['<#>'] and iter_num < 40:\n",
    "            prediction = sample_prediction.eval({sample_input: np.array([sym])}, session)[0]\n",
    "            prediction[word_to_index['<#>']] *= ((1 + iter_num) / (1 + 10)) ** 0.005\n",
    "            prediction = sample(prediction)\n",
    "            if prediction == word_to_index['<$>']:\n",
    "                break\n",
    "            tweet.append(index_to_word[prediction])\n",
    "            sym = prediction\n",
    "            iter_num += 1\n",
    "        print '--> %s' % ' '.join(tweet[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}