BTP_latest.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BTP_latest.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python2",
      "display_name": "Python 2"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "FqoGKzRZGCXP",
        "colab_type": "code",
        "outputId": "4fda7c41-1dd4-4d41-843a-6efe9085203d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 63
        }
      },
      "source": [
        "import sys\n",
        "reload(sys)  # need to reload to set default encoding\n",
        "sys.setdefaultencoding('utf8')\n",
        "\n",
        "import argparse\n",
        "import collections\n",
        "import json\n",
        "import logging\n",
        "import numpy as np\n",
        "import os\n",
        "import shutil\n",
        "import tensorflow as tf\n",
        "import pandas as pd\n",
        "import itertools\n",
        "import re\n",
        "import pickle\n",
        "import random"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kXzfY9K5Ger2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import matplotlib\n",
        "matplotlib.rc('font', family='DejaVu Sans')\n",
        "from matplotlib import pyplot\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AXk6au8xGjEm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# logging\n",
        "# output_vocab.Save(output_vocab_filename)\n",
        "logging.basicConfig(filename='logfile.log',filemode='w', level=logging.DEBUG)\n",
        "logger = logging.getLogger()\n",
        "logger.setLevel(logging.DEBUG)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C4Z2NNaSGlpP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "config = tf.ConfigProto(inter_op_parallelism_threads=10, intra_op_parallelism_threads=10)\n",
        "tf.set_random_seed(666)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "76ImxssuGpJS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "f = open('./train.txt', 'r')\n",
        "f1 = f.readlines()\n",
        "sentences = []\n",
        "words = []\n",
        "labels = []\n",
        "i = 0\n",
        "line = f1[0]\n",
        "for line in f1:\n",
        "# for i in xrange(2):\n",
        "#   line = f1[i]\n",
        "  sentences.append(line)\n",
        "  w1 = line.split()\n",
        "  word = [] \n",
        "  label = []\n",
        "  for w in w1:\n",
        "    w2 = w.split('/')\n",
        "#     if(w2[1]=='amb' or w2[1]=='ne'):\n",
        "#       continue\n",
        "    \n",
        "  \n",
        "    word.append(w2[0])   \n",
        "    label.append(w2[1])    \n",
        "  words.append(word)  \n",
        "  labels.append(label)  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NZtMrHSVG5Pc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# line\n",
        "# labels[1]\n",
        "# # words[1]\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3zk6hSRZG7mX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_IDs = []\n",
        "eval_IDs = []\n",
        "train_Lang = []\n",
        "eval_Lang = []\n",
        "train_Words_unicode = []\n",
        "eval_Words_unicode = []\n",
        "tweetID = 0\n",
        "for row in words:\n",
        "  if tweetID < 0.8 * len(words):\n",
        "    train_IDs.append(tweetID)\n",
        "    train_Lang.append(labels[tweetID])\n",
        "    #train_Words_unicode.append([unicode(x_.decode('utf8')) for x_ in row])\n",
        "    train_Words_unicode.append(['{'] + [unicode(x_.decode('utf8')) for x_ in row] + ['}'])\n",
        "    \n",
        "  else:\n",
        "    eval_IDs.append(tweetID)\n",
        "    eval_Lang.append(labels[tweetID])\n",
        "#     eval_Words_unicode.append([unicode(x_.decode('utf8')) for x_ in row])\n",
        "    eval_Words_unicode.append(['{'] + [unicode(x_.decode('utf8')) for x_ in row] + ['}'])\n",
        "  tweetID = tweetID + 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4CA5bPnPHCkv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# batch_size = 1\n",
        "batch_size = 25\n",
        "und_symbol = 'und'\n",
        "weight = 1.0\n",
        "maxlen = 40\n",
        "\n",
        "dataset_weights = []\n",
        "dataset_weights.append(weight)\n",
        "\n",
        "################## vocab ##################################\n",
        "\n",
        "class Vocab(object):\n",
        "\n",
        "  def _normalize(self, word):\n",
        "    if re.match(ur'^<.*>$', word):\n",
        "      return word\n",
        "\n",
        "    if self.specialcase:\n",
        "      newword = []\n",
        "      prev_is_lower = True\n",
        "      for letter in word:\n",
        "        if letter.isupper():\n",
        "          if prev_is_lower:\n",
        "            newword.append('~')\n",
        "          prev_is_lower = False\n",
        "        else:\n",
        "          prev_is_lower = True\n",
        "        newword.append(letter.lower())\n",
        "      word = ''.join(newword)\n",
        "\n",
        "    if self.lowercase:\n",
        "      word = word.lower()\n",
        "\n",
        "    if self.numreplace:\n",
        "      word = re.sub(ur'\\d', '#', word)\n",
        "\n",
        "    return word\n",
        "\n",
        "  def __init__(self, tokenset, unk_symbol='<UNK>',\n",
        "               lowercase=False, numreplace=False):\n",
        "    self.lowercase = lowercase\n",
        "    self.numreplace = numreplace\n",
        "\n",
        "    self.specialcase = False\n",
        "    \n",
        "    \n",
        "    tokenset = set([self._normalize(w) for w in tokenset])   #########\n",
        "\n",
        "    self.vocab_size = len(tokenset)\n",
        "    self.unk_symbol = unk_symbol\n",
        "\n",
        "    self.word_to_idx = dict(zip((tokenset), range(self.vocab_size)))\n",
        "    self.idx_to_word = dict(zip(self.word_to_idx.values(),\n",
        "                                self.word_to_idx.keys()))\n",
        "    self.t = tokenset\n",
        "  @staticmethod\n",
        "  def Load(filename):\n",
        "    with open(filename, 'rb') as f:\n",
        "      v = pickle.load(f)\n",
        "    return v\n",
        "\n",
        "  @classmethod\n",
        "  def MakeFromData(cls, lines, min_count, unk_symbol='<UNK>',\n",
        "                   max_length=None, no_special_syms=False, normalize=False):\n",
        "    lowercase=False\n",
        "    numreplace=False\n",
        "\n",
        "    if normalize:\n",
        "      lowercase=True\n",
        "      numreplace=True\n",
        "\n",
        "    token_counts = collections.Counter()\n",
        "\n",
        "    for line in lines:\n",
        "      token_counts.update(line)\n",
        "\n",
        "    tokenset = set()\n",
        "    for word in token_counts:\n",
        "      if max_length and len(word) > max_length:      #############\n",
        "        continue\n",
        "      if token_counts[word] >= min_count:\n",
        "        tokenset.add(word)\n",
        "\n",
        "    if not no_special_syms:\n",
        "      tokenset.add(unk_symbol)\n",
        "      tokenset.add('<S>')\n",
        "      tokenset.add('</S>')\n",
        "\n",
        "    return cls(tokenset, unk_symbol=unk_symbol, lowercase=lowercase,\n",
        "               numreplace=numreplace)\n",
        "\n",
        "  @classmethod\n",
        "  def ByteVocab(cls):\n",
        "    \"\"\"Creates a vocab that has a token for each possible byte.\n",
        "\n",
        "    It's useful to have a fixed byte vocab so that the subset of bytes\n",
        "    that form the vocab is not dependent on the dataset being used. Thus,\n",
        "    the learned byte embeddings can be reused on different datasets.\n",
        "    \"\"\"\n",
        "    c = '0123456789abcdef'\n",
        "    tokens = ['<S>', '</S>']\n",
        "    for i in c:\n",
        "      for j in c:\n",
        "        tokens.append(i + j)\n",
        "    return cls(tokens)\n",
        "\n",
        "  @classmethod\n",
        "  def LoadFromTextFile(cls, filename, unk_symbol='<UNK>'):\n",
        "    tokens = []\n",
        "    with open(filename, 'r') as f:\n",
        "      for line in f:\n",
        "        line = line.strip()\n",
        "        tokens.append(line)\n",
        "    return cls(tokens, unk_symbol=unk_symbol)\n",
        "\n",
        "  def GetWords(self):\n",
        "    \"\"\"Get a list of words in the vocabulary.\"\"\"\n",
        "    return self.word_to_idx.keys()\n",
        "\n",
        "  def LookupIdx(self, token):\n",
        "    token = self._normalize(token)\n",
        "    if token in self.word_to_idx:\n",
        "      return self.word_to_idx[token]\n",
        "    return self.word_to_idx.get(self.unk_symbol, None)\n",
        "\n",
        "  def __contains__(self, key):\n",
        "    key = self._normalize(key)\n",
        "    return key in self.word_to_idx\n",
        "\n",
        "  def __getitem__(self, key):\n",
        "    \"\"\"If key is an int lookup word by id, if key is a word then lookup id.\"\"\"\n",
        "    if type(key) == int or type(key) == np.int64:\n",
        "      return self.idx_to_word[key]\n",
        "\n",
        "    return self.LookupIdx(key)\n",
        "\n",
        "  def __iter__(self):\n",
        "    word_list = [self.idx_to_word[x] for x in xrange(self.vocab_size)]\n",
        "    return word_list.__iter__()\n",
        "\n",
        "  def __len__(self):\n",
        "    return self.vocab_size\n",
        "\n",
        "  def Save(self, filename):\n",
        "    if filename.endswith('.pickle'):\n",
        "     with open(filename, 'wb') as f:\n",
        "      pickle.dump(self, f)\n",
        "    elif filename.endswith('.txt'):\n",
        "      with open(filename, 'w') as f:\n",
        "        for i in range(self.vocab_size):\n",
        "          f.write('{0}\\n'.format(self.idx_to_word[i]))\n",
        "\n",
        "\n",
        "    \n",
        "###########################################################################\n",
        "\n",
        "\n",
        "input_vocab = Vocab.MakeFromData(train_Words_unicode, max_length=maxlen, min_count=1)\n",
        "test_input_vocab = Vocab.MakeFromData(eval_Words_unicode, max_length=maxlen, min_count=1)\n",
        "# input_vocab = Vocab.MakeFromData(itertools.chain(train_Words_unicode), max_length=maxlen, min_count=1)\n",
        "\n",
        "  \n",
        "def Graphemes(s):\n",
        "  graphemes = []\n",
        "  current = []\n",
        "\n",
        "  if type(s) == unicode:\n",
        "    s = s.encode('utf8')\n",
        "\n",
        "  for c in s:\n",
        "    val = ord(c) & 0xC0\n",
        "    if val == 128:\n",
        "      # this is a continuation\n",
        "      current.append(c)\n",
        "    else:\n",
        "      # this is a new grapheme\n",
        "      if len(current) > 0:\n",
        "        graphemes.append(''.join(current))\n",
        "        current = []\n",
        "\n",
        "      if val < 128:\n",
        "        graphemes.append(c)  # single byte grapheme\n",
        "      else:\n",
        "        current.append(c)  # multi-byte grapheme\n",
        "\n",
        "  if len(current) > 0:\n",
        "    graphemes.append(''.join(current))\n",
        "\n",
        "  return graphemes\n",
        "\n",
        "  \n",
        "###################  training ####################\n",
        "\n",
        "# x = [util.Graphemes(w) for w in input_vocab.GetWords()]\n",
        "x = [Graphemes(w) for w in input_vocab.GetWords()]      #########\n",
        "char_vocab = Vocab.MakeFromData(x, min_count=2)         #######\n",
        "char_vocab.Save('./sample_data/char_vocab.pickle')      ########\n",
        "\n",
        "train_label_set = set()\n",
        "for d in train_Lang:\n",
        "  train_label_set.update(d)\n",
        "\n",
        "eval_label_set = set()\n",
        "for d in eval_Lang:\n",
        "  eval_label_set.update(d)\n",
        "\n",
        "output_vocab_filename = './sample_data/out_vocab.pickle'\n",
        "labels = [[x1] for x1 in train_label_set]\n",
        "if [und_symbol] not in labels: labels += [[und_symbol]]\n",
        "output_vocab = Vocab.MakeFromData(labels, min_count=1, no_special_syms=True)\n",
        "output_vocab.Save(output_vocab_filename)\n",
        "\n",
        "test_output_vocab_filename = './sample_data/test_out_vocab.pickle'\n",
        "eval_labels = [[x1] for x1 in eval_label_set]\n",
        "if [und_symbol] not in eval_labels: eval_labels += [[und_symbol]]\n",
        "test_output_vocab = Vocab.MakeFromData(eval_labels, min_count=1, no_special_syms=True)\n",
        "test_output_vocab.Save(test_output_vocab_filename)\n",
        "\n",
        "\n",
        "# with open(args.params, 'r') as f:\n",
        "#   model_params = json.load(f)\n",
        "\n",
        "# with open(os.path.join(args.expdir, 'model_params.json'), 'w') as f:\n",
        "#   json.dump(model_params, f)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uQlS32_5HNio",
        "colab_type": "code",
        "outputId": "32d3339d-a2dd-4dac-a3ba-07c5b9931ca1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "input_vocab.vocab_size\n",
        "\n",
        "train_label_set\n",
        "labels"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['oth'], ['en'], ['bn'], ['amb'], ['ne'], ['as'], ['und']]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DMtCpZ-4HPSS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "######################## Dataset ################################\n",
        "counts=0\n",
        "class Dataset(object):\n",
        "\n",
        "  def __init__(self, batch_size, preshuffle=False, name='unnamed'):\n",
        "    \"\"\"Init the dataset object.\n",
        "\n",
        "    Args:\n",
        "      batch_size: size of mini-batch\n",
        "      preshuffle: should the order be scrambled before the first epoch\n",
        "      name: optional name for the dataset\n",
        "    \"\"\"\n",
        "    self._sentences = []\n",
        "    self._labels = []\n",
        "    self._ids = []\n",
        "    self.dataset_weights = []\n",
        "    self._lines = []\n",
        "    self.name = name\n",
        "\n",
        "    self.batch_size = batch_size\n",
        "    self.preshuffle = preshuffle\n",
        "\n",
        "  def ReadData(self, filename, mode, modeltype, weight=1.0):\n",
        "    d = LoadData(filename, mode, modeltype)\n",
        "    self.AddDataSource(d, weight=weight)\n",
        "\n",
        "  def AddDataSource(self, data, weight=1.0):\n",
        "    sentences, labels, ids = data\n",
        "    self._sentences.append(sentences)\n",
        "    self._labels.append(labels)\n",
        "    self.dataset_weights.append(weight)\n",
        "    self._ids.append(ids)\n",
        "\n",
        "  def GetSentences(self):\n",
        "    return itertools.chain(*self._sentences)\n",
        "\n",
        "  def GetIds(self):\n",
        "    return [x for x in itertools.chain(*self._ids)]\n",
        "\n",
        "  def GetLabelSet(self):\n",
        "    label_set = set()\n",
        "    for d in self._labels:\n",
        "      label_set.update([x for x in itertools.chain(*d)])\n",
        "    return label_set\n",
        "\n",
        "  def Prepare(self,in_vocab,out_vocab,und_label='und',ignore_categories=[]):\n",
        "    # Add a dummy dataset to make the batch size evenly divide the number\n",
        "    # of sentences.\n",
        "    batch_size = self.batch_size\n",
        "    total_sentences = sum([len(x) for x in self._sentences])\n",
        "    r = total_sentences % batch_size\n",
        "    if r > 0:\n",
        "      n = batch_size - r\n",
        "      self.AddDataSource(([list('{dummy}')] * n, [[und_label]] * n, [0] * n),\n",
        "                         weight=0.0)\n",
        "\n",
        "    sentences = list(itertools.chain(*self._sentences))\n",
        "    labels = list(itertools.chain(*self._labels))\n",
        "\n",
        "#     self.example_weights = []\n",
        "#     for i in xrange(len(self.dataset_weights)):\n",
        "#       w = self.dataset_weights[i]\n",
        "#       for j in xrange(len(self._sentences[i])):\n",
        "#         ex_wt = []\n",
        "#         for _ in xrange(len(self._sentences[i][j])):\n",
        "#           ex_wt.append(w)\n",
        "#         self.example_weights.append(ex_wt)\n",
        "#     self.example_weights = np.array(self.example_weights)\n",
        "\n",
        "    self.example_weights = []\n",
        "    for i in xrange(len(self.dataset_weights)):\n",
        "      w = self.dataset_weights[i]\n",
        "      for _ in xrange(len(self._sentences[i])):\n",
        "        self.example_weights.append(w)\n",
        "    self.example_weights = np.array(self.example_weights)\n",
        "\n",
        "\n",
        "    self.seq_lens = np.array([len(x) for x in sentences])\n",
        "#     self.max_sequence_len = self.seq_lens.max()\n",
        "    self.max_sequence_len = 0\n",
        "    for x in self.seq_lens:\n",
        "      if x > self.max_sequence_len:\n",
        "        self.max_sequence_len = x\n",
        "\n",
        "    self.batch_size = batch_size\n",
        "    self.current_idx = 0\n",
        "    self.max_sequence_len = 2453               #############################\n",
        "    self.sentences = self.GetNumberLines(sentences, in_vocab,\n",
        "                                         self.max_sequence_len)\n",
        "    self.labels = np.zeros((len(labels), self.max_sequence_len, len(out_vocab)))\n",
        "#     label_size = 4\n",
        "    label_size = 7\n",
        "    for i, w in enumerate(labels):\n",
        "      self.labels[i,0,label_size-1] = 1.0\n",
        "      z = 1\n",
        "      for w_ in w:\n",
        "        print w_ +\" \"+ str(i) +\" \"+str(z)+\" \"+str(out_vocab[w_])\n",
        "        self.labels[i,z, out_vocab[w_]] = 1.0\n",
        "        z=z+1\n",
        "      while (z<2453):\n",
        "        self.labels[i,z,label_size-1] = 1.0\n",
        "        z=z+1\n",
        "#       self.labels[i, :] /= self.labels[i, :].sum()\n",
        "#     for i, w in enumerate(labels):\n",
        "#       for w_ in w:\n",
        "#         self.labels[i, out_vocab[w_]] = 1.0\n",
        "#       self.labels[i, :] /= self.labels[i, :].sum()\n",
        "    # class weights\n",
        "    # There is a hack to only use the examples with weight 1 as a way\n",
        "    # to prevent wikipedia from dominating the weights.\n",
        "    \n",
        "#     counts = self.labels[np.sum(self.example_weights, axis = 1) == 1, :].sum(axis=0)\n",
        "    counts = self.labels[self.example_weights == 1, :].sum(axis=0)\n",
        "    self.w = 1.0/(1.0 + counts)\n",
        "    self.w /= self.w.mean()  # scale the class weights to reasonable values\n",
        "#     self.w = 1.0\n",
        "    \n",
        "    self.N = len(sentences)\n",
        "    if self.preshuffle:\n",
        "      self._Permute()\n",
        "\n",
        "  @staticmethod\n",
        "  def GetNumberLines(lines, vocab, pad_length):\n",
        "    \"\"\"Convert list of words to matrix of word ids.\"\"\"\n",
        "    out = []\n",
        "    for line in lines:\n",
        "      if len(line) < pad_length:\n",
        "        line += ['}'] * (pad_length - len(line))       #########\n",
        "      out.append([vocab[w] for w in line])            #########\n",
        "    return np.array(out)                            #######\n",
        "\n",
        "  def GetNumBatches(self):\n",
        "    \"\"\"Returns num batches per epoch.\"\"\"\n",
        "    return self.N / self.batch_size\n",
        "\n",
        "  def _Permute(self):\n",
        "    \"\"\"Shuffle the training data.\"\"\"\n",
        "    s = np.arange(self.N)\n",
        "    np.random.shuffle(s)\n",
        "\n",
        "    self.sentences = self.sentences[s, :]\n",
        "    self.seq_lens = self.seq_lens[s]\n",
        "    self.labels = self.labels[s, :]\n",
        "    self.example_weights = self.example_weights[s]\n",
        "\n",
        "  def GetNextBatch(self):\n",
        "    if self.current_idx + self.batch_size > self.N:\n",
        "      self.current_idx = 0\n",
        "#       self._Permute()\n",
        "\n",
        "    idx = range(self.current_idx, self.current_idx + self.batch_size)\n",
        "    self.current_idx += self.batch_size\n",
        "\n",
        "    return (self.sentences[idx, :], self.seq_lens[idx],\n",
        "            self.labels[idx, :], self.example_weights[idx])\n",
        "\n",
        "    #########################################################################"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8nlpXIf9Hyug",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dataset = Dataset(batch_size)\n",
        "test_dataset = Dataset(batch_size)\n",
        "\n",
        "train_data = [train_Words_unicode, train_Lang, train_IDs]\n",
        "test_data = [eval_Words_unicode, eval_Lang, eval_IDs]\n",
        "\n",
        "dataset.AddDataSource(train_data)\n",
        "dataset.Prepare(input_vocab, output_vocab, und_symbol)\n",
        "\n",
        "test_dataset.AddDataSource(test_data)\n",
        "test_dataset.Prepare(test_input_vocab, test_output_vocab, und_symbol)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u5RyFmd_yBpZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# x\n",
        "# char_vocab.word_to_idx"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "acA5D5xDH4H0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Char2Vec(object):\n",
        "  \"\"\"Maps character sequences to word embeddings.\"\"\"\n",
        "\n",
        "  def __init__(self, char_vocab, max_sequence_len=15, initializer = 0):\n",
        "    \"\"\"Initialize the Char2Vec model.\n",
        "\n",
        "    Args:\n",
        "      char_vocab: vocab class instance for the character vocabulary\n",
        "      max_sequence_len: length of the longest word\n",
        "    \"\"\"\n",
        "    self.max_sequence_len = max_sequence_len\n",
        "    self.char_vocab = char_vocab\n",
        "    self._vocab_size = char_vocab.vocab_size\n",
        "\n",
        "    # Placeholder for the character sequences in the form of an\n",
        "    # n x k array where n is the number of words and k is the\n",
        "    # length of the longest word. Characters are encoded as ints.\n",
        "    self.words_as_chars = tf.placeholder(tf.int32, [None, max_sequence_len],\n",
        "                                         name='words_as_chars')\n",
        "#     self.words_as_chars = tf.Variable(tf.zeros([None, max_sequence_len]), name='words_as_chars', dtype=tf.int32, initializer=initializer)\n",
        "#     self.words_as_chars = tf.get_variable(\"words_as_chars\",\n",
        "#                                      [None, max_sequence_len],\n",
        "#                                      initializer=initializer, dtype=tf.int32)\n",
        "\n",
        "  def GetEmbeddings(self, x):\n",
        "    return tf.nn.embedding_lookup(self.word_embeddings, x)\n",
        "\n",
        "  def MakeMat(self, word_list, pad_len=None):\n",
        "    \"\"\"Make a matrix to hold the character sequences in.\n",
        "\n",
        "    Special start and end tokens are added to the beggining and end of\n",
        "    each word.\n",
        "\n",
        "    Args:\n",
        "      word_list: A list of strings\n",
        "      pad_len: Pad all character sequences to this length. If a word is\n",
        "               longer than the pad_len it will be truncated.\n",
        "\n",
        "    Returns:\n",
        "      Array containing character sequences and a vector of sequence lengths.\n",
        "    \"\"\"\n",
        "    if not pad_len:\n",
        "      pad_len = self.max_sequence_len\n",
        "\n",
        "    # make the padded char mat\n",
        "    the_words = []\n",
        "    word_lengths = []\n",
        "    for word in word_list:\n",
        "      word_idx = [self.char_vocab[c] for c in Graphemes(word)]\n",
        "      word_idx = ([self.char_vocab['<S>']] + word_idx[:pad_len-2] +\n",
        "                  [self.char_vocab['</S>']])\n",
        "      if len(word_idx) < pad_len:\n",
        "        word_idx += [self.char_vocab['</S>']] * (pad_len - len(word_idx))\n",
        "      the_words.append(word_idx)\n",
        "      word_lengths.append(min(pad_len, len(word)+2))\n",
        "\n",
        "    the_words = np.array(the_words)\n",
        "    word_lengths = np.array(word_lengths)\n",
        "    return the_words, word_lengths\n",
        "\n",
        "  @staticmethod\n",
        "  def GetBatchVocab(words):\n",
        "    batch_vocab = np.unique(words)\n",
        "    words_remapped = np.copy(words)\n",
        "    for i in xrange(len(batch_vocab)):\n",
        "      np.place(words_remapped, words==batch_vocab[i], i)\n",
        "    return batch_vocab, words_remapped\n",
        "\n",
        "class CharCNN(Char2Vec):\n",
        "  \"\"\"CNN implementation of char2vec.\n",
        "\n",
        "  The model uses two layers of convolution. The second one is followed by a\n",
        "  max pooling operation. After that there is a resnet layer.\n",
        "  \"\"\"\n",
        "\n",
        "  def __init__(self, char_vocab,\n",
        "               max_sequence_len=15, dropout_keep_prob=None, initializer = 0):\n",
        "    super(CharCNN, self).__init__(char_vocab, max_sequence_len, initializer = initializer)\n",
        "\n",
        "    char_embed_dims = int(np.log(len(char_vocab))) + 1\n",
        "\n",
        "    layer1_out_size = 60\n",
        "    hidden_size = 70\n",
        "    word_embed_dims = 80\n",
        "\n",
        "    # The following variables define the model.\n",
        "    with tf.variable_scope('c2v', reuse=tf.AUTO_REUSE):\n",
        "      \n",
        "      # continuous space character embeddings\n",
        "      self.embedding = tf.get_variable(\"embedding\",\n",
        "                                       [self._vocab_size, char_embed_dims],\n",
        "                                       initializer=initializer)\n",
        "      the_filter, filter_b = MakeFilter(3, char_embed_dims, layer1_out_size,\n",
        "                                        'filt')\n",
        "\n",
        "      # z is a tensor of dimensions batch_sz x word_len x embed_dims.\n",
        "      z = tf.nn.embedding_lookup(self.embedding, self.words_as_chars)\n",
        "      z_expanded = tf.expand_dims(z, -1)\n",
        "\n",
        "      conv = tf.nn.conv2d(z_expanded, the_filter, strides=[1, 1, 1, 1],\n",
        "                          padding='VALID' )\n",
        "      h = tf.nn.relu(tf.nn.bias_add(tf.squeeze(conv), filter_b))\n",
        "      h.set_shape((None, max_sequence_len - 2, layer1_out_size))\n",
        "      if dropout_keep_prob is not None:\n",
        "        h_expanded = tf.nn.dropout(tf.expand_dims(h, -1), dropout_keep_prob)\n",
        "      else:\n",
        "        h_expanded = tf.expand_dims(h, -1)\n",
        "\n",
        "      pools = []\n",
        "      filter_sizes = range(3,6)\n",
        "      for width in filter_sizes:\n",
        "        f, f_bias = MakeFilter(width, layer1_out_size, hidden_size,\n",
        "                               'filter_w{0}'.format(width))\n",
        "        conv2 = tf.nn.conv2d(h_expanded, f, strides=[1, 1, 1, 1],\n",
        "                             padding='VALID')\n",
        "        h2 = tf.nn.relu(tf.nn.bias_add(conv2, f_bias))\n",
        "        pooled = tf.nn.max_pool(h2, ksize=[1, max_sequence_len-1-width, 1, 1],\n",
        "                                strides=[1, 1, 1, 1], padding='VALID')\n",
        "        pools.append(pooled)\n",
        "\n",
        "        if width == 3:  # debugging\n",
        "          self.hh = tf.squeeze(pooled)\n",
        "          self.hidx = tf.argmax(h2, 1)\n",
        "\n",
        "      pooled = tf.squeeze(tf.concat(pools,3), [])\n",
        "\n",
        "      # resnet layer https://arxiv.org/abs/1512.03385\n",
        "      sz = len(filter_sizes) * hidden_size\n",
        "      t_mat = tf.get_variable('t_mat', [sz, sz])\n",
        "      t_bias = tf.Variable(tf.constant(0.1, shape=[sz]), name='t_bias')\n",
        "      t = tf.nn.relu(tf.matmul(pooled, t_mat) + t_bias)\n",
        "\n",
        " \n",
        "      self.word_embeddings = t + pooled\n",
        "      self.embedding_dims = sz\n",
        "\n",
        "#       self.word_embeddings = tf.placeholder(tf.float32, [None, max_sequence_len],\n",
        "#                                          name='word_embeddings')\n",
        "\n",
        "def MakeFilter(width, in_size, num_filters, name):\n",
        "  filter_sz = [width, in_size, 1, num_filters]\n",
        "  filter_b = tf.Variable(tf.constant(0.1, shape=[num_filters]),\n",
        "                         name='{0}_bias'.format(name))\n",
        "  the_filter = tf.get_variable(name, filter_sz)\n",
        "  return the_filter, filter_b\n",
        "\n",
        "\n",
        "def reverse_seq(input_seq, lengths):\n",
        "  \"\"\"Reverse a list of Tensors up to specified lengths.\n",
        "  Args:\n",
        "    input_seq: Sequence of seq_len tensors of dimension (batch_size, depth)\n",
        "    lengths:   A tensor of dimension batch_size, containing lengths for each\n",
        "               sequence in the batch. If \"None\" is specified, simply reverses\n",
        "               the list.\n",
        "  Returns:\n",
        "    time-reversed sequence\n",
        "  \"\"\"\n",
        "  for input_ in input_seq:\n",
        "    input_.set_shape(input_.get_shape().with_rank(2))\n",
        "\n",
        "  # Join into (time, batch_size, depth)\n",
        "  s_joined = tf.pack(input_seq)\n",
        "\n",
        "  # Reverse along dimension 0\n",
        "  s_reversed = tf.reverse_sequence(s_joined, lengths, 0, 1)\n",
        "  # Split again into list\n",
        "  result = tf.unpack(s_reversed)\n",
        "  return result\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "umdAWaCsH5BV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "initializer = tf.random_uniform_initializer(-0.1, 0.1)\n",
        "\n",
        "max_word_len = max([len(x) for x in input_vocab.GetWords()]) + 2\n",
        "# with tf.variable_scope(tf.get_variable_scope(), reuse=True):\n",
        "c2v = CharCNN(char_vocab,max_sequence_len=max_word_len,dropout_keep_prob=0.7, initializer = initializer)\n",
        "the_words, word_lengths = c2v.MakeMat(input_vocab, pad_len=max_word_len)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iJNTD5opVF7C",
        "colab_type": "code",
        "outputId": "09d223e2-b183-42c1-d7ab-e61f56954601",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "c2v.embedding_dims"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "210"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gmZsbjv4z-Vk",
        "colab_type": "code",
        "outputId": "51799c48-baee-42c1-82e1-6b4d66db82c1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# c2v.word_embeddings.shape\n",
        "c2v.words_as_chars"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor 'words_as_chars:0' shape=(?, 24) dtype=int32>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wcif23pYH9z4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class WordLevelModel(object):\n",
        "  \"\"\"\n",
        "    Model to evaluate on word-level predictions\n",
        "\n",
        "    Args:\n",
        "      batch_size: minibatch size\n",
        "      model_params: dictionary of other model parameters\n",
        "      c2v: char2vec class instance\n",
        "      max_sequence_len: length of all the input/output sequences\n",
        "      out_vocab_size: how many languages we are predicting\n",
        "      dropout_keep_prob: dropout probability indicator\n",
        "      weights: class weights\n",
        "  \"\"\"\n",
        "\n",
        "  def __init__(self, batch_size, c2v, max_sequence_len,\n",
        "               out_vocab_size, dropout_keep_prob=0.0, weights=None):\n",
        "    self._batch_size = batch_size\n",
        "    self._dropout_keep_prob = dropout_keep_prob\n",
        "    self._out_vocab_size = out_vocab_size\n",
        "\n",
        "    self.x = tf.placeholder(tf.int32, [batch_size, max_sequence_len],\n",
        "                            name='x')\n",
        "    self.y = tf.placeholder(tf.float32,\n",
        "                            [batch_size, max_sequence_len, out_vocab_size],\n",
        "                            name='y')\n",
        "    # The bidirectional rnn code requires seq_lens as int64\n",
        "    self.seq_lens = tf.placeholder(tf.int64, [batch_size], name='seq_lens')\n",
        "#     self.example_weights = tf.placeholder(tf.float32, [batch_size, max_sequence_len],\n",
        "#                                          name='example_weights')\n",
        "#     self.example_weights = tf.placeholder(tf.float32, [batch_size],\n",
        "#                                          name='example_weights')\n",
        "\n",
        "    embeddings = c2v.GetEmbeddings(self.x)        ##before\n",
        "    self.temp = embeddings\n",
        "#     embeddings = tf.reshape(embeddings, [batch_size, max_sequence_len, 50])    ###########\n",
        "    embeddings = tf.reshape(embeddings, [batch_size, max_sequence_len, 210])\n",
        "#     tf.squeeze(embeddings)\n",
        "    self._inputs = embeddings\n",
        "  \n",
        "    # Need to prepare a mask to zero out the padding symbols.\n",
        "\n",
        "    # Make a batch_size x max_sequence_len matrix where each\n",
        "    # row contains the length repeated max_sequence_len times.\n",
        "    lengths_transposed = tf.expand_dims(self.seq_lens, 1)\n",
        "#     lengths_transposed = tf.expand_dims(tf.to_int64(self.seq_lens), 1)\n",
        "    lengths_tiled = tf.tile(lengths_transposed, [1, max_sequence_len])\n",
        "\n",
        "    # Make a matrix where each row contains [0, 1, ..., max_sequence_len]\n",
        "    r = tf.range(0, max_sequence_len, 1)\n",
        "    range_row = tf.expand_dims(r, 0)\n",
        "    range_tiled = tf.tile(range_row, [batch_size, 1])\n",
        "\n",
        "    self.lengths_transposed = lengths_transposed\n",
        "    self.lengths_tiled = lengths_tiled\n",
        "    self.range_row = range_row\n",
        "    self.range_tiled = range_tiled\n",
        "\n",
        "    # Use the logical operations to create a mask\n",
        "    indicator = tf.less(range_tiled, tf.to_int32(lengths_tiled+1)) #i.e. where seq len is less than index\n",
        "    trim = np.ones(indicator.get_shape())\n",
        "    trim[:,0] = 0 #ignore start symbol\n",
        "    indicator = tf.logical_and(indicator, trim.astype(bool))\n",
        "    self.indicator = indicator\n",
        "\n",
        "    sz = [batch_size, max_sequence_len]\n",
        "    self._mask = tf.where(indicator, tf.ones(sz), tf.zeros(sz))\n",
        "\n",
        "    #-------------------------------#\n",
        "\n",
        "    self.weights = tf.constant(weights, dtype=tf.float32, name='class_weights')\n",
        "\n",
        "    hidden_size = 100\n",
        "#     hidden_size = 40\n",
        "    proj_size = None\n",
        "    def GetCell():\n",
        "      \"\"\"Creates an LSTM cell with dropout.\"\"\"\n",
        "      c = tf.nn.rnn_cell.LSTMCell(hidden_size,\n",
        "                                  use_peepholes=False,\n",
        "                                  num_proj=proj_size)\n",
        "      if dropout_keep_prob is not None:\n",
        "        c = tf.nn.rnn_cell.DropoutWrapper(c, input_keep_prob=dropout_keep_prob)\n",
        "      return c\n",
        "\n",
        "    # Create the bi-directional LSTM\n",
        "    with tf.variable_scope('wordrnn', reuse=tf.AUTO_REUSE):\n",
        "      with tf.variable_scope('fw', reuse=tf.AUTO_REUSE):\n",
        "        cell_fw = GetCell()\n",
        "      with tf.variable_scope('bw', reuse=tf.AUTO_REUSE):\n",
        "        cell_bw = GetCell()\n",
        "        \n",
        "\n",
        "      rnnout, _ = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_bw, self._inputs,\n",
        "                                             dtype=tf.float32,\n",
        "                                             sequence_length=self.seq_lens)\n",
        "\n",
        "      if proj_size:\n",
        "        out_size = 1 * proj_size\n",
        "#         out_size = 2 * proj_size\n",
        "      else:\n",
        "        out_size = 1 * hidden_size\n",
        "#         out_size = 2 * hidden_size\n",
        "      self._DoPredictions(out_size, rnnout, self.weights)\n",
        "\n",
        "      self.cost = tf.reduce_mean(self._xent)\n",
        "#       self.cost = tf.reduce_mean(tf.multiply(self._xent,tf.expand_dims(self._mask,2)))\n",
        "#       self.cost = tf.reduce_mean(self.example_weights * self._xent)\n",
        "\n",
        "  def _DoPredictions(self, in_size, mats, class_weights=None):\n",
        "    \"\"\"Takes in an array of states and calculates predictions.\n",
        "\n",
        "    Get the cross-entropy for each example in the vector self._xent.\n",
        "\n",
        "    Args:\n",
        "      in_size: size of the hidden state vectors\n",
        "      mats: list of hidden state vectors\n",
        "    \"\"\"\n",
        "    pred_mat = tf.get_variable('pred_mat0',\n",
        "                               [in_size, self._out_vocab_size])\n",
        "    pred_bias = tf.get_variable('pred_bias', [self._out_vocab_size])\n",
        "\n",
        "    # Make a prediction on every word.\n",
        "    def GetWordPred(o_):\n",
        "      logits = tf.nn.xw_plus_b(o_, pred_mat, pred_bias)\n",
        "      return tf.nn.softmax(logits)\n",
        "\n",
        "    #self.preds_by_word1 = tf.pack([GetWordPred(o_) for o_ in mats])\n",
        "    #self.preds_by_word = tf.reshape(self.preds_by_word1, self.y.get_shape())\n",
        "    #self.probs = tf.mul(tf.expand_dims(self._mask,2), self.preds_by_word)\n",
        "\n",
        "#     self.preds_by_word = tf.pack([GetWordPred(o_) for o_ in mats])\n",
        "#     self.preds_by_instance = tf.pack([self.preds_by_word[:,i,:] for i in range(self.preds_by_word.get_shape()[1])])\n",
        "#     self.probs = tf.mul(tf.expand_dims(self._mask,2), self.preds_by_instance)\n",
        "\n",
        "    self.preds_by_word = tf.stack([GetWordPred(o_) for o_ in mats])\n",
        "    self.preds_by_instance = tf.stack([self.preds_by_word[:,i,:] for i in range(self.preds_by_word.get_shape()[1])])\n",
        "    self.probs1 = tf.multiply(tf.expand_dims(self._mask,2), tf.reduce_sum(self.preds_by_instance,1))\n",
        "    self.prob_div = tf.reduce_sum(self.probs1,2)\n",
        "    self.probs = self.probs1/tf.expand_dims(self.prob_div,-1)\n",
        "#     self.probs = tf.multiply(tf.expand_dims(tf.expand_dims(self._mask,1),3), tf.reduce_sum(self.preds_by_instance,2))\n",
        "#     self.probs = tf.multiply(tf.expand_dims(self._mask,2), self.preds_by_word)\n",
        "#     self.probs = tf.multiply(tf.expand_dims(self._mask,2), self.preds_by_instance)\n",
        "\n",
        "#     self._xent = _SafeXEnt(self.y, self.probs, class_weights=class_weights, sumd=[2])\n",
        "    self._xent, self.adjprob = _SafeXEnt(self.y, self.probs1, class_weights=class_weights, sumd=[1,2])\n",
        "\n",
        "\n",
        "def _SafeXEnt(y, probs, eps=0.0001, class_weights=None, sumd=[1]):\n",
        "  \"\"\"Version of cross entropy loss that should not produce NaNs.\n",
        "\n",
        "  If the predicted proability for the true class is near zero then when\n",
        "  taking the log it can produce a NaN, which ruins everything. This\n",
        "  function ensures each probability is at least eps and no more than one\n",
        "  before taking the log.\n",
        "\n",
        "  Args:\n",
        "    y: matrix of true probabilities same size as probs\n",
        "    probs: matrix of probabilities for the minibatch\n",
        "    eps: value to clip the probabilities at\n",
        "    class_weights: vector of relative weights to be assigned to each class\n",
        "    sumd: dimensions along which to sum the x-ent matrix\n",
        "\n",
        "  Returns:\n",
        "    cross entropy loss for each example in the minibatch\n",
        "  \"\"\"\n",
        "#   eps = 0\n",
        "  adjusted_probs = tf.clip_by_value(probs, eps, 1.0 - eps)\n",
        "  xent_mat = -y * tf.log(adjusted_probs)\n",
        "  if class_weights is not None:\n",
        "    xent_mat *= class_weights\n",
        "\n",
        "  return xent_mat, adjusted_probs\n",
        "#   return tf.reduce_sum(xent_mat, sumd)\n",
        "\n",
        "\n",
        "def _SafeNegEntropy(probs, batch_size, eps=0.0001):\n",
        "  \"\"\"Computes negative entropy in a way that will not overflow.\"\"\"\n",
        "  adjusted_probs = tf.clip_by_value(probs, eps, 1.0 - eps)\n",
        "  entropy = tf.multiply(probs, tf.log(adjusted_probs))\n",
        "  return tf.reduce_sum(entropy) / batch_size\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iVLrPvk_IXIQ",
        "colab_type": "code",
        "outputId": "9ae4cf3b-38d9-489d-9a5f-7ca53e87e301",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "model = WordLevelModel(batch_size=batch_size,\n",
        "                           max_sequence_len=dataset.max_sequence_len,\n",
        "                           weights=dataset.w,\n",
        "                           dropout_keep_prob=0.7,\n",
        "                           out_vocab_size=len(output_vocab), c2v=c2v)\n",
        "# model = WordLevelModel(batch_size=batch_size,\n",
        "#                            max_sequence_len=dataset.max_sequence_len,\n",
        "#                            dropout_keep_prob=0.7,\n",
        "#                            out_vocab_size=len(output_vocab), c2v=c2v)\n",
        "saver = tf.train.Saver(tf.all_variables())\n",
        "session = tf.Session(config=config)\n",
        "model.preds_by_word\n",
        "model.preds_by_instance\n",
        "model._xent\n",
        "model.prob_div\n",
        "model.probs1\n",
        "# tf.expand_dims(tf.expand_dims(model._mask,1),3)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor 'wordrnn/Mul:0' shape=(25, 2453, 7) dtype=float32>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pKThy6cCIe_X",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "scores = []\n",
        "# confusion_matrix = [] \n",
        "def Metrics(preds, labs, show=True):\n",
        "  \"\"\"Print precision, recall and F1 for each language.\n",
        "  Assumes a single language per example, i.e. no code switching.\n",
        "  Args:\n",
        "    preds: list of predictions\n",
        "    labs: list of labels\n",
        "    show: flag to toggle printing\n",
        "  \"\"\"\n",
        "  all_langs = set(preds + labs)\n",
        "  preds = np.array(preds)\n",
        "  labs = np.array(labs)\n",
        "\n",
        "  label_totals = collections.Counter(labs)\n",
        "  pred_totals = collections.Counter(preds)\n",
        "  confusion_matrix = collections.Counter(zip(preds, labs))\n",
        "\n",
        "  num_correct = 0\n",
        "  for lang in all_langs:\n",
        "    num_correct += confusion_matrix[(lang, lang)]\n",
        "  acc = num_correct / float(len(preds))\n",
        "  print 'accuracy = {0:.3f}'.format(acc)\n",
        "\n",
        "  if show:\n",
        "    print ' Lang     Prec.   Rec.   F1'\n",
        "    print '------------------------------'\n",
        "\n",
        "#   scores = []\n",
        "  fmt_str = '  {0:6}  {1:6.2f} {2:6.2f} {3:6.2f}'\n",
        "  for lang in sorted(all_langs):\n",
        "    idx = preds == lang\n",
        "    total = max(1.0, pred_totals[lang])\n",
        "    precision = 100.0 * confusion_matrix[(lang, lang)] / total\n",
        "\n",
        "    idx = labs == lang\n",
        "    total = max(1.0, label_totals[lang])\n",
        "    recall = 100.0 * confusion_matrix[(lang, lang)] / total\n",
        "\n",
        "    if precision + recall == 0.0:\n",
        "      f1 = 0.0\n",
        "    else:\n",
        "      f1 = 2.0 * precision * recall / (precision + recall)\n",
        "\n",
        "    scores.append([precision, recall, f1])\n",
        "    if show:\n",
        "      print fmt_str.format(lang, precision, recall, f1)\n",
        "\n",
        "  totals = np.array(scores).mean(axis=0)\n",
        "  if show:\n",
        "    print '------------------------------'\n",
        "    print fmt_str.format('Total:', totals[0], totals[1], totals[2])\n",
        "  return totals[2]\n",
        "\n",
        "\n",
        "def ConfusionMat(preds, labs):\n",
        "  \"\"\"Plot and show a confusion matrix.\n",
        "  Args:\n",
        "    preds: list of predicted labels\n",
        "    labs: list of true labels\n",
        "  \"\"\"\n",
        "  all_langs = set(preds + labs)  # this is the set of all possible labels\n",
        "  num_langs = len(all_langs)\n",
        "\n",
        "  # create a mapping from labels to id numbers\n",
        "  lookup = dict(zip(sorted(all_langs), range(num_langs)))\n",
        "\n",
        "  # make the counts for the confusion matrix\n",
        "  counts = np.zeros((num_langs, num_langs))\n",
        "  for p, l in zip(preds, labs):\n",
        "    counts[lookup[p], lookup[l]] += 1\n",
        "\n",
        "  # plot a colormap using log scale\n",
        "  pyplot.imshow(np.log(counts+1.0), interpolation='none')\n",
        "\n",
        "  # plot the text labels\n",
        "  for i in xrange(num_langs):\n",
        "    for j in xrange(num_langs):\n",
        "      pyplot.text(i, j, str(int(counts[i, j])), color='white',\n",
        "                  horizontalalignment='center')\n",
        "  # take care of the axes\n",
        "  pyplot.xticks(range(num_langs), sorted(all_langs))\n",
        "  pyplot.yticks(range(num_langs), sorted(all_langs))\n",
        "  pyplot.xlabel('Prediction')\n",
        "  pyplot.ylabel('True Label')\n",
        "  pyplot.show()\n",
        "  return 0\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XlHo9AaeIhXh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def MakeFeedDict(words, seqlen, labs, ws=None):\n",
        "  \"\"\"Create the feed dict to process each batch.\n",
        "\n",
        "  All the inputs should be from the GetNextBatch command.\n",
        "\n",
        "  Args:\n",
        "    words: matrix of word ids\n",
        "    seqlen: vector of sequence lengths\n",
        "    labs: target matrix\n",
        "    ws: per-example weights\n",
        "\n",
        "  Returns:\n",
        "    dictionary to be used as feed dict.\n",
        "  \"\"\"\n",
        "  batch_data = {\n",
        "    model.seq_lens: seqlen,\n",
        "    model.x: words,\n",
        "  }\n",
        "\n",
        "#   if mode == 'train':\n",
        "  batch_data[model.y] = labs\n",
        "#   batch_data[model.example_weights] = ws\n",
        "#   batch_data[dropout_keep_prob] = 0.7\n",
        "\n",
        "#   if not baseline:\n",
        "  batch_vocab, words_remapped = Char2Vec.GetBatchVocab(words)      ######\n",
        "\n",
        "  batch_data.update({\n",
        "    c2v.words_as_chars: the_words[batch_vocab, :],\n",
        "    model.x: words_remapped\n",
        "  })\n",
        "\n",
        "  if hasattr(c2v, 'seq_lens'):\n",
        "    batch_data.update({\n",
        "      c2v.seq_lens: word_lengths[batch_vocab],\n",
        "      c2v.batch_dim: len(batch_vocab)\n",
        "      })                                                     ################\n",
        "\n",
        "  return batch_data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uzqo1YOvIobT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "###################### train ############################################\n",
        "\n",
        "dropout_keep_prob = tf.placeholder_with_default(1.0, (), name='keep_prob')\n",
        "\n",
        "# logging.info('Input Vocab Size: {0}'.format(len(input_vocab)))\n",
        "# logger.info('Char Vocab Size: {0}'.format(len(char_vocab)))\n",
        "\n",
        "tvars = tf.trainable_variables()\n",
        "grads, _ = tf.clip_by_global_norm(tf.gradients(model.cost, tvars), 5.0)\n",
        "optimizer = tf.train.AdamOptimizer(0.001)\n",
        "train_op = optimizer.apply_gradients(zip(grads, tvars))\n",
        "\n",
        "session.run(tf.initialize_all_variables())\n",
        "# if args.start:\n",
        "#   saver.restore(session, os.path.join(args.start, 'model.bin'))\n",
        "# util.PrintParams(tf.trainable_variables(), handle=logging.info)\n",
        "\n",
        "# maxitr = model_params.get('num_training_iters', 80001)\n",
        "# maxitr = 10001\n",
        "maxitr = 101\n",
        "print \"Training for {} iterations...\".format(maxitr)\n",
        "for idx in xrange(maxitr): \n",
        "#   if args.data == 'codeswitch':\n",
        "#     words, seqlen, labs, ws, lines = dataset.GetNextBatch()\n",
        "#   else:\n",
        "#   words = dataset.sentences[[0],:]\n",
        "#   seqlen = dataset.seq_lens[[0]]\n",
        "#   labs = dataset.labels[[0],:]\n",
        "#   ws = dataset.example_weights[[0]]\n",
        "#   words = dataset.GetNextBatch()\n",
        "  words, seqlen, labs, ws = dataset.GetNextBatch()\n",
        "  batch_data = MakeFeedDict(words, seqlen, labs, ws)\n",
        "\n",
        "#   if idx % 25 == 0:\n",
        "  print idx\n",
        "#     probs = session.run([model.probs], batch_data)[0]\n",
        "#     s = [input_vocab[i] for i in words[0, :seqlen[0]]]\n",
        "#     print ' '.join(s)\n",
        "#     print \"Predict:\", GetTopWordLevelPreds(probs[0,:,:], seqlen[0])\n",
        "#     print \"Actual: \", GetTopWordLevelPreds(labs[0,:,:], seqlen[0])\n",
        "\n",
        "  cost, inpu, xx, tmp, xnt, ap, pw, pi, p1, pd, p, m, we2, _ = session.run([model.cost, model._inputs, model.x, model.temp, model._xent, model.adjprob, model.preds_by_word, model.preds_by_instance, model.probs1, model.prob_div, model.probs, model._mask, c2v.word_embeddings, train_op], batch_data)\n",
        "#   cost, _ = session.run([model.cost, train_op], batch_data)\n",
        "#   preds_by_word, inst = session.run([model.preds_by_word, model.preds_by_instance], batch_data)\n",
        "  logger.info({'iter': idx, 'cost': float(cost)})\n",
        "  if idx == maxitr-1 :\n",
        "    saver.save(session, './model.bin')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gO4UvXEWhpq1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# totals"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bIFiEPWrryiU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "saver.restore(session,  'model.bin')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OmmvyRBhIziZ",
        "colab_type": "code",
        "outputId": "05729d5b-eed1-4d1b-ac1a-801b5810f770",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "\n",
        "len(tmp[0][0])      # 1 * 2453 * 210\n",
        "cost\n",
        "xnt[0][24]\n",
        "ap\n",
        "len(pw[0][0][0])\n",
        "len(pi[0][0][0])\n",
        "len(p1[0][0])\n",
        "len(pd[0])\n",
        "# len(p[0])\n",
        "len(ap[0][0])\n",
        "len(m[0])\n",
        "\n",
        "ap[0][51]\n",
        "# p1[0][2]\n",
        "ap\n",
        "# len(p)\n",
        "p1[0][1]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.14389476, 1.0629226 , 0.12079805, 0.0932117 , 0.32992396,\n",
              "       0.2056115 , 0.04363749], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bhFC8KYfI14k",
        "colab_type": "code",
        "outputId": "eae5b352-1790-40d3-cc48-6f2a96ea574d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "################################ eval ###########################################3\n",
        "\n",
        "# model_vars=[]\n",
        "# def Eval():\n",
        "  \"\"\"Evaluates on dev data.\n",
        "  Writes results to a results.tsv file in the expdir for use in the \n",
        "  scoring script.\n",
        "  Args:\n",
        "    expdir: path to experiment directory\n",
        "  \"\"\" \n",
        "\n",
        "# saver.restore(session,  'model.bin')\n",
        "\n",
        "all_preds, all_labs = [], []\n",
        "pred, lab = [], []\n",
        "for _ in xrange(test_dataset.GetNumBatches()):\n",
        "#  words  = dataset.GetNextBatch()\n",
        "#  words = dataset.sentences[[0],:]\n",
        "#  seqlen = dataset.seq_lens[[0]]\n",
        "#  labs = dataset.labels[[0],:]\n",
        "#  weights = dataset.example_weights[[0]]\n",
        " words, seqlen, labs, weights  = test_dataset.GetNextBatch()\n",
        " batch_data = MakeFeedDict(words, seqlen, labs, weights)\n",
        "#  batch_data = MakeFeedDict(words, seqlen, labs)\n",
        "#  all_words += [words]\n",
        "\n",
        " model_vars = [model.probs, model.preds_by_word]\n",
        " probs, pp = session.run(model_vars, batch_data)\n",
        "#  for i in xrange(len(labs)):\n",
        "#     print labs[i],  probs[i], i\n",
        "#  prob_mat=np.argmax(probs, axis=2)\n",
        "#  lab_mat = np.argmax(labs, axis=2)\n",
        "#  for p in np.argmax(probs[0], axis=1):\n",
        "#     print str(p) + \" . \" + output_vocab[p] + \" ... \"\n",
        "#  for p in np.argmax(labs[0], axis=1):\n",
        "#     print str(p) + \" - \" + output_vocab[p] + \" ||| \"\n",
        " for i in xrange (batch_size):\n",
        "#     for p in np.argmax(probs[i], axis=1):\n",
        "#       out_voc = output_vocab[p]\n",
        "#       if (output_vocab[p]=='und'):\n",
        "#         out_voca\n",
        "    for j in xrange(2453):\n",
        "      p=output_vocab[np.argmax(labs[i][j])]\n",
        "      if(p=='und'):\n",
        "        continue\n",
        "      \n",
        "      all_preds+=[output_vocab[np.argmax(probs[i][j])]]\n",
        "      all_labs+=[p]\n",
        "#     print lab\n",
        "#     all_preds = pred\n",
        "#     all_labs = lab\n",
        "    \n",
        "  \n",
        "Metrics(all_preds, all_labs)\n",
        "# print all_labs\n",
        "# print all_preds\n",
        "\n",
        "# This output file is in the format needed to score for TweetLID\n",
        "ids = test_dataset.GetIds()\n",
        "with open('results.tsv', 'w') as f:\n",
        " for p, l in zip(all_preds, all_labs):\n",
        "     f.write('{0}\\t{1}\\n'.format(p,l))\n",
        "    \n",
        "totals = np.array(scores).mean(axis=0)\n",
        "totals\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([11.58665896, 16.66666667, 13.66996474])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "desSRXDEJHnY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# words, seqlen, labs, weights  = dataset.GetNextBatch()\n",
        "# batch_data = MakeFeedDict(words, seqlen, labs, weights)\n",
        "# inp, xx = tf.Session().run([model._inputs, model.x], batch_data)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bxWjUNTqJKKh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "inp = inpu\n",
        "\n",
        "with open('inputs.tsv', 'w') as f:\n",
        " w = 0\n",
        " for word in inp[0]:\n",
        "   for p in inp[0][w]:\n",
        "       f.write('{0}\\t'.format(p))\n",
        "   f.write('\\n')\n",
        "   w = w+1\n",
        "\n",
        "with open('we.tsv', 'w') as f:\n",
        " for sentence in we2:\n",
        "   for w in sentence:\n",
        "       f.write('{0}\\t'.format(w))\n",
        "   f.write('\\n')\n",
        "\n",
        "with open('x.tsv', 'w') as f:\n",
        " for sentence in xx[0]:\n",
        "   f.write('{0}\\t'.format(sentence))\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dtLRwUR0MVIh",
        "colab_type": "code",
        "outputId": "4b78e33b-8352-438e-fee6-edd8e1d93998",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "input_vocab['i']\n",
        "# input_vocab['think']\n",
        "# input_vocab['know']\n",
        "# input_vocab['it']\n",
        "# input_vocab['well']\n",
        "# input_vocab['{']\n",
        "input_vocab[42]\n",
        "input_vocab[0]\n",
        "input_vocab[5]\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "u'eligible'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mAJU9NetNhMT",
        "colab_type": "code",
        "outputId": "486683cc-d073-4c2c-d811-373d3580785c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "c2v.word_embeddings"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor 'c2v/add_1:0' shape=<unknown> dtype=float32>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SB3PdYY6Tzu7",
        "colab_type": "code",
        "outputId": "02fe020c-7387-46fd-992d-52a76be7d0d9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# inputs - 2453 * 210\n",
        "# embedd - 46   * 210\n",
        "len(input_vocab.word_to_idx)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "9031"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E_MlH0HQc596",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}