jupyterBengio.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nfrom google.colab import drive\\ndrive.mount('/content/gdrive')\\nimport os\\nos.chdir('/content/gdrive/My Drive/bengio')\\n\""
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# uncomment after you've opened this through Google Colab\n",
    "\"\"\"\n",
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive')\n",
    "import os\n",
    "os.chdir('/content/gdrive/My Drive/bengio')\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import math\n",
    "import numpy as np\n",
    "import re\n",
    "import sys\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import Counter\n",
    "\n",
    "# helper functions\n",
    "\n",
    "\n",
    "# reads the file as a list of words\n",
    "def read_file(path_to_file):\n",
    "    with open(path_to_file, \"r\") as text:\n",
    "        ls_words = text.read().replace(\"\\n\", \"<eos>\").split()\n",
    "    # Clean the vocab from random characters within the corpora\n",
    "    regex = re.compile(r'[.a-zA-Z0-9]')\n",
    "    if a3 == 'wiki':\n",
    "        return [i.lower() for i in ls_words if (regex.search(i) or i == '<eos>')]\n",
    "    else:\n",
    "        return [i for i in ls_words if (regex.search(i) or i == '<eos>')]\n",
    "\n",
    "\n",
    "# makes batches of data for later use\n",
    "def make_batches(data, batch_size, window_size):\n",
    "    x_data = []\n",
    "    y_data = []\n",
    "    for i in range(len(data)):\n",
    "        if i > window_size - 1:\n",
    "            x_data.append(data[i - window_size:i])\n",
    "            y_data.append(data[i])\n",
    "    batches = int(len(x_data) / batch_size)\n",
    "    batch_out = list()\n",
    "    for i in range(batches):\n",
    "        # For each batch\n",
    "        start_i = batch_size * i\n",
    "        end_i = start_i + batch_size\n",
    "        x_values = x_data[start_i:end_i]\n",
    "        y_values = y_data[start_i:end_i]\n",
    "        batch_out.append([x_values, y_values])\n",
    "    return batch_out\n",
    "\n",
    "\n",
    "# brown corpus is huge, splits according to Bengio layout in paper\n",
    "def split_brown():\n",
    "    with open('data/brown.txt') as file:\n",
    "        text_list = file.read().split()\n",
    "    training = ' '.join(text_list[:800000])\n",
    "    training_file = open(\"data/brown.train.txt\", \"w\")\n",
    "    training_file.write(training)\n",
    "    training_file.close()\n",
    "\n",
    "    validation = ' '.join(text_list[800000:1000000])\n",
    "    validation_file = open(\"data/brown.valid.txt\", \"w\")\n",
    "    validation_file.write(validation)\n",
    "    validation_file.close()\n",
    "\n",
    "    testing = ' '.join(text_list[1000000:])\n",
    "    testing_file = open(\"data/brown.test.txt\", \"w\")\n",
    "    testing_file.write(testing)\n",
    "    testing_file.close()\n",
    "\n",
    "\n",
    "# plots the learning curve in plt\n",
    "def plot_learning(accuracy, cost):\n",
    "    loss = [1 - x for x in accuracy]\n",
    "    figure = plt.figure(figsize=(10, 6))\n",
    "    x = np.arange(0, np.shape(cost)[0])\n",
    "    plt.subplot(2, 1, 1)\n",
    "    plt.plot(x, cost, color='red')\n",
    "    plt.title('Validation Cost')\n",
    "    plt.subplot(2, 1, 2)\n",
    "    plt.plot(x, accuracy, c='b')\n",
    "    plt.title('Validation Accuracy')\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "class Preprocessor:\n",
    "\n",
    "    def __init__(self, path):\n",
    "        raw_list = read_file(path)\n",
    "        top_words = Counter(raw_list).most_common()\n",
    "        words = [word[0] for word in top_words if word[1] >= 3]\n",
    "        if '<unk>' in words:\n",
    "            words.remove('<unk>')\n",
    "        self.word_dict = {'<unk>': 0}\n",
    "        for i in range(1, len(words)):\n",
    "            self.word_dict[words[i]] = i\n",
    "        self.vocab_size = len(self.word_dict)\n",
    "        self.word_dict_reverse = dict(zip(self.word_dict.values(), self.word_dict.keys()))\n",
    "        self.text_as_index = []\n",
    "        for word in words:\n",
    "            idx = 0\n",
    "            if word in self.word_dict:\n",
    "                idx = self.word_dict[word]\n",
    "            self.text_as_index.append(idx)\n",
    "\n",
    "    def generate_data(self, path):\n",
    "        words = read_file(path)\n",
    "        text_as_index = []\n",
    "        for word in words:\n",
    "            idx = 0\n",
    "            if word in self.word_dict:\n",
    "                idx = self.word_dict[word]\n",
    "            text_as_index.append(idx)\n",
    "        return text_as_index\n",
    "\n",
    "\n",
    "class BengioModel:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.batch_size = 256\n",
    "        self.embedding_size = user_input['embedding_size']\n",
    "        self.window_size = user_input['window_size']\n",
    "        self.hidden_units = user_input['hidden_units']\n",
    "\n",
    "    def train_model(self, training_data, validation_data, num_epochs=5):\n",
    "        if tf.test.is_gpu_available(cuda_only=False, min_cuda_compute_capability=None):\n",
    "            device = '/gpu:0'\n",
    "            print('Currently using GPU device to run code')\n",
    "        # need to figure out how to run this on a TPU\n",
    "        else:\n",
    "            device = '/cpu:0'\n",
    "            print('There is no GPU available, using CPU device to run code')\n",
    "        with tf.device(device):\n",
    "            # all code that I want to run on GPU insert here\n",
    "            self.x_input = tf.placeholder(tf.int64, [None, self.window_size])\n",
    "            self.y_input = tf.placeholder(tf.int64, [None])\n",
    "\n",
    "            z = self.embedding_size * self.window_size\n",
    "\n",
    "            # hidden layer biases\n",
    "            d = tf.Variable(tf.random_uniform([self.hidden_units]))\n",
    "            # output biases\n",
    "            b = tf.Variable(tf.random_uniform([vocab_size]))\n",
    "\n",
    "            # weights\n",
    "            # C matrix function\n",
    "            word_embeddings = tf.Variable(tf.random_uniform([vocab_size,\n",
    "                                                             self.embedding_size],\n",
    "                                                            -1.0,\n",
    "                                                            1.0))\n",
    "            flattened_exes = tf.layers.flatten(self.x_input)\n",
    "            lookup = tf.nn.embedding_lookup(word_embeddings, flattened_exes)\n",
    "            xt = tf.reshape(lookup, [self.batch_size, z])\n",
    "\n",
    "            # H Weight\n",
    "            H = tf.Variable(tf.truncated_normal([z, self.hidden_units],\n",
    "                                                stddev=1.0 / math.sqrt(z)))\n",
    "            # W Weight\n",
    "            W = tf.Variable(tf.truncated_normal([z, vocab_size],\n",
    "                                                stddev=1.0 / math.sqrt(z)))\n",
    "            # U Weight\n",
    "            U = tf.Variable(tf.truncated_normal([self.hidden_units, vocab_size],\n",
    "                                                stddev=1.0 / vocab_size))\n",
    "\n",
    "            # hidden layers\n",
    "            tanh = tf.nn.tanh(tf.nn.bias_add(tf.matmul(xt, H), d))\n",
    "            y = tf.nn.bias_add(tf.matmul(xt, W), b) + tf.matmul(tanh, U)\n",
    "\n",
    "            # softmax\n",
    "            y_probability_distribution = tf.nn.softmax(y)\n",
    "            y_ideal = tf.argmax(y_probability_distribution, axis=1)\n",
    "            # produces labels to use in the softmax_cross_entropy_with_logits\n",
    "            y_labels = tf.one_hot(self.y_input, vocab_size)\n",
    "\n",
    "            self.ce_result = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_labels,\n",
    "                                                                                       logits=y_probability_distribution))\n",
    "\n",
    "            # stochastic gradient descent optimizer\n",
    "            learning_rate = 0.001\n",
    "            beta1 = 0.9\n",
    "            beta2 = 0.999\n",
    "            adam = tf.train.AdamOptimizer(learning_rate, beta1, beta2).minimize(self.ce_result)\n",
    "            ra = tf.equal(y_ideal, self.y_input)\n",
    "            self.accuracy = tf.reduce_mean(tf.cast(ra, tf.float32))\n",
    "\n",
    "            self.session = tf.Session()\n",
    "            self.session.run(tf.initializers.global_variables())\n",
    "            saver = tf.train.Saver()\n",
    "\n",
    "            print('training beginning . . . . .')\n",
    "            global training_accuracy, training_cost\n",
    "            for i in range(num_epochs):\n",
    "                batches = make_batches(training_data,\n",
    "                                       self.batch_size,\n",
    "                                       self.window_size)\n",
    "                total_batches = len(batches)\n",
    "                batch_count = 0\n",
    "                last_complete = 0\n",
    "                num_messages = 10  # the number of  printouts  per  epoch\n",
    "                for batch in batches:\n",
    "                    batch_count += 1\n",
    "                    x_batch = batch[0]\n",
    "                    y_batch = batch[1]\n",
    "                    feed_dict_train = {self.x_input: x_batch,\n",
    "                                       self.y_input: y_batch}\n",
    "                    self.session.run(adam, feed_dict=feed_dict_train)\n",
    "                    completion = 100 * batch_count / total_batches\n",
    "                    if batch_count % (int(total_batches / num_messages)) == 0:\n",
    "                        print('Epoch #%2d-   Batch #%5d:   %4.2f %% completed.' % (i + 1, batch_count, completion))\n",
    "                        a_t, c_t = self.test(training_data)\n",
    "                        a, c = self.test(validation_data)\n",
    "                        training_accuracy.append(a)\n",
    "                        training_cost.append(c)\n",
    "\n",
    "                        if sum(training_cost[-4:]) > sum(training_cost[-8:-4]):\n",
    "                            patience = patience - 1\n",
    "                        else:\n",
    "                            patience = 2\n",
    "\n",
    "                        if patience == 0:\n",
    "                            print(\"Cost Too High, Early Stop Activated\")\n",
    "                            save_path = saver.save(self.session, \"../models/\" + a2 + '_' + a3 + \".ckpt\")\n",
    "                            print(\"Model saved in path: %s\" % save_path)\n",
    "                            return\n",
    "\n",
    "        print(\"Training is finished\")\n",
    "        save_path = saver.save(self.session, \"../models/\" + a2 + '_' + a3 + \".ckpt\")\n",
    "        print(\"Model saved in path: %s\" % save_path)\n",
    "        return\n",
    "\n",
    "    def restore_model(self, path):\n",
    "        if tf.test.is_gpu_available(cuda_only=False, min_cuda_compute_capability=None):\n",
    "            device = '/gpu:0'\n",
    "            print('Currently using GPU device to run code')\n",
    "        # need to figure out how to run this on a TPU\n",
    "        else:\n",
    "            device = '/cpu:0'\n",
    "            print('There is no GPU available, using CPU device to run code')\n",
    "        with tf.device(device):\n",
    "            # C/P everything from def train_model above until SGD optimizer\n",
    "            # all code that I want to run on GPU insert here\n",
    "            self.x_input = tf.placeholder(tf.int64, [None, self.window_size])\n",
    "            self.y_input = tf.placeholder(tf.int64, [None])\n",
    "\n",
    "            z = self.embedding_size * self.window_size\n",
    "\n",
    "            # hidden layer biases\n",
    "            d = tf.Variable(tf.random_uniform([self.hidden_units]))\n",
    "            # output biases\n",
    "            b = tf.Variable(tf.random_uniform([vocab_size]))\n",
    "\n",
    "            # weights\n",
    "            # C matrix function\n",
    "            word_embeddings = tf.Variable(tf.random_uniform([vocab_size,\n",
    "                                                             self.embedding_size],\n",
    "                                                            -1.0,\n",
    "                                                            1.0))\n",
    "            flattened_exes = tf.layers.flatten(self.x_input)\n",
    "            lookup = tf.nn.embedding_lookup(word_embeddings, flattened_exes)\n",
    "            xt = tf.reshape(lookup, [self.batch_size, z])\n",
    "\n",
    "            # H Weight\n",
    "            H = tf.Variable(tf.truncated_normal([z, self.hidden_units],\n",
    "                                                stddev=1.0 / math.sqrt(z)))\n",
    "            # W Weight\n",
    "            W = tf.Variable(tf.truncated_normal([z, vocab_size],\n",
    "                                                stddev=1.0 / math.sqrt(z)))\n",
    "            # U Weight\n",
    "            U = tf.Variable(tf.truncated_normal([self.hidden_units, vocab_size],\n",
    "                                                stddev=1.0 / vocab_size))\n",
    "\n",
    "            # hidden layers\n",
    "            tanh = tf.nn.tanh(tf.nn.bias_add(tf.matmul(xt, H), d))\n",
    "            y = tf.nn.bias_add(tf.matmul(xt, W), b) + tf.matmul(tanh, U)\n",
    "\n",
    "            # softmax\n",
    "            y_probability_distribution = tf.nn.softmax(y)\n",
    "            y_ideal = tf.argmax(y_probability_distribution, axis=1)\n",
    "            # produces labels to use in the softmax_cross_entropy_with_logits\n",
    "            y_labels = tf.one_hot(self.y_input, vocab_size)\n",
    "\n",
    "            self.ce_result = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(y_labels,\n",
    "                                                                                            y_probability_distribution))\n",
    "\n",
    "            # stochastic gradient descent optimizer\n",
    "            learning_rate = 0.001\n",
    "            beta1 = 0.9\n",
    "            beta2 = 0.999\n",
    "            adam = tf.train.AdamOptimizer(learning_rate, beta1, beta2).maximize(self.ce_result)\n",
    "            ra = tf.equal(y_ideal, self.y_input)\n",
    "            self.accuracy = tf.reduce_mean(tf.cast(ra, tf.float32))\n",
    "\n",
    "            self.session = tf.Session()\n",
    "            self.session.run(tf.global_variables_initalizer())\n",
    "            saver = tf.train.Saver()\n",
    "\n",
    "            with tf.Session() as session:\n",
    "                # returns variables from the disk\n",
    "                saver.restore(session, path)\n",
    "                print(\"The Model has been restored from disk\")\n",
    "                test_batches = make_batches(test_data, self.batch_size, self.window_size)\n",
    "                cost, accuracy = [], []\n",
    "                for batch in test_batches:\n",
    "                    feed_dict_test = {self.x_input: batch[0],\n",
    "                                      self.y_input: batch[1]}\n",
    "                    accuracy.append(session.run(self.accuracy,\n",
    "                                                feed_dict=feed_dict_test))\n",
    "                    cost.append(session.run(self.ce_result,\n",
    "                                            feed_dict=feed_dict_test))\n",
    "                average_accuracy = sum(accuracy) / float(len(accuracy))\n",
    "                average_cost = sum(cost) / float(len(cost))\n",
    "                print(\"   Accuracy on test-set:   %4.2f %% \\n\" % (average_accuracy * 100),\n",
    "                      \"   Cost on test-set:       %4.2f \\n\" % average_cost,\n",
    "                      \"   Perplexity on test-set:       %4.2f \\n\" % np.exp(average_cost))\n",
    "\n",
    "    def test(self, test_data):\n",
    "        test_batches = make_batches(test_data, self.batch_size, self.window_size)\n",
    "        cost, accuracy = [], []\n",
    "        for batch in test_batches:\n",
    "            feed_dict_test = {self.x_input: batch[0],\n",
    "                              self.y_input: batch[1]}\n",
    "            accuracy.append(self.session.run(self.accuracy, feed_dict=feed_dict_test))\n",
    "            cost.append(self.session.run(self.ce_result, feed_dict=feed_dict_test))\n",
    "        average_accuracy = sum(accuracy) / float(len(accuracy))\n",
    "        average_cost = sum(cost) / float(len(cost))\n",
    "        print(\"   Accuracy on valid-set:   %4.2f %%\" % (average_accuracy * 100),\n",
    "              \"   Cost on valid-set:       %4.2f \\n\" % average_cost)\n",
    "        return average_accuracy, average_cost\n",
    "\n",
    "\n",
    "# key variables\n",
    "a1 = sys.argv[1]\n",
    "a2 = sys.argv[2]\n",
    "a3 = sys.argv[3]\n",
    "\n",
    "user_inputs = {'MLP1': {'window_size': 5, 'hidden_units': 50, 'embedding_size': 60, 'direct': True, 'mix': False},\n",
    "           'MLP5': {'window_size': 5, 'hidden_units': 0, 'embedding_size': 30, 'direct': True, 'mix': False}}\n",
    "\n",
    "corpora = ['wiki', 'brown']\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    if a1 not in ['train', 'load'] or a2 not in user_inputs or a3 not in corpora:\n",
    "        print(\"please enter in a valid input\")\n",
    "        sys.exit()\n",
    "    elif a1 == 'train':\n",
    "        if a3 == 'wiki':\n",
    "            train_path = \"data/wiki.train.txt\"\n",
    "            validation_path = \"data/wiki.valid.txt\"\n",
    "            test_path = \"data/wiki.test.txt\"\n",
    "        elif a3 == 'brown':\n",
    "            split_brown()\n",
    "            train_path = \"data/brown.train.txt\"\n",
    "            validation_path = \"data/brown.valid.txt\"\n",
    "            test_path = \"data/brown.test.txt\"\n",
    "\n",
    "        user_input = user_inputs[a2]\n",
    "        corpus = Preprocessor(train_path)\n",
    "        vocab_size = corpus.vocab_size\n",
    "        train_data = corpus.generate_data(train_path)\n",
    "        validate_data = corpus.generate_data(validation_path)\n",
    "        test_data = corpus.generate_data(test_path)\n",
    "        model = BengioModel()\n",
    "        acc_hist_train, cost_hist_train = [.1] * 10, [7] * 10\n",
    "        model.train_model(train_data, validate_data)\n",
    "        plot_learning(training_accuracy[10:], training_cost[10:])\n",
    "\n",
    "    elif a1 == 'load':\n",
    "        if a3 == 'brown':\n",
    "            split_brown()\n",
    "            train_path = \"data/brown.train.txt\"\n",
    "            path_to_validation_file = \"data/brown.valid.txt\"\n",
    "            path_to_testing_file = \"data/brown.test.txt\"\n",
    "        elif a3 == 'wiki':\n",
    "            train_path = \"data/wiki.train.txt\"\n",
    "            validation_path = \"data/wiki.valid.txt\"\n",
    "            test_path = \"data/wiki.test.txt\"\n",
    "\n",
    "        configuration = configs[a2]\n",
    "        corpus = Preprocessor(train_path)\n",
    "        vocab_size = corpus.vocab_size\n",
    "        test_data = corpus.generate_data(test_path)\n",
    "        model = BengioModel()\n",
    "        model.restore_model('../models/' + a2 + '_' + a3 + '.ckpt')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "default",
   "language": "python",
   "name": "default"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}