NLP_using_TF2_and_BERT.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2IjSWx7-O8yY"
   },
   "source": [
    "# Natural Language Processing using BERT\n",
    "\n",
    "Please study AMA Lecture 12 \"Natural Language Processing Using BERT\" before practicing this code.\n",
    "\n",
    "In addition to `tensorflow` and `keras` packages, this code also requires two new packages:\n",
    "+ `tensorflow_hub` -- \"a repository of trained machine learning models ready for fine-tuning and deployable anywhere\" (https://www.tensorflow.org/hub)\n",
    "  + install `tensorflow-hub` via Anaconda Navigator\n",
    "  + after that, downgrade package `tensorflow-estimator` to version 2.3.0 (because the newer versions are buggy)\n",
    "+ `bert`, which we'll install via \"pip install\" (see later in this code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "executionInfo": {
     "elapsed": 217,
     "status": "ok",
     "timestamp": 1619120042362,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "aYS2t29Im7o1"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1739,
     "status": "ok",
     "timestamp": 1619120061030,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "1HzU3Wionpo9",
    "outputId": "e943ccc0-21bc-4755-f148-77b6d36f3e3b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TF version:  2.6.0\n"
     ]
    }
   ],
   "source": [
    "# Need tf version >=2.0\n",
    "import tensorflow as tf\n",
    "print(\"TF version: \", tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1739,
     "status": "ok",
     "timestamp": 1619120061030,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "1HzU3Wionpo9",
    "outputId": "e943ccc0-21bc-4755-f148-77b6d36f3e3b"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-30 20:20:22.761897: E tensorflow/core/lib/monitoring/collection_registry.cc:77] Cannot register 2 metrics with the same name: /tensorflow/api/estimator\n"
     ]
    },
    {
     "ename": "AlreadyExistsError",
     "evalue": "Another metric with the same name already exists.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAlreadyExistsError\u001b[0m                        Traceback (most recent call last)",
      "\u001b[0;32m/var/folders/fz/40ncrw9j217gvpsrqr3x2j6m0000gn/T/ipykernel_14853/3427377671.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Need hub version >=0.7\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow_hub\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mhub\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Hub version: \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhub\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__version__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.7/site-packages/tensorflow_hub/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     86\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow_hub\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLatestModuleExporter\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow_hub\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mregister_module_for_export\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow_hub\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeature_column\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mimage_embedding_column\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.7/site-packages/tensorflow_hub/estimator.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 62\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mLatestModuleExporter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mv1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mExporter\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     63\u001b[0m   \"\"\"Regularly exports registered modules into timestamped directories.\n\u001b[1;32m     64\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.7/site-packages/tensorflow/python/util/lazy_loader.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, item)\u001b[0m\n\u001b[1;32m     60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m__getattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 62\u001b[0;31m     \u001b[0mmodule\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_load\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     63\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodule\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.7/site-packages/tensorflow/python/util/lazy_loader.py\u001b[0m in \u001b[0;36m_load\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     43\u001b[0m     \u001b[0;34m\"\"\"Load the module and insert it into the parent's globals.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m     \u001b[0;31m# Import the target module and insert it into the parent's namespace\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m     \u001b[0mmodule\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mimportlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimport_module\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent_module_globals\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_local_name\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.7/importlib/__init__.py\u001b[0m in \u001b[0;36mimport_module\u001b[0;34m(name, package)\u001b[0m\n\u001b[1;32m    125\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    126\u001b[0m             \u001b[0mlevel\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 127\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_bootstrap\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gcd_import\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlevel\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpackage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    128\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.7/site-packages/tensorflow_estimator/python/estimator/api/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msys\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0m_sys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow_estimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_v1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mestimator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;32mdel\u001b[0m \u001b[0m_print_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.7/site-packages/tensorflow_estimator/python/estimator/api/_v1/estimator/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msys\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0m_sys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow_estimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_v1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mexperimental\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow_estimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_v1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mexport\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow_estimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_v1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.7/site-packages/tensorflow_estimator/python/estimator/api/_v1/estimator/experimental/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msys\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0m_sys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow_estimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcanned\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdnn\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdnn_logit_fn_builder\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow_estimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcanned\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkmeans\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mKMeansClustering\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mKMeans\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow_estimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcanned\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLinearSDCA\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.7/site-packages/tensorflow_estimator/python/estimator/canned/dnn.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     25\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframework\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtf_export\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mestimator_export\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow_estimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mestimator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow_estimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcanned\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mhead\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mhead_lib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow_estimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcanned\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0moptimizers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.7/site-packages/tensorflow_estimator/python/estimator/estimator.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     60\u001b[0m     ['features', 'labels', 'mode', 'params', 'self', 'config'])\n\u001b[1;32m     61\u001b[0m _estimator_api_gauge = monitoring.BoolGauge('/tensorflow/api/estimator',\n\u001b[0;32m---> 62\u001b[0;31m                                             'estimator api usage', 'method')\n\u001b[0m\u001b[1;32m     63\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m _canned_estimator_api_gauge = monitoring.StringGauge(\n",
      "\u001b[0;32m/anaconda3/lib/python3.7/site-packages/tensorflow/python/eager/monitoring.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, name, description, *labels)\u001b[0m\n\u001b[1;32m    359\u001b[0m     \"\"\"\n\u001b[1;32m    360\u001b[0m     super(BoolGauge, self).__init__('BoolGauge', _bool_gauge_methods,\n\u001b[0;32m--> 361\u001b[0;31m                                     len(labels), name, description, *labels)\n\u001b[0m\u001b[1;32m    362\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    363\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mget_cell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda3/lib/python3.7/site-packages/tensorflow/python/eager/monitoring.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, metric_name, metric_methods, label_length, *args)\u001b[0m\n\u001b[1;32m    133\u001b[0m           self._metric_name, len(self._metric_methods)))\n\u001b[1;32m    134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 135\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_metric\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_metric_methods\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_label_length\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    136\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m__del__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAlreadyExistsError\u001b[0m: Another metric with the same name already exists."
     ]
    }
   ],
   "source": [
    "# Need hub version >=0.7\n",
    "import tensorflow_hub as hub\n",
    "print(\"Hub version: \", hub.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 176,
     "status": "ok",
     "timestamp": 1619120095486,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "SOunH6Q4nsEf"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras import models\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import optimizers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Cyn23fDFJpZ8"
   },
   "source": [
    "## Case study: the IMDB dataset\n",
    "\n",
    "This is a widely used large dataset for text mining from a [2011 ACL meeting paper](https://ai.stanford.edu/~amaas/data/sentiment/) by Maas et al. I processed the data so it fits in a single CSV file 'IMDB_small.csv'.\n",
    "\n",
    "The original dataset has 50,000 balanced records, and the data file takes too long to upload. For our course, I randomly sampled 10,000 records and saved them in file 'IMDB_small.csv'. This is still a balanced sample, where the first 5000 are negative reviews and the rest are positive reviews."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 202
    },
    "executionInfo": {
     "elapsed": 2225,
     "status": "ok",
     "timestamp": 1619120101878,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "jsAPeTuHphgT",
    "outputId": "e900b6b1-e29e-4af7-eed2-37586a3b9982",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# load the IMDB dataset\n",
    "df = pd.read_csv('IMDB_small.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 188,
     "status": "ok",
     "timestamp": 1619120266357,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "VBNefBTeHcqH",
    "outputId": "bbbc4e19-b61e-4149-8ebf-c30417bd321a"
   },
   "outputs": [],
   "source": [
    "df.sentiment.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 194,
     "status": "ok",
     "timestamp": 1619120267355,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "K_NJpLIpSGTO",
    "outputId": "4399e5f2-4b0c-4cf3-adcb-77137ffa54c1"
   },
   "outputs": [],
   "source": [
    "# one negative example:\n",
    "import textwrap\n",
    "print(textwrap.fill(df.review[2], 80))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 182,
     "status": "ok",
     "timestamp": 1619120314009,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "GfXyJ2aFW0I9",
    "outputId": "ee789642-e55d-40ef-bef3-ca4babcf8e52"
   },
   "outputs": [],
   "source": [
    "# one positive example:\n",
    "print(textwrap.fill(df.review[5000], 80))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 183,
     "status": "ok",
     "timestamp": 1619120493637,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "ICTlwBaqsgZO"
   },
   "outputs": [],
   "source": [
    "# The following codes make it easier for you to adopt\n",
    "# this file for other text mining datasets.\n",
    "DATA_COLUMN = 'review'\n",
    "LABEL_COLUMN = 'sentiment'\n",
    "label_list = [0, 1] #0-negative, 1-positive"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "J5HURQ9CIdHs"
   },
   "source": [
    "## Introducing BERT\n",
    "\n",
    "**BERT (Bidirectional Encoder Representations from Transformers)** is the state-of-the-art feature extraction model for natural language.\n",
    "\n",
    "Some resources on BERT:\n",
    "- See BERT on paper: https://arxiv.org/pdf/1810.04805.pdf\n",
    "- See BERT on GitHub: https://github.com/google-research/bert\n",
    "- See BERT on TensorHub: https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1\n",
    "- See 'old' use of BERT for comparison: https://colab.research.google.com/github/google-research/bert/blob/master/predicting_movie_reviews_with_bert_on_tf_hub.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "1dU6PU4emdgQ"
   },
   "source": [
    "Next, we will use BERT in four steps:\n",
    "* Import and build the BERT model\n",
    "* Tokenization\n",
    "* Convert tokens to BERT input format\n",
    "* Sentence/word embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PtaEqbjwrI4s"
   },
   "source": [
    "## Importing and building the BERT model\n",
    "\n",
    "This part of code might confuse you a bit for now. We will come back and explain it more."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 6301,
     "status": "ok",
     "timestamp": 1619120562842,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "cEbWuAdFqB2S",
    "outputId": "98475347-6578-4578-8c8e-9849c6b310aa"
   },
   "outputs": [],
   "source": [
    "# !pip install sentencepiece\n",
    "!pip install bert-for-tf2\n",
    "import bert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 216,
     "status": "ok",
     "timestamp": 1619120570067,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "IUInJmh7qpvo"
   },
   "outputs": [],
   "source": [
    "# BERT requires a MAX_SEQ_LENGTH that can be any integer<=512.\n",
    "# Here we pick a smaller number to cut down computation cost.\n",
    "max_seq_length = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 200,
     "status": "ok",
     "timestamp": 1619120804144,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "6IqEqHw8pxPo"
   },
   "outputs": [],
   "source": [
    "# BERT requires the following three types of inputs (more on them later)\n",
    "input_word_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n",
    "                                       name=\"input_word_ids\")\n",
    "input_mask = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n",
    "                                   name=\"input_mask\")\n",
    "segment_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n",
    "                                    name=\"segment_ids\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 17145,
     "status": "ok",
     "timestamp": 1619120838567,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "rs-kXm1P39Lo"
   },
   "outputs": [],
   "source": [
    "# Now we load the already pre-trained BERT layers\n",
    "# Ignore the warning message, which won't affect our usage of bert\n",
    "bert_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1\",\n",
    "                            trainable=True)\n",
    "pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 179,
     "status": "ok",
     "timestamp": 1619120856417,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "LETaImqhruXz"
   },
   "outputs": [],
   "source": [
    "model = models.Model(inputs=[input_word_ids, input_mask, segment_ids], \n",
    "                     outputs=[pooled_output, sequence_output])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 396,
     "status": "ok",
     "timestamp": 1619121016554,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "cAxh7sZe3EcV",
    "outputId": "29b6022d-f900-4858-f2cc-9e339394057b"
   },
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FShEbcDZmvF_"
   },
   "source": [
    "## BERT for tokenization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "44u2pruZSbMX"
   },
   "source": [
    "Import tokenizer using the original vocab file:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 294,
     "status": "ok",
     "timestamp": 1619121300938,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "sm3lGfQb-1J8"
   },
   "outputs": [],
   "source": [
    "vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\n",
    "do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\n",
    "tokenizer = bert.bert_tokenization.FullTokenizer(vocab_file, do_lower_case)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 194,
     "status": "ok",
     "timestamp": 1619121342378,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "5bQ-FXvtg_tY",
    "outputId": "4943323a-ec3b-4727-ca37-64f65f0f9354"
   },
   "outputs": [],
   "source": [
    "# The tokenizer converts a sentence to a sequence of tokens. Here's an example:\n",
    "text = \"Here is an example sentence that I want to tokenize.\"\n",
    "tokenized_text = tokenizer.tokenize(text)\n",
    "print(tokenized_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AihvrFWcSzd6"
   },
   "source": [
    "Now we tokenize every review in the IMDB dataset. This may take a minute to finish."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 38415,
     "status": "ok",
     "timestamp": 1619121433354,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "IeM20UdZ59i1"
   },
   "outputs": [],
   "source": [
    "df['tokens'] = df[DATA_COLUMN].apply(lambda x : tokenizer.tokenize(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 190,
     "status": "ok",
     "timestamp": 1619121437607,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "X798BKV_Co71",
    "outputId": "ba9a3761-1126-40ec-9783-4a9659f281a9"
   },
   "outputs": [],
   "source": [
    "# An example of how the tokens for a review look like:\n",
    "print(df['tokens'][2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 185,
     "status": "ok",
     "timestamp": 1619121453208,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "JhRfWt7S7lsE",
    "outputId": "e25bcfa3-7de5-4d6b-e746-229910e6d74c"
   },
   "outputs": [],
   "source": [
    "# Some reviews are long. For example:\n",
    "len(df['tokens'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 499,
     "status": "ok",
     "timestamp": 1619121500094,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "KQlSpWu_n4RY",
    "tags": []
   },
   "outputs": [],
   "source": [
    "# We now truncate any review with >=(MAX_SEQ_LENGTH-2) tokens.\n",
    "# And add special tokens [CLS] and [SEP].\n",
    "\n",
    "def truncate_and_add(x, max_seq_length):\n",
    "  a = [\"[CLS]\"] + x\n",
    "  if len(a)>max_seq_length-1:\n",
    "    a[max_seq_length-1] = \"[SEP]\"\n",
    "    return a[:max_seq_length]\n",
    "  else:\n",
    "    return a + [\"[SEP]\"]\n",
    "\n",
    "df['tokens'] = df['tokens'].apply(lambda x : truncate_and_add(x, max_seq_length))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ul1Y5afr5GCJ"
   },
   "source": [
    "## Converting tokens to BERT input format"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "y9PNZoFj3N6q"
   },
   "source": [
    "We'll need to transform our data into a format BERT understands. This involves two steps. First, we create  `InputExamples` using the constructor provided in the BERT library.\n",
    "\n",
    "- `text_a` is the text we want to classify, which in this case, is the `review` field in our Dataframe. \n",
    "- `text_b` is used if we're training a model to understand the relationship between sentences (i.e. is `text_b` a translation of `text_a`? Is `text_b` an answer to the question asked by `text_a`?). This doesn't apply to our task, so we can leave `text_b` blank.\n",
    "- `label` is the target in supervised learning, which is `sentiment` in our example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tU2OpvYrRFNf"
   },
   "source": [
    "To use BERT embedding, we need to convert the tokens of each text input into the following format:\n",
    " - input token ids (tokenizer converts tokens using vocab file)\n",
    " - input masks (1 for useful tokens, 0 for padding)\n",
    " - segment ids (for 2 text training: 0 for the first one, 1 for the second one)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BFDpzy1-STOh"
   },
   "source": [
    "Define some functions for ease of preprocessing:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 181,
     "status": "ok",
     "timestamp": 1619121594212,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "e4Y_r3lmFO1E"
   },
   "outputs": [],
   "source": [
    "def get_ids(tokens, tokenizer, max_seq_length):\n",
    "    token_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "    token_ids = token_ids + [0] * (max_seq_length-len(token_ids))\n",
    "    return np.array(token_ids, dtype=np.int32)\n",
    "    \n",
    "def get_masks(tokens, max_seq_length):\n",
    "    token_masks = [1]*len(tokens) + [0] * (max_seq_length - len(tokens))\n",
    "    return np.array(token_masks, dtype=np.int32)\n",
    "\n",
    "def get_segments(tokens, max_seq_length):\n",
    "    \"\"\"Segments: 0 for the first sequence, 1 for the second\"\"\"\n",
    "    segments = []\n",
    "    current_segment_id = 0\n",
    "    for token in tokens:\n",
    "        segments.append(current_segment_id)\n",
    "        if token == \"[SEP]\":\n",
    "            current_segment_id = 1\n",
    "    segments = segments + [0] * (max_seq_length - len(tokens))\n",
    "    return np.array(segments, dtype=np.int32)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 1310,
     "status": "ok",
     "timestamp": 1619121601505,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "b_vqng48LuTs"
   },
   "outputs": [],
   "source": [
    "df['ids'] = df['tokens'].apply(lambda x : get_ids(x, tokenizer, max_seq_length))\n",
    "df['masks'] = df['tokens'].apply(lambda x : get_masks(x, max_seq_length))\n",
    "df['segments'] = df['tokens'].apply(lambda x : get_segments(x, max_seq_length))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 185,
     "status": "ok",
     "timestamp": 1619121603417,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "sb4GSTEkMgE3",
    "outputId": "63c982cc-c56c-4b89-e105-1b64077adb02"
   },
   "outputs": [],
   "source": [
    "# Let's see what the first movie review is now converted to:\n",
    "df.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 1101,
     "status": "ok",
     "timestamp": 1619121666055,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "nnaj5QdqleTS"
   },
   "outputs": [],
   "source": [
    "# Now assemble the data as required by the definition of BERT inputs\n",
    "n = df.shape[0]\n",
    "all_ids = np.zeros(shape=(n,max_seq_length))\n",
    "all_masks = np.zeros(shape=(n,max_seq_length)) \n",
    "all_segments = np.zeros(shape=(n,max_seq_length))\n",
    "i = 0\n",
    "for index, row in df.iterrows():\n",
    "  all_ids[i] = row.ids\n",
    "  all_masks[i] = row.masks\n",
    "  all_segments[i] = row.segments\n",
    "  i += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mi2mj4EUTi0X"
   },
   "source": [
    "\n",
    "## Using the pre-trained BERT model for sentence embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "u-RTA4QosF1H"
   },
   "source": [
    "BERT converts each text input (in our example, a tokenized movie review) into the following.\n",
    "* **pooled output** (also called pooled embedding, sentence embedding): this is a vector of size `768`, which represents the whole sentence.\n",
    "* **sequence outputs** (also called sequence embeddings, word embeddings): this is a matrix of size `[max_seq_length, 768]`, where each token is now represented by a vector of size `768`.\n",
    "\n",
    "**For sentiment analysis, we only need the pooled output.**\n",
    "\n",
    "Similar to other deep learning models, BERT doesn't transform text one record at a time. Instead, BERT takes a batch of texts (e.g., a batch of movie reviews in our case) and convert them all at once. Thus the output shapes are:\n",
    " - pooled output of shape `[batch_size, 768]` with representations for the entire input sequences\n",
    " - sequence output of shape `[batch_size, max_seq_length, 768]`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "rH0bhH2S9cRs"
   },
   "source": [
    "### A big data problem\n",
    "\n",
    "The output size from BERT can be huge. For example, in our dataset of 10000 movie reviews, where each review has a (truncated) length of 256, the total size of sequence embeddings is: `10000 * 256 * 768 * 4 ~= 8 Gigabyte`. This is too large to fit in the memory of most personal computers. So the following single-line code will likely trigger a \"ResourceExhaustedError\".\n",
    "\n",
    "`pool_embs, seq_embs = model.predict([all_ids,all_masks,all_segments])`\n",
    "\n",
    "Below is an workaround to avoid this bid data problem. We process our data 1000 records a time, i.e., set batch size at 1000. After each batch is processed, discard the sequence embeddings because we don't need them, and only save the pooled embeddings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 209546,
     "status": "ok",
     "timestamp": 1619122297785,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "aag7a3JDd2kP",
    "outputId": "ef9bffaf-a134-4003-882d-7ee11689e558"
   },
   "outputs": [],
   "source": [
    "pool_embs = np.zeros(shape=(n,768))\n",
    "for i in np.arange(10):\n",
    "  j = i*1000\n",
    "  pool_embs[j:j+1000], seq_embs = model.predict([all_ids[j:j+1000],\n",
    "                                                 all_masks[j:j+1000],\n",
    "                                                 all_segments[j:j+1000]])\n",
    "  print(f'{i+1}/10 of the data processed.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 254,
     "status": "ok",
     "timestamp": 1619122427117,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "DP43272orwSR",
    "outputId": "041b8817-1c98-4523-ae78-8462352df8d1"
   },
   "outputs": [],
   "source": [
    "pool_embs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 628,
     "status": "ok",
     "timestamp": 1619122446166,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "Ktdixe_2tfyx",
    "outputId": "d791a837-86c7-4305-a040-354928c84578",
    "tags": []
   },
   "outputs": [],
   "source": [
    "pool_embs[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "r5HcG7LpAAZE"
   },
   "source": [
    "## Assembling a new dataset with features extracted by BERT\n",
    "\n",
    "For each text, the corresponding pooled output is a vector of 768 numbers that summaries this whole text. We can now treat these 768 numbers as features extracted by BERT. Let's assemble a new DataFrame with these figures and the sentiment data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 251
    },
    "executionInfo": {
     "elapsed": 352,
     "status": "ok",
     "timestamp": 1619122519906,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "jxzOuyZd_O92",
    "outputId": "4ef89e36-20a2-4386-ba21-9cc190ea83d0"
   },
   "outputs": [],
   "source": [
    "feature_df = pd.DataFrame(pool_embs)\n",
    "feature_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 251
    },
    "executionInfo": {
     "elapsed": 347,
     "status": "ok",
     "timestamp": 1619122549920,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "pZi0XUqcrRay",
    "outputId": "ccd23669-27a0-4d08-a3fb-8f02986d5a83",
    "tags": []
   },
   "outputs": [],
   "source": [
    "feature_df['sentiment'] = df['sentiment']\n",
    "feature_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "46LGa84M8pOw"
   },
   "outputs": [],
   "source": [
    "# Warning: this file will be large, about 150MB\n",
    "feature_df.to_csv(\"IMDB_small_BERT.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bgAk3J6zDf2y"
   },
   "source": [
    "## Building and evaluating the prediction model\n",
    "\n",
    "The rest is similar to what we did with the business loan dataset earlier this semester. I'll use the simple logistic regression model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 910,
     "status": "ok",
     "timestamp": 1619122760593,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "WSd-GQC1VolF"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 288,
     "status": "ok",
     "timestamp": 1619122764218,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "gRVdYop0VKjN"
   },
   "outputs": [],
   "source": [
    "X = feature_df.drop(columns=['sentiment'])\n",
    "y = feature_df['sentiment']\n",
    "\n",
    "# reserve 30% dataset as testing\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, \n",
    "                                                    random_state=1,\n",
    "                                                    stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 182,
     "status": "ok",
     "timestamp": 1619122771248,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "IczZQOIUCqWD"
   },
   "outputs": [],
   "source": [
    "model2 = models.Sequential()\n",
    "model2.add(layers.Dense(128, activation='relu', input_dim=768))\n",
    "# model2.add(layers.Dropout(0.5))\n",
    "model2.add(layers.Dense(1, activation='sigmoid'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 240,
     "status": "ok",
     "timestamp": 1619122782883,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "h9KN2l8HWXJa"
   },
   "outputs": [],
   "source": [
    "model2.compile(optimizer='adam',\n",
    "              loss='binary_crossentropy',\n",
    "              metrics=['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 173,
     "status": "ok",
     "timestamp": 1619122790452,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "NlwtEMbuXg3k",
    "outputId": "99f7b1de-40f8-4154-90da-0eea357c2dd1"
   },
   "outputs": [],
   "source": [
    "model2.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 15930,
     "status": "ok",
     "timestamp": 1619122813395,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "MJZGmll0Wu0b",
    "outputId": "1d15ee4f-4a4b-4aab-bf66-2852e02d349f"
   },
   "outputs": [],
   "source": [
    "model2.fit(X_train, y_train, epochs=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 509,
     "status": "ok",
     "timestamp": 1619122820687,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "_RRh4XkYXuw7",
    "outputId": "bbc7ab66-97ff-4922-e3c8-63074b656b71"
   },
   "outputs": [],
   "source": [
    "test_loss, test_acc = model2.evaluate(X_test,  y_test, verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 286,
     "status": "ok",
     "timestamp": 1619122908349,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "W-sBWYvAXzer",
    "outputId": "56bbc494-fbb3-4e23-b1fb-9d8976d993e8"
   },
   "outputs": [],
   "source": [
    "# prediction\n",
    "model2.predict(X_test.iloc[[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 192,
     "status": "ok",
     "timestamp": 1619122938673,
     "user": {
      "displayName": "Xianjun Geng",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gjp2zNPbIYU0DaDSSzFs4-oEwNRuLg3LhJILAgh3RY=s64",
      "userId": "03603659341670711497"
     },
     "user_tz": 300
    },
    "id": "ZdCN6PUCYnGm",
    "outputId": "bf68b4b1-9652-4a25-b279-1fab3f24ee05"
   },
   "outputs": [],
   "source": [
    "print(y_test[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "NLP_using_TF2_and_BERT.ipynb",
   "provenance": [
    {
     "file_id": "1hMLd5-r82FrnFnBub-B-fVW78Px4KPX1",
     "timestamp": 1587333874208
    }
   ]
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}