gad_dataset_with_bert.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "gad-dataset-with-bert.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1XNyKyu3e8WrOW5w0rNqXxZOoKV3LBoZ5",
      "authorship_tag": "ABX9TyNMrYU6ACFM8FUruYQJTSSm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "771166825e5c44edb84c5a169d2fc5d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1b2daa66e48b4c53aeb917fe083b6b79",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_29ce5519e64143feaadf378ee3563959",
              "IPY_MODEL_855af5aff4274d27b0df55d7e2c0cb27"
            ]
          }
        },
        "1b2daa66e48b4c53aeb917fe083b6b79": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "29ce5519e64143feaadf378ee3563959": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e2e3157a013845758e2399003afea158",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f686665e91e44eb0881ab51fde8d5664"
          }
        },
        "855af5aff4274d27b0df55d7e2c0cb27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_29f8601cdad74e51b721f5c734ac0a82",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 2.73MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8b5f1c2ca5784ad5a9e112662644fb56"
          }
        },
        "e2e3157a013845758e2399003afea158": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f686665e91e44eb0881ab51fde8d5664": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "29f8601cdad74e51b721f5c734ac0a82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8b5f1c2ca5784ad5a9e112662644fb56": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "902527e80b1d48e79dc9907764ff8aad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_df10712f8b524ce8aea552949e074f61",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5e04972fd1fb435aa1868af95ec45826",
              "IPY_MODEL_4ef02997094e473f90dfb37c1acb8690"
            ]
          }
        },
        "df10712f8b524ce8aea552949e074f61": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5e04972fd1fb435aa1868af95ec45826": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5d9b9a5a9581434c8870f8e79426b0d4",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_07d31c9453204f9eb21495247b9b7eaa"
          }
        },
        "4ef02997094e473f90dfb37c1acb8690": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a88966a43a3b4258aadfb982340691ac",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:00&lt;00:00, 2.28kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_58408fc1546447ffb4e3f8f2ef8ce1ce"
          }
        },
        "5d9b9a5a9581434c8870f8e79426b0d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "07d31c9453204f9eb21495247b9b7eaa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a88966a43a3b4258aadfb982340691ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "58408fc1546447ffb4e3f8f2ef8ce1ce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3906fc4524c04b1898cdeb1e54872909": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_857a4529a6d14a71b2d9ea63ad300168",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_fe419efad8e24a75ac97e1a386b7af86",
              "IPY_MODEL_0b00eee2473e4892965ca8bf777836dc"
            ]
          }
        },
        "857a4529a6d14a71b2d9ea63ad300168": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fe419efad8e24a75ac97e1a386b7af86": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ae949cc7adbe461dadaca5b7592becb0",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_149c1b2b56654132b4b7ff39f424d1a6"
          }
        },
        "0b00eee2473e4892965ca8bf777836dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_7981fa086dca479da989f58f9ae659d3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:25&lt;00:00, 17.3MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_05ed9f383ec64fec862db44333ace00b"
          }
        },
        "ae949cc7adbe461dadaca5b7592becb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "149c1b2b56654132b4b7ff39f424d1a6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7981fa086dca479da989f58f9ae659d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "05ed9f383ec64fec862db44333ace00b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/hushee69/biobert-relation-extraction/blob/main/gad_dataset_with_bert.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zNU6EMsKnqng"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "asONPSIFkEx3",
        "outputId": "f2c2fbb0-98d1-4f81-b035-88ad8b3851eb"
      },
      "source": [
        "# datasets link: https://drive.google.com/open?id=1-jDKGcXREb2X9xTFnuiJ36PvsqoyHWcw\r\n",
        "\r\n",
        "!pip install transformers"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/50/0c/7d5950fcd80b029be0a8891727ba21e0cd27692c407c51261c3c921f6da3/transformers-4.1.1-py3-none-any.whl (1.5MB)\n",
            "\r\u001b[K     |▏                               | 10kB 29.6MB/s eta 0:00:01\r\u001b[K     |▍                               | 20kB 34.8MB/s eta 0:00:01\r\u001b[K     |▋                               | 30kB 21.0MB/s eta 0:00:01\r\u001b[K     |▉                               | 40kB 23.6MB/s eta 0:00:01\r\u001b[K     |█                               | 51kB 22.6MB/s eta 0:00:01\r\u001b[K     |█▎                              | 61kB 16.8MB/s eta 0:00:01\r\u001b[K     |█▌                              | 71kB 16.8MB/s eta 0:00:01\r\u001b[K     |█▊                              | 81kB 17.9MB/s eta 0:00:01\r\u001b[K     |██                              | 92kB 17.4MB/s eta 0:00:01\r\u001b[K     |██▏                             | 102kB 17.2MB/s eta 0:00:01\r\u001b[K     |██▍                             | 112kB 17.2MB/s eta 0:00:01\r\u001b[K     |██▋                             | 122kB 17.2MB/s eta 0:00:01\r\u001b[K     |██▉                             | 133kB 17.2MB/s eta 0:00:01\r\u001b[K     |███                             | 143kB 17.2MB/s eta 0:00:01\r\u001b[K     |███▎                            | 153kB 17.2MB/s eta 0:00:01\r\u001b[K     |███▌                            | 163kB 17.2MB/s eta 0:00:01\r\u001b[K     |███▊                            | 174kB 17.2MB/s eta 0:00:01\r\u001b[K     |████                            | 184kB 17.2MB/s eta 0:00:01\r\u001b[K     |████▏                           | 194kB 17.2MB/s eta 0:00:01\r\u001b[K     |████▎                           | 204kB 17.2MB/s eta 0:00:01\r\u001b[K     |████▌                           | 215kB 17.2MB/s eta 0:00:01\r\u001b[K     |████▊                           | 225kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████                           | 235kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 245kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████▍                          | 256kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████▋                          | 266kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████▉                          | 276kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████                          | 286kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 296kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████▌                         | 307kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████▊                         | 317kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████                         | 327kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████▏                        | 337kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████▍                        | 348kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 358kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 368kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████                        | 378kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████▎                       | 389kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 399kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████▋                       | 409kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 419kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████                       | 430kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████▎                      | 440kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 450kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 460kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████                      | 471kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████▏                     | 481kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 491kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████▋                     | 501kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████▉                     | 512kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████                     | 522kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████▎                    | 532kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████▌                    | 542kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 552kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████                    | 563kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 573kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████▍                   | 583kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 593kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 604kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 614kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 624kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████▍                  | 634kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████▋                  | 645kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████▉                  | 655kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 665kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████▎                 | 675kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 686kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████▊                 | 696kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 706kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 716kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 727kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████▋                | 737kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████▉                | 747kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████                | 757kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 768kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████▌               | 778kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 788kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 798kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 808kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████▎              | 819kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████▌              | 829kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████▊              | 839kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 849kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████▏             | 860kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████▍             | 870kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 880kB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████▉             | 890kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 901kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████▎            | 911kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 921kB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 931kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 942kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████████▏           | 952kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 962kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████████▋           | 972kB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 983kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 993kB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 1.0MB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████▍          | 1.0MB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████▋          | 1.0MB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████▉          | 1.0MB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 1.0MB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 1.1MB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████▌         | 1.1MB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 1.1MB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 1.1MB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████▏        | 1.1MB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 1.1MB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████▋        | 1.1MB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 1.1MB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 1.1MB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 1.1MB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████▌       | 1.2MB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 1.2MB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 1.2MB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 1.2MB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▍      | 1.2MB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 1.2MB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▊      | 1.2MB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 1.2MB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 1.2MB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 1.2MB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▋     | 1.3MB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 1.3MB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 1.3MB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 1.3MB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 1.3MB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 1.3MB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 1.3MB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 1.3MB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 1.3MB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▋   | 1.4MB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▉   | 1.4MB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 1.4MB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▎  | 1.4MB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 1.4MB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▊  | 1.4MB 17.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▉  | 1.4MB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 1.4MB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 1.4MB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▌ | 1.4MB 17.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 1.5MB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 1.5MB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▏| 1.5MB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 1.5MB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▋| 1.5MB 17.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 1.5MB 17.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.5MB 17.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.8)\n",
            "Collecting tokenizers==0.9.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0f/1c/e789a8b12e28be5bc1ce2156cf87cb522b379be9cadc7ad8091a4cc107c4/tokenizers-0.9.4-cp36-cp36m-manylinux2010_x86_64.whl (2.9MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9MB 54.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 57.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.19.4)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.0.0)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=64c15692e12746daca10dcc1cda0b6e3edee5f8f8cfa44da654f983f9b572114\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.9.4 transformers-4.1.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7VzzbgtPnv1Y"
      },
      "source": [
        "import numpy as np;\r\n",
        "import pandas as pd;\r\n",
        "\r\n",
        "import torch;\r\n",
        "\r\n",
        "from transformers import BertTokenizer, BertForSequenceClassification, AdamW, BertConfig;\r\n",
        "from torch.utils.data import TensorDataset, random_split, DataLoader, RandomSampler, SequentialSampler;\r\n",
        "from transformers import get_linear_schedule_with_warmup;\r\n",
        "\r\n",
        "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score;"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TWQSAHb2nwOg",
        "outputId": "ed69a468-bb2b-45a2-dc9d-23304d8cf5fe"
      },
      "source": [
        "from google.colab import drive;\r\n",
        "\r\n",
        "drive.mount('/content/gdrive');"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zSROJxLHn2Tj"
      },
      "source": [
        "path='/content/gdrive/MyDrive/biobert_re/';"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jhcSAgfon-cC"
      },
      "source": [
        "device = torch.device('cuda');\r\n",
        "\r\n",
        "SEED = 42;\r\n",
        "\r\n",
        "torch.manual_seed(SEED);\r\n",
        "torch.backends.cudnn.deterministic = True;"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-DVOubFsoADX"
      },
      "source": [
        "\"\"\"\r\n",
        "    Params:\r\n",
        "        filepath: path of the dataset\r\n",
        "        tokenizer: tokenizer to use\r\n",
        "        maxlen: maxlength of text\r\n",
        "\"\"\"\r\n",
        "def load_and_process_gad_train_data(filepath, tokenizer, maxlen=512, train_percentage=0.7):\r\n",
        "    # load dataset\r\n",
        "    df = pd.read_csv(filepath, header=None, delimiter='\\t', names=['sentence', 'label']);\r\n",
        "\r\n",
        "    sentences = df.sentence.values;\r\n",
        "    labels = df.label.values;\r\n",
        "    \r\n",
        "    input_ids = [];\r\n",
        "    attention_masks = [];\r\n",
        "    \r\n",
        "    for sent in sentences:\r\n",
        "        encoded_dict = tokenizer.encode_plus(\r\n",
        "            sent,\r\n",
        "            add_special_tokens=True,\r\n",
        "            max_length=maxlen,\r\n",
        "            padding='max_length',\r\n",
        "            truncation=True,\r\n",
        "            return_attention_mask=True,\r\n",
        "            return_tensors='pt'\r\n",
        "        );\r\n",
        "\r\n",
        "        input_ids.append(encoded_dict['input_ids']);\r\n",
        "        attention_masks.append(encoded_dict['attention_mask']);\r\n",
        "\r\n",
        "    # convert lists into tensors\r\n",
        "    input_ids = torch.cat(input_ids, dim=0);\r\n",
        "    attention_masks = torch.cat(attention_masks, dim=0);\r\n",
        "    labels_tensor = torch.tensor(labels);\r\n",
        "\r\n",
        "    dataset = TensorDataset(input_ids, attention_masks, labels_tensor);\r\n",
        "\r\n",
        "    train_size = int(train_percentage * len(dataset));\r\n",
        "    val_size = len(dataset) - train_size;\r\n",
        "\r\n",
        "    train_dataset, val_dataset = random_split(dataset, [train_size, val_size]);\r\n",
        "\r\n",
        "    return (train_dataset, val_dataset);"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kPzh6Gq_pW12"
      },
      "source": [
        "def train_gad_model(model, data, optimizer=AdamW, batch_size=32, epochs=3):\r\n",
        "    max_val_loss = np.float('inf');\r\n",
        "\r\n",
        "    train_ds = data[0];\r\n",
        "    val_ds = data[1];\r\n",
        "\r\n",
        "    train_dataloader = DataLoader(\r\n",
        "        train_ds,\r\n",
        "        sampler=RandomSampler(train_ds),\r\n",
        "        batch_size=batch_size\r\n",
        "    );\r\n",
        "\r\n",
        "    val_dataloader = DataLoader(\r\n",
        "        val_ds,\r\n",
        "        sampler=SequentialSampler(val_ds),\r\n",
        "        batch_size=batch_size\r\n",
        "    );\r\n",
        "\r\n",
        "    for e in range(epochs):\r\n",
        "        train_loss = 0;\r\n",
        "        train_acc = 0;\r\n",
        "\r\n",
        "        model.train();\r\n",
        "\r\n",
        "        optim = optimizer(model.parameters(), lr=2e-5, eps=1e-8);\r\n",
        "\r\n",
        "        scheduler = get_linear_schedule_with_warmup(\r\n",
        "            optim,\r\n",
        "            num_warmup_steps=0,\r\n",
        "            num_training_steps=len(train_dataloader) * epochs\r\n",
        "        );\r\n",
        "\r\n",
        "        for batch in train_dataloader:\r\n",
        "            b_input_ids = batch[0].to(device);\r\n",
        "            b_input_mask = batch[1].to(device);\r\n",
        "            b_labels = batch[2].to(device);\r\n",
        "\r\n",
        "            model.zero_grad();\r\n",
        "\r\n",
        "            output = model(\r\n",
        "                b_input_ids,\r\n",
        "                token_type_ids=None,\r\n",
        "                attention_mask=b_input_mask,\r\n",
        "                labels=b_labels\r\n",
        "            );\r\n",
        "\r\n",
        "            loss = output['loss'];\r\n",
        "            preds = output['logits'].detach().cpu().numpy();\r\n",
        "            labels = b_labels.to('cpu').numpy();\r\n",
        "            preds = np.argmax(preds, axis=1);\r\n",
        "\r\n",
        "            train_loss += loss.item();\r\n",
        "            train_acc += accuracy_score(labels, preds);\r\n",
        "\r\n",
        "            loss.backward();\r\n",
        "\r\n",
        "            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0);\r\n",
        "\r\n",
        "            optim.step();\r\n",
        "\r\n",
        "            scheduler.step();\r\n",
        "        \r\n",
        "        avg_train_loss = train_loss / len(train_dataloader);\r\n",
        "        avg_train_acc = train_acc / len(train_dataloader);\r\n",
        "\r\n",
        "        print('average training loss for epoch: {}'.format(avg_train_loss));\r\n",
        "        print('average training accuracy for epoch: {}'.format(avg_train_acc));\r\n",
        "\r\n",
        "        # validation\r\n",
        "        val_loss = 0;\r\n",
        "        val_acc = 0;\r\n",
        "\r\n",
        "        model.eval();\r\n",
        "\r\n",
        "        for batch in val_dataloader:\r\n",
        "            b_input_ids = batch[0].to(device);\r\n",
        "            b_attention_mask = batch[1].to(device);\r\n",
        "            b_labels = batch[2].to(device);\r\n",
        "\r\n",
        "            with torch.no_grad():\r\n",
        "                output = model(\r\n",
        "                    b_input_ids,\r\n",
        "                    token_type_ids=None,\r\n",
        "                    attention_mask=b_attention_mask,\r\n",
        "                    labels=b_labels\r\n",
        "                );\r\n",
        "            \r\n",
        "            loss = output['loss'];\r\n",
        "            preds = output['logits'].detach().cpu().numpy();\r\n",
        "            labels = b_labels.to('cpu').numpy();\r\n",
        "            preds = np.argmax(preds, axis=1);\r\n",
        "\r\n",
        "            val_loss += loss.item();\r\n",
        "            val_acc += accuracy_score(labels, preds);\r\n",
        "        \r\n",
        "        avg_val_loss = val_loss / len(val_dataloader);\r\n",
        "        avg_val_acc = val_acc / len(val_dataloader);\r\n",
        "\r\n",
        "        if avg_val_loss < max_val_loss:\r\n",
        "            max_val_loss = avg_val_loss;\r\n",
        "            torch.save(model.state_dict(), 'best_model.pt');\r\n",
        "\r\n",
        "        print('average validation loss for epoch: {}'.format(avg_val_loss));\r\n",
        "        print('average validation accuracy for epoch: {}'.format(avg_val_acc));"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 280,
          "referenced_widgets": [
            "771166825e5c44edb84c5a169d2fc5d7",
            "1b2daa66e48b4c53aeb917fe083b6b79",
            "29ce5519e64143feaadf378ee3563959",
            "855af5aff4274d27b0df55d7e2c0cb27",
            "e2e3157a013845758e2399003afea158",
            "f686665e91e44eb0881ab51fde8d5664",
            "29f8601cdad74e51b721f5c734ac0a82",
            "8b5f1c2ca5784ad5a9e112662644fb56",
            "902527e80b1d48e79dc9907764ff8aad",
            "df10712f8b524ce8aea552949e074f61",
            "5e04972fd1fb435aa1868af95ec45826",
            "4ef02997094e473f90dfb37c1acb8690",
            "5d9b9a5a9581434c8870f8e79426b0d4",
            "07d31c9453204f9eb21495247b9b7eaa",
            "a88966a43a3b4258aadfb982340691ac",
            "58408fc1546447ffb4e3f8f2ef8ce1ce",
            "3906fc4524c04b1898cdeb1e54872909",
            "857a4529a6d14a71b2d9ea63ad300168",
            "fe419efad8e24a75ac97e1a386b7af86",
            "0b00eee2473e4892965ca8bf777836dc",
            "ae949cc7adbe461dadaca5b7592becb0",
            "149c1b2b56654132b4b7ff39f424d1a6",
            "7981fa086dca479da989f58f9ae659d3",
            "05ed9f383ec64fec862db44333ace00b"
          ]
        },
        "id": "-Icy_Qfjoqs5",
        "outputId": "3db6c793-805d-4210-ba16-dc15f515c41a"
      },
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased');\r\n",
        "\r\n",
        "model = BertForSequenceClassification.from_pretrained(\r\n",
        "    'bert-base-uncased',\r\n",
        "    num_labels=2,\r\n",
        "    output_attentions=False,\r\n",
        "    output_hidden_states=False\r\n",
        ");\r\n",
        "\r\n",
        "model.cuda();\r\n",
        "\r\n",
        "epochs = 2;"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "771166825e5c44edb84c5a169d2fc5d7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "902527e80b1d48e79dc9907764ff8aad",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3906fc4524c04b1898cdeb1e54872909",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lc6BqA3Bocxg",
        "outputId": "3ab474d1-6a1f-47c5-cb11-66544fd85ea0"
      },
      "source": [
        "for i in range(10):\r\n",
        "    data = load_and_process_gad_train_data(path + 'GAD/' + str(i + 1) + '/train.tsv', tokenizer=tokenizer, maxlen=128);\r\n",
        "    train_gad_model(model, data, epochs=epochs);"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "average training loss for epoch: 0.6656335345336369\n",
            "average training accuracy for epoch: 0.5767446633825944\n",
            "average validation loss for epoch: 0.5719270514117347\n",
            "average validation accuracy for epoch: 0.7025985663082437\n",
            "average training loss for epoch: 0.5498493762243362\n",
            "average training accuracy for epoch: 0.7244560755336616\n",
            "average validation loss for epoch: 0.5708223395877414\n",
            "average validation accuracy for epoch: 0.717831541218638\n",
            "average training loss for epoch: 0.49102474451065065\n",
            "average training accuracy for epoch: 0.7682060755336617\n",
            "average validation loss for epoch: 0.4801348282231225\n",
            "average validation accuracy for epoch: 0.7736111111111111\n",
            "average training loss for epoch: 0.4188929921104794\n",
            "average training accuracy for epoch: 0.8173747947454844\n",
            "average validation loss for epoch: 0.48590636220243244\n",
            "average validation accuracy for epoch: 0.7868055555555555\n",
            "average training loss for epoch: 0.39413421196596965\n",
            "average training accuracy for epoch: 0.8304084564860427\n",
            "average validation loss for epoch: 0.386714819404814\n",
            "average validation accuracy for epoch: 0.8298611111111112\n",
            "average training loss for epoch: 0.33280270724069505\n",
            "average training accuracy for epoch: 0.8683600164203613\n",
            "average validation loss for epoch: 0.4007485346661674\n",
            "average validation accuracy for epoch: 0.8354166666666667\n",
            "average training loss for epoch: 0.33728284126236324\n",
            "average training accuracy for epoch: 0.8677647783251232\n",
            "average validation loss for epoch: 0.27231811144285734\n",
            "average validation accuracy for epoch: 0.8958333333333334\n",
            "average training loss for epoch: 0.2650878833872931\n",
            "average training accuracy for epoch: 0.8980295566502463\n",
            "average validation loss for epoch: 0.2922418655620681\n",
            "average validation accuracy for epoch: 0.8840277777777777\n",
            "average training loss for epoch: 0.2587142593094281\n",
            "average training accuracy for epoch: 0.8958333333333334\n",
            "average validation loss for epoch: 0.23541439928942257\n",
            "average validation accuracy for epoch: 0.9111111111111111\n",
            "average training loss for epoch: 0.20496602697031838\n",
            "average training accuracy for epoch: 0.9248768472906403\n",
            "average validation loss for epoch: 0.32953002659810915\n",
            "average validation accuracy for epoch: 0.8763888888888889\n",
            "average training loss for epoch: 0.20603883917487803\n",
            "average training accuracy for epoch: 0.9279454022988506\n",
            "average validation loss for epoch: 0.1823159413619174\n",
            "average validation accuracy for epoch: 0.9375\n",
            "average training loss for epoch: 0.16866382099688054\n",
            "average training accuracy for epoch: 0.9389573070607554\n",
            "average validation loss for epoch: 0.1767416734662321\n",
            "average validation accuracy for epoch: 0.9402777777777778\n",
            "average training loss for epoch: 0.17719080547491708\n",
            "average training accuracy for epoch: 0.9318144499178983\n",
            "average validation loss for epoch: 0.09849742862085502\n",
            "average validation accuracy for epoch: 0.9701388888888889\n",
            "average training loss for epoch: 0.13653154992276714\n",
            "average training accuracy for epoch: 0.9511596880131363\n",
            "average validation loss for epoch: 0.11400858879917197\n",
            "average validation accuracy for epoch: 0.9583333333333334\n",
            "average training loss for epoch: 0.14391028905908268\n",
            "average training accuracy for epoch: 0.9519909688013137\n",
            "average validation loss for epoch: 0.09565248017509778\n",
            "average validation accuracy for epoch: 0.9694444444444444\n",
            "average training loss for epoch: 0.11966362505973804\n",
            "average training accuracy for epoch: 0.9642549261083745\n",
            "average validation loss for epoch: 0.12756224626468288\n",
            "average validation accuracy for epoch: 0.9597222222222223\n",
            "average training loss for epoch: 0.11473324476696906\n",
            "average training accuracy for epoch: 0.9607142857142857\n",
            "average validation loss for epoch: 0.08259317700285465\n",
            "average validation accuracy for epoch: 0.9743055555555555\n",
            "average training loss for epoch: 0.08932954708136441\n",
            "average training accuracy for epoch: 0.9726190476190476\n",
            "average validation loss for epoch: 0.11607877319135393\n",
            "average validation accuracy for epoch: 0.9666666666666667\n",
            "average training loss for epoch: 0.10611478573687019\n",
            "average training accuracy for epoch: 0.9696230158730159\n",
            "average validation loss for epoch: 0.049309914698824286\n",
            "average validation accuracy for epoch: 0.9833333333333333\n",
            "average training loss for epoch: 0.08642698555098226\n",
            "average training accuracy for epoch: 0.9744047619047619\n",
            "average validation loss for epoch: 0.06075412740206553\n",
            "average validation accuracy for epoch: 0.9840277777777777\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AxKhAhlMohGw"
      },
      "source": [
        "\"\"\"\r\n",
        "    Params:\r\n",
        "        filepath: path of the dataset\r\n",
        "        tokenizer: tokenizer to use\r\n",
        "        maxlen: maxlength of text\r\n",
        "\"\"\"\r\n",
        "def load_and_process_gad_test_data(filepath, tokenizer, maxlen=512):\r\n",
        "    # load dataset\r\n",
        "    df = pd.read_csv(filepath, delimiter='\\t', header=None, names=['wrong_index', 'sentence', 'label']);\r\n",
        "\r\n",
        "    sentences = df.sentence.values;\r\n",
        "    labels = df.label.values;\r\n",
        "    \r\n",
        "    input_ids = [];\r\n",
        "    attention_masks = [];\r\n",
        "    \r\n",
        "    for sent in sentences:\r\n",
        "        encoded_dict = tokenizer.encode_plus(\r\n",
        "            sent,\r\n",
        "            add_special_tokens=True,\r\n",
        "            max_length=maxlen,\r\n",
        "            padding='max_length',\r\n",
        "            truncation=True,\r\n",
        "            return_attention_mask=True,\r\n",
        "            return_tensors='pt'\r\n",
        "        );\r\n",
        "\r\n",
        "        input_ids.append(encoded_dict['input_ids']);\r\n",
        "        attention_masks.append(encoded_dict['attention_mask']);\r\n",
        "\r\n",
        "    # convert lists into tensors\r\n",
        "    input_ids = torch.cat(input_ids, dim=0);\r\n",
        "    attention_masks = torch.cat(attention_masks, dim=0);\r\n",
        "    labels_tensor = torch.tensor(labels);\r\n",
        "\r\n",
        "    dataset = TensorDataset(input_ids, attention_masks, labels_tensor);\r\n",
        "\r\n",
        "    return dataset;"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EnFgqv59sR7e"
      },
      "source": [
        "def test_gad_model(model, data, batch_size=32):\r\n",
        "    model.load_state_dict(torch.load('best_model.pt'));\r\n",
        "\r\n",
        "    model.eval();\r\n",
        "\r\n",
        "    ds = data;\r\n",
        "\r\n",
        "    test_dataloader = DataLoader(\r\n",
        "        ds,\r\n",
        "        sampler=SequentialSampler(ds),\r\n",
        "        batch_size=batch_size,\r\n",
        "    );\r\n",
        "\r\n",
        "    preds_list, real_labels_list = [], [];\r\n",
        "\r\n",
        "    for batch_nbr, batch in enumerate(test_dataloader):\r\n",
        "        b_input_ids = batch[0].to(device);\r\n",
        "        b_input_mask = batch[1].to(device);\r\n",
        "        b_labels = batch[2].to(device);\r\n",
        "\r\n",
        "        with torch.no_grad():\r\n",
        "            outputs = model(\r\n",
        "                b_input_ids,\r\n",
        "                token_type_ids=None,\r\n",
        "                attention_mask=b_input_mask\r\n",
        "            );\r\n",
        "        \r\n",
        "        logits = outputs['logits'].detach().cpu().numpy();\r\n",
        "        label_ids = b_labels.to('cpu').numpy();\r\n",
        "        preds = np.argmax(logits, axis=1);\r\n",
        "\r\n",
        "        preds_list.append(preds);\r\n",
        "        real_labels_list.append(label_ids);\r\n",
        "\r\n",
        "        print('finished batch {}'.format(batch_nbr));\r\n",
        "    \r\n",
        "    ret = (\r\n",
        "        preds_list,\r\n",
        "        real_labels_list\r\n",
        "    );\r\n",
        "    \r\n",
        "    return ret;"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6Qb4xU5wsUf1",
        "outputId": "8b653d6f-420c-4795-adda-aca4d7a29c85"
      },
      "source": [
        "# load test_accumulated.tsv file\r\n",
        "avg_test_accuracy = 0;\r\n",
        "\r\n",
        "test_data = load_and_process_gad_test_data(path + 'GAD/test_accumulated.tsv', tokenizer=tokenizer);\r\n",
        "\r\n",
        "ret = test_gad_model(model, test_data);"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "finished batch 0\n",
            "finished batch 1\n",
            "finished batch 2\n",
            "finished batch 3\n",
            "finished batch 4\n",
            "finished batch 5\n",
            "finished batch 6\n",
            "finished batch 7\n",
            "finished batch 8\n",
            "finished batch 9\n",
            "finished batch 10\n",
            "finished batch 11\n",
            "finished batch 12\n",
            "finished batch 13\n",
            "finished batch 14\n",
            "finished batch 15\n",
            "finished batch 16\n",
            "finished batch 17\n",
            "finished batch 18\n",
            "finished batch 19\n",
            "finished batch 20\n",
            "finished batch 21\n",
            "finished batch 22\n",
            "finished batch 23\n",
            "finished batch 24\n",
            "finished batch 25\n",
            "finished batch 26\n",
            "finished batch 27\n",
            "finished batch 28\n",
            "finished batch 29\n",
            "finished batch 30\n",
            "finished batch 31\n",
            "finished batch 32\n",
            "finished batch 33\n",
            "finished batch 34\n",
            "finished batch 35\n",
            "finished batch 36\n",
            "finished batch 37\n",
            "finished batch 38\n",
            "finished batch 39\n",
            "finished batch 40\n",
            "finished batch 41\n",
            "finished batch 42\n",
            "finished batch 43\n",
            "finished batch 44\n",
            "finished batch 45\n",
            "finished batch 46\n",
            "finished batch 47\n",
            "finished batch 48\n",
            "finished batch 49\n",
            "finished batch 50\n",
            "finished batch 51\n",
            "finished batch 52\n",
            "finished batch 53\n",
            "finished batch 54\n",
            "finished batch 55\n",
            "finished batch 56\n",
            "finished batch 57\n",
            "finished batch 58\n",
            "finished batch 59\n",
            "finished batch 60\n",
            "finished batch 61\n",
            "finished batch 62\n",
            "finished batch 63\n",
            "finished batch 64\n",
            "finished batch 65\n",
            "finished batch 66\n",
            "finished batch 67\n",
            "finished batch 68\n",
            "finished batch 69\n",
            "finished batch 70\n",
            "finished batch 71\n",
            "finished batch 72\n",
            "finished batch 73\n",
            "finished batch 74\n",
            "finished batch 75\n",
            "finished batch 76\n",
            "finished batch 77\n",
            "finished batch 78\n",
            "finished batch 79\n",
            "finished batch 80\n",
            "finished batch 81\n",
            "finished batch 82\n",
            "finished batch 83\n",
            "finished batch 84\n",
            "finished batch 85\n",
            "finished batch 86\n",
            "finished batch 87\n",
            "finished batch 88\n",
            "finished batch 89\n",
            "finished batch 90\n",
            "finished batch 91\n",
            "finished batch 92\n",
            "finished batch 93\n",
            "finished batch 94\n",
            "finished batch 95\n",
            "finished batch 96\n",
            "finished batch 97\n",
            "finished batch 98\n",
            "finished batch 99\n",
            "finished batch 100\n",
            "finished batch 101\n",
            "finished batch 102\n",
            "finished batch 103\n",
            "finished batch 104\n",
            "finished batch 105\n",
            "finished batch 106\n",
            "finished batch 107\n",
            "finished batch 108\n",
            "finished batch 109\n",
            "finished batch 110\n",
            "finished batch 111\n",
            "finished batch 112\n",
            "finished batch 113\n",
            "finished batch 114\n",
            "finished batch 115\n",
            "finished batch 116\n",
            "finished batch 117\n",
            "finished batch 118\n",
            "finished batch 119\n",
            "finished batch 120\n",
            "finished batch 121\n",
            "finished batch 122\n",
            "finished batch 123\n",
            "finished batch 124\n",
            "finished batch 125\n",
            "finished batch 126\n",
            "finished batch 127\n",
            "finished batch 128\n",
            "finished batch 129\n",
            "finished batch 130\n",
            "finished batch 131\n",
            "finished batch 132\n",
            "finished batch 133\n",
            "finished batch 134\n",
            "finished batch 135\n",
            "finished batch 136\n",
            "finished batch 137\n",
            "finished batch 138\n",
            "finished batch 139\n",
            "finished batch 140\n",
            "finished batch 141\n",
            "finished batch 142\n",
            "finished batch 143\n",
            "finished batch 144\n",
            "finished batch 145\n",
            "finished batch 146\n",
            "finished batch 147\n",
            "finished batch 148\n",
            "finished batch 149\n",
            "finished batch 150\n",
            "finished batch 151\n",
            "finished batch 152\n",
            "finished batch 153\n",
            "finished batch 154\n",
            "finished batch 155\n",
            "finished batch 156\n",
            "finished batch 157\n",
            "finished batch 158\n",
            "finished batch 159\n",
            "finished batch 160\n",
            "finished batch 161\n",
            "finished batch 162\n",
            "finished batch 163\n",
            "finished batch 164\n",
            "finished batch 165\n",
            "finished batch 166\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "czRj4lzVsUis",
        "outputId": "298029b0-012e-481a-8074-3c9bd3a82150"
      },
      "source": [
        "preds = np.concatenate(ret[0]);\r\n",
        "real = np.concatenate(ret[1]);\r\n",
        "\r\n",
        "print('test predictions: {}'.format(preds));\r\n",
        "print('real values: {}'.format(real));\r\n",
        "\r\n",
        "test_acc = accuracy_score(real, preds);\r\n",
        "test_prec = precision_score(real, preds);\r\n",
        "test_rec = recall_score(real, preds);\r\n",
        "test_f1 = f1_score(real, preds);\r\n",
        "\r\n",
        "print('test accuracy: {}'.format(test_acc));\r\n",
        "print('test precision: {}'.format(test_prec));\r\n",
        "print('test recall: {}'.format(test_rec));\r\n",
        "print('test f1: {}'.format(test_f1));"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "test predictions: [1 1 1 ... 0 0 0]\n",
            "real values: [1 1 1 ... 0 0 0]\n",
            "test accuracy: 0.9859287054409006\n",
            "test precision: 0.9785814606741573\n",
            "test recall: 0.9950017850767583\n",
            "test f1: 0.9867233138608603\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M9Ohky-l03CP"
      },
      "source": [
        ""
      ],
      "execution_count": 13,
      "outputs": []
    }
  ]
}