LongformerHotpotQA_Baseline.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Baseline_Inference_LongformerHotpotQA.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e1a76081aea746538ea69d4f4b787614": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_66ce91fe4ce14eebad559ec6b637acd8",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f52f1ce5869c42998181613e178eb563",
              "IPY_MODEL_ee8b154d6c9e4dad95fee4e15203be1a"
            ]
          }
        },
        "66ce91fe4ce14eebad559ec6b637acd8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f52f1ce5869c42998181613e178eb563": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2d82458910d442ecb4408f10c906a34f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 898823,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 898823,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c729425ec35e49c0920459155c59e211"
          }
        },
        "ee8b154d6c9e4dad95fee4e15203be1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1ac0156886f24c0fb6fbf7fc5258265c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 899k/899k [00:00&lt;00:00, 4.22MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b3436f51426c4d108b47e234f372a478"
          }
        },
        "2d82458910d442ecb4408f10c906a34f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c729425ec35e49c0920459155c59e211": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1ac0156886f24c0fb6fbf7fc5258265c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b3436f51426c4d108b47e234f372a478": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "631f420f4fdb438c9b1f72f14b2af1ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9b7a5d7c0eae4e5fb3a64fd670e96576",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7d0431a261e5447dbe3940339cca5495",
              "IPY_MODEL_7464e0123eb940dfa05356c7a8b70af1"
            ]
          }
        },
        "9b7a5d7c0eae4e5fb3a64fd670e96576": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7d0431a261e5447dbe3940339cca5495": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_35ff544a0ec64e8fbf915499d5dfafac",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 456318,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 456318,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a2e24519e5174a63bd875a31889db118"
          }
        },
        "7464e0123eb940dfa05356c7a8b70af1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f6741deaeb1740ec94609301c90f130e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 456k/456k [00:00&lt;00:00, 4.81MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_29dd956b39174fffa3b1d4958e5d7a23"
          }
        },
        "35ff544a0ec64e8fbf915499d5dfafac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a2e24519e5174a63bd875a31889db118": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f6741deaeb1740ec94609301c90f130e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "29dd956b39174fffa3b1d4958e5d7a23": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "61b93144426e42cc8c3b263d10ef882c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a21e051ade454c7bbac6f5dec4863d22",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_984cd5e29d0647a38af81edd59d3e1a4",
              "IPY_MODEL_caa9f61b46b54f5e9d416c196c2afe06"
            ]
          }
        },
        "a21e051ade454c7bbac6f5dec4863d22": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "984cd5e29d0647a38af81edd59d3e1a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2afdbd3182f54d8388e48701ba8f4e05",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 866,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 866,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a21f8658766b4f4d9b8611176dfe0aa4"
          }
        },
        "caa9f61b46b54f5e9d416c196c2afe06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_c67c9f8abc4943f986eb728c33497d2d",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 866/866 [00:00&lt;00:00, 2.76kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_98ebea51b1924544b0072127aef21b9b"
          }
        },
        "2afdbd3182f54d8388e48701ba8f4e05": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a21f8658766b4f4d9b8611176dfe0aa4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c67c9f8abc4943f986eb728c33497d2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "98ebea51b1924544b0072127aef21b9b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7d4138bb08c54c408134434c1bc22f24": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_03365c120b844e2b9937b8ffeddc6025",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4efd055fbd2a485993ca363581b3031a",
              "IPY_MODEL_a38fc080994c48ac9bd3aca34b5eff49"
            ]
          }
        },
        "03365c120b844e2b9937b8ffeddc6025": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4efd055fbd2a485993ca363581b3031a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_33a33241cfc943bfae87270208e1a5ac",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1739063632,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1739063632,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3d31e7dcfbab4bf9a9da773cfbe19dc8"
          }
        },
        "a38fc080994c48ac9bd3aca34b5eff49": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fe2d4619d3fc40ae8da726f18766cce8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.74G/1.74G [00:50&lt;00:00, 34.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fbe5febc21904c32979f6d309050380e"
          }
        },
        "33a33241cfc943bfae87270208e1a5ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3d31e7dcfbab4bf9a9da773cfbe19dc8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fe2d4619d3fc40ae8da726f18766cce8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fbe5febc21904c32979f6d309050380e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wex1IRIw5XB3"
      },
      "source": [
        "# **Baseline Model for Multi-hop Question & Answering (Longformer + HotpotQA)**\r\n",
        "The pre-trained `allenai/longformer-large-4096-finetuned-triviaqa` model checkpoint (for single-hop QA) is taken from Hugging Face for setting the baseline for the HotpotQA data set.\r\n",
        "\r\n",
        "\r\n",
        "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa<br>\r\n",
        "https://hotpotqa.github.io/index.html\r\n",
        "\r\n",
        "NOTE: Runtime is about 1.5 hours even for inferencing the answers on test set. \r\n",
        "It is an issue Tensorflow 2.0. \r\n",
        "For quick experiments, execute the notebook till Manual Inference block and change the id with random values in range (0 - 7478)\r\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vEKHaA6jV7u-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3c51dd54-19b2-4d83-d04f-24844fb66cde"
      },
      "source": [
        "!pip install -q transformers==3.5.1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 1.3MB 23.8MB/s \n",
            "\u001b[K     |████████████████████████████████| 2.9MB 57.1MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1MB 58.2MB/s \n",
            "\u001b[K     |████████████████████████████████| 890kB 56.4MB/s \n",
            "\u001b[?25h  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hEqZn8G6V-pi"
      },
      "source": [
        "import re\n",
        "import json\n",
        "import time\n",
        "import random\n",
        "import string\n",
        "import tarfile\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "from tqdm import tqdm\n",
        "from collections import Counter\n",
        "\n",
        "import tensorflow as tf\n",
        "from transformers import LongformerTokenizer, TFLongformerForQuestionAnswering"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k0CLI0pO5nG5"
      },
      "source": [
        "# Configurations\r\n",
        "MODEL = 'allenai/longformer-large-4096-finetuned-triviaqa'\r\n",
        "MAX_LEN = 512\r\n",
        "PUBLIC_ACCESS = True"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Xj-XQjNf72C",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8dd4eca7-4108-4924-dc63-78bd2e8f8da1"
      },
      "source": [
        "# TAR.GZ extractor\n",
        "def extract_tar(fname):\n",
        "  tar = tarfile.open(fname, \"r:gz\")\n",
        "  tar.extractall()\n",
        "  tar.close()\n",
        "\n",
        "# Download Files\n",
        "if PUBLIC_ACCESS:\n",
        "  !gdown --id \"1pFJ0NAvMSn7C-vI-hzeSsCG7ppJGa8EV\"\n",
        "  extract_tar('HotPotQA_EM.tar.gz')\n",
        "else:\n",
        "  extract_tar('/content/drive/MyDrive/data/hotpot/HotPotQA_EM.tar.gz')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1pFJ0NAvMSn7C-vI-hzeSsCG7ppJGa8EV\n",
            "To: /content/HotPotQA_EM.tar.gz\n",
            "221MB [00:03, 67.5MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2vdC9V33WApt"
      },
      "source": [
        "# Import Data\n",
        "with open('train_em.json') as f:\n",
        "    train_data = json.load(f)\n",
        "with open('val_em.json') as f:\n",
        "    val_data = json.load(f)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VciEYxesAaK9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4e50d9c5-d8cc-48ce-87ea-d6e75450bd93"
      },
      "source": [
        "# Print header\r\n",
        "train_data[0].keys()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['supporting_facts', 'level', 'question', 'context', 'answer', '_id', 'type'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LhAUqHNXA6D4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2d8f11f2-5111-4a01-c14f-94576a301b21"
      },
      "source": [
        "# Print a question, context and answer triplet\r\n",
        "print(train_data[0]['question'], train_data[0]['answer'], train_data[0]['context'], sep='\\n')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Who was once considered the best kick boxer in the world, however he has been involved in a number of controversies relating to his \"unsportsmanlike conducts\" in the sport and crimes of violence outside of the ring.\n",
            "Badr Hari\n",
            "['The 1998 Verano de Escándalo (Spanish for \"Summer of Scandal\") was the second annual \"Verano de Escandalo\" professional wrestling show promoted by AAA. The show took place on September 18, 1998, in Madero, Tamaulipas, Mexico. The main event featured steel cage match between the teams of Heavy Metal and Blue Demon Jr. and Kick Boxer and Abismo Negro. The stipulation of the main event was that if the team of Heavy Metal and Blue Demon Jr. lost referee Guicho Dominguez would be referee El Tirante\\'s \"slave\" for a week. If Kick Boxer and Abismo Negro lost El Tirantes would be Guicho Dominguez\\'s slave for a week.', 'Triplemanía VII was the seventh \"Triplemanía\" wrestling show promoted by AAA. The show took place on June 11, 1999, in Madero, Mexico. The Main event featured a Six-man \"Lucha Libre rules\" tag team match between the teams of Perro Aguayo, Octagón and El Cobarde II and El Texano, Perro Aguayo Jr. and Sangre Chicana. In the semi-main event Heavy Metal and El Felino defended the hair of their father, referee Pepe \"Tropi\" Casas while Kick Boxer and Thai Boxer defended the hair of referee El Tirantes. As a result, El Tirantes had his hair shaved off after the match.', 'A protection racket is a scheme whereby a group provides protection to businesses or other groups through violence outside the sanction of the law. Through the credible threat of violence, the racketeers deter people from swindling, robbing, injuring, sabotaging or otherwise harming their clients. Protection rackets tend to appear in markets where the police and judiciary cannot be counted on to provide legal protection, either because of incompetence (as in weak or failed states) or illegality (black markets).', 'Elwood Gordon Gee (born February 2, 1944) is an American academic and is currently serving his second term as President of West Virginia University. He has served as the chief executive at several universities in the United States, previously serving at Ohio State University. Gee had been heading an Ohio State-based think tank following his retirement from the Ohio State presidency on July 1, 2013. He retired in response to a series of controversies relating to comments he made, the last of which involved anti-Catholic comments allegedly made in jest about the University of Notre Dame. His resignation thus ended his second term as the president; he had previously served as president of Ohio State from 1990 to 1997.', 'Badr Hari (Arabic: بدر هاري\\u200e \\u200e ; born 8 December 1984) is a Moroccan-Dutch super heavyweight kickboxer from Amsterdam, fighting out of Mike\\'s Gym in Oostzaan. He is a former K-1 Heavyweight champion (2007—2008), It\\'s Showtime Heavyweight world champion (2009-2010) and \"K-1 World Grand Prix 2009\" finalist. Hari has been a prominent figure in the world of kickboxing and was once considered the best kickboxer in the world, however he has been involved in a number of controversies relating to his \"unsportsmanlike conducts\" in the sport and crimes of violence outside of the ring.', '\"Guerra de Titanes\" (1998) (\"War of the Titans\") was the second \"Guerra de Titanes\" professional wrestling show promoted by AAA. The show took place on December 13, 1998 in Chihuahua, Chihuahua, Mexico. The Main event featured a Steel Cage Match that highlighted two storyline feuds between Octagón and his \"Evil twin\" Pentagón and the feud between Heavy Metal and Kick Boxer as Octagón and Heavy Metal teamed together to take on Pentagón and Kick Boxer.', 'Global Fighting Championship (also known as GFC) was a UAE-based kickboxing and mixed martial arts (MMA) event. Fighters from around world on the roster include Badr Hari, Peter Aerts, Peter Graham, Dewey Cooper, Zabit Samedov. It was considered as one of the biggest kickboxing and MMA promotion in Middle East.<ref name=\"Emirates 24/7\"> </ref>', 'Outrageous Betrayal: The Dark Journey of Werner Erhard from est to Exile is a non-fiction book written by freelance journalist Steven Pressman and first published in 1993 by St. Martin\\'s Press. The book gives an account of Werner H. Erhard\\'s early life as Jack Rosenberg, his exploration of various forms of self-improvement techniques, and his foundation of Erhard Seminars Training \"est\" and later of Werner Erhard and Associates and of the Est successor course, \"The Forum\". Pressman details the rapid financial success Erhard had with these companies, as well as controversies relating to litigation involving former participants in his courses. The work concludes by going over the impact of a March 3, 1991 \"60 Minutes\" broadcast on CBS where members of Erhard\\'s family made allegations against him, and Erhard\\'s decision to leave the United States.', 'Cricket has had a number of controversies relating to players being involved with the betting aspects of the game. In particular, numerous players have been approached by bookmakers and bribed to throw matches, aspects of matches (e.g. the toss) or provide other information.', 'Prosecution of gender-targeted crimes is the legal proceedings to prosecute crimes such as rape and domestic violence. The earliest documented prosecution of gender-based/targeted crimes is from 1474 when Sir Peter von Hagenbach was convicted for rapes committed by his troops. However, the trial was unsuccessful in indicting Sir von Hagenbach with the charge of rape because the war in which the rapes occurred was \"undeclared\" and thus the rapes were only considered illegal. Gender-targeted crimes continued to be prosecuted, but it was not until after World War II when an international criminal tribunal- the International Military Tribunal for the Far East (Tokyo Tribunal)- were officers charged for being responsible of the gender-targeted crimes (particularly rape) and other crimes against humanity. Despite the various rape charges, the Charter of the Tokyo Tribunal did not make references to rape, and rape was considered as subordinate to other war crimes. This is also the situation for other tribunals that followed, but with the establishments of the International Criminal Tribunal for the former Yugoslavia (ICTY) and the International Criminal Tribunal for Rwanda (ICTR), there was more attention to the prosecution of gender-targeted crimes with each of the statutes explicitly referring to rape and other forms of gender-targeted violence.']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9JE610EBWDnz"
      },
      "source": [
        "# Convert JSON to DF\n",
        "df_train = pd.DataFrame(train_data)\n",
        "df_val = pd.DataFrame(val_data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zp2oWwRpWFzP"
      },
      "source": [
        "# Converts list of sentences to paragraph string\n",
        "def paragraph_maker(context):\n",
        "    # context = []\n",
        "    # for i in x:\n",
        "    #     context.append(''.join(i))\n",
        "    context = ' '.join(context)\n",
        "    return context\n",
        "\n",
        "# Apply paragraph maker\n",
        "df_train['text'] = df_train['context'].apply(paragraph_maker)\n",
        "df_val['text'] = df_val['context'].apply(paragraph_maker)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u0TQkm1fgSPD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 210
        },
        "outputId": "da1af769-3d08-4f3d-94a2-f5a2a0ece78e"
      },
      "source": [
        "# Print head\r\n",
        "df_train.head(3)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>supporting_facts</th>\n",
              "      <th>level</th>\n",
              "      <th>question</th>\n",
              "      <th>context</th>\n",
              "      <th>answer</th>\n",
              "      <th>_id</th>\n",
              "      <th>type</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[[Global Fighting Championship, 1], [Global Fi...</td>\n",
              "      <td>easy</td>\n",
              "      <td>Who was once considered the best kick boxer in...</td>\n",
              "      <td>[The 1998 Verano de Escándalo (Spanish for \"Su...</td>\n",
              "      <td>Badr Hari</td>\n",
              "      <td>5a832c3455429954d2e2ec41</td>\n",
              "      <td>bridge</td>\n",
              "      <td>The 1998 Verano de Escándalo (Spanish for \"Sum...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[[Malcolm Smith (American football), 3], [Supe...</td>\n",
              "      <td>easy</td>\n",
              "      <td>In which American football game was Malcolm Sm...</td>\n",
              "      <td>[Winston Lagoon ( ) is a lagoon indenting the ...</td>\n",
              "      <td>Super Bowl XLVIII</td>\n",
              "      <td>5ade025e5542997dc790711e</td>\n",
              "      <td>bridge</td>\n",
              "      <td>Winston Lagoon ( ) is a lagoon indenting the s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[[Hurt Locker (musical), 3], [Metallica, 2]]</td>\n",
              "      <td>easy</td>\n",
              "      <td>What are the names of the current members of  ...</td>\n",
              "      <td>[Joseph \"Joey\" Vera (born April 24, 1963) is a...</td>\n",
              "      <td>Hetfield and Ulrich, longtime lead guitarist K...</td>\n",
              "      <td>5ac2a5d455429921a00ab01b</td>\n",
              "      <td>bridge</td>\n",
              "      <td>Joseph \"Joey\" Vera (born April 24, 1963) is an...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                    supporting_facts  ...                                               text\n",
              "0  [[Global Fighting Championship, 1], [Global Fi...  ...  The 1998 Verano de Escándalo (Spanish for \"Sum...\n",
              "1  [[Malcolm Smith (American football), 3], [Supe...  ...  Winston Lagoon ( ) is a lagoon indenting the s...\n",
              "2       [[Hurt Locker (musical), 3], [Metallica, 2]]  ...  Joseph \"Joey\" Vera (born April 24, 1963) is an...\n",
              "\n",
              "[3 rows x 8 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J3T7wCC-v_Gr"
      },
      "source": [
        "# Divide data according to difficulty levels\r\n",
        "df_val_easy = df_val.loc[df_val['level'] == 'easy']\r\n",
        "df_val_medium = df_val.loc[df_val['level'] == 'medium']\r\n",
        "df_val_hard = df_val.loc[df_val['level'] == 'hard']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8vJjivL9YAxm"
      },
      "source": [
        "## Plot Token Count Distribution"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MqiO1p3CX7Q1"
      },
      "source": [
        "# Extract number of tokens per doc\r\n",
        "sizes = []\r\n",
        "for i in df_train['text'].to_list()+df_val['text'].to_list():\r\n",
        "    sizes.append(len(i.split()))\r\n",
        "sort_size = sorted(sizes)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 530
        },
        "id": "Xff6vLZVsP4-",
        "outputId": "76cfd761-d27d-4819-d87f-f7aae9265d02"
      },
      "source": [
        "# Plot token distribution\r\n",
        "plt.figure(figsize=(12,8))\r\n",
        "plt.title('Token Distribution for Context')\r\n",
        "plt.xlabel('Sample IDs sorted in increasing order of token counts')\r\n",
        "plt.ylabel('Token Counts')\r\n",
        "plt.plot(sort_size)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7fa41813c320>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtoAAAHwCAYAAACYMcj+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXicV333//fXlhetXiRZXmXZsZ3E2YlJSMMSIAlpgAZaoCxlK5T2KRT6FPpAS1sotJD2R2nLA21JC5QlEGhZmoemQNgJIYuzb068xPumxZa1WLKW8/tjbimy40UJM5oZ6f26Ll2aue977vmO4yv5zMn3nBMpJSRJkiTl17RiFyBJkiRNRgZtSZIkqQAM2pIkSVIBGLQlSZKkAjBoS5IkSQVg0JYkSZIKwKAtaVKKiMsiYmex6wCIiD+NiH/L4/26I2Jl9vjfI+Kv8njvf4mIP8/X/cbcNyLicxFxICLuyPf9JakUGbQllbwsWI78DEfE4THPX1fk2n4cEX0R0RURhyLiroh4X0TMGrkmpfSRlNJbx3mvU16XUqpJKW3JQ+1viohbjrn376WUPvzL3vs4ng1cASxNKV2UjxtGxMyI+GBEbIyInojYGhGfjYiWPNx7a0Rc/stXmd97SSovBm1JJS8LljUppRpgO/DSMceuL3Z9wDtSSrXAIuDdwKuBmyIi8vkmEVGRz/tNsOXA1pRSz1N94Uk+938Cvwa8FpgDnAfcBbzw6RYpSflk0JZUtiJiVkT8Q0Tszn7+YexI8jHXvjMiHo6IpdnrPhYR2yNiX9YuUZldd1lE7IyId0fE/ojYExFvHk89KaWelNKPyYW/S4AXZ/f8YER8KXs8OyK+FBHtEXEwIu6MiKaI+GvgOcAns5H6T2bXp4h4e0RsBDaOObZqzFs3RMTN2aj6TyJieXZdS3btaFAdGTWPiDOBfwEuyd7vYHb+qFaUiPidiNgUER0RcWNELB5zLkXE72Ujygcj4lPH+3IREW8B/m3Me/3lOO991Oc+5p6XkxshvyaldGdKaTCl1JlS+lRK6TPZNYuz+3Zk7/M7Y17/wYj4WkR8Iftzeygi1mXnvgg0A/8vq/f/ZMefFRG3Zp/1voi4LDv+KxHRFhHLsufnRa5F5owT3UvS1GDQllTO3g88Czif3GjmRcCfHXtRRPwF8CbgeSmlncC1wJrsdauAJcBfjHnJQnIjpEuAtwCfioh54y0qpbQdWE8uOB/rjdm9lwH1wO8Bh1NK7wd+Rm50vCal9I4xr3kZcDGw9gRv+Trgw0ADcC9wylH+lNIj2Xv/Inu/ucdeExEvAD4KvIrcaP024IZjLnsJ8Ezg3Oy6Fx3nvT5zzHt9YJz3Ptnnvhy4I6W04yQf8wZgJ7AYeAXwkex9R/xads1c4Ebgk1m9r+fo/3PytxGxBPhv4K+A+cB7gK9HRGNK6Vbg08Dnsy9sXwL+PKW04Xj3Okm9kiYZg7akcvY64EMppf0ppVbgL4HXjzkfEfFx4Erg+Sml1mzE9W3A/04pdaSUuoCPkGv3GDGQ3XcgpXQT0A2c/hRr200ukB1rgFzAXpVSGkop3ZVSOnSKe300q/XwCc7/d0rppymlfnJfPi4ZGV39Jb0O+GxK6e7s3n+S3btlzDXXppQOZl8ufkTuy0u+7n2yz10P7DnRzbPPfynw3pRSX0rpXnKj6m8Yc9ktKaWbUkpDwBfJfVk7kd8CbsquH04p3Uzuy9TV2fkPkvsCdQewC/jUSe4laYowaEsqZ4vJjYSO2JYdGzGXXKj+aEqpMzvWCFQBd2UtAAeB72THR7SnlAbHPO8Fap5ibUuAjuMc/yLwXeCGrN3lbyNixinudbJR26POp5S6s/ddfOLLx+2oP9/s3u3kPtuIvWMeP5U/p/Hc+2Sfu53cSPjJ7j/yRWrENk5e++yT9IMvB1458ncm+3vz7JEaUkoDwL8DZwN/l1JKJ6lN0hRh0JZUznaTC0AjmrNjIw6Qa234XERcmh1rAw4DZ6WU5mY/c7KJlnmRjaZeSK4V5CjZKPlfppTWAr+S1TcyynqicHaq0DY6eh0RNeRG0ncDIxMPq8Zcu/Ap3PeoP9+IqCY3krzrFK8bj/Hc+2T1fR+4KCKWnuT+8yOidsyxZsZf+7HvvQP44pi/M3NTStUppWuz+pcAHwA+B/xdHD1XwNAtTVEGbUnl7CvAn0VEY0Q0kOuz/tLYC7LJia8DvhERF6WUhoF/Bf4+IhZALiRFxJN6i5+qiKiKiOcB/0WuheCm41zz/Ig4JyKmA4fItZIMZ6f3ASufxltfHRHPjoiZ5Hq1b0sp7cjaaXYBvxUR0yPit4HTxrxuH7A0e93xfAV4c0ScnwXHjwC3p5S2Po0a83rvlNL3gZuBb0bEhRFRERG12eTM3856t28FPhq5Cajnkuu3/9LJ7jvGsf8svgS8NCJelP1Zzo7cxNmlWTvSvwOfyd5jD7l/Die6l6QpwqAtqZz9Fbk+2fuBB4C7s2NHyfppf5vcyg/PAN4LbAJui4hD5EZHn2oP9lifjIgucoHqH4CvA1dlof5YC8ktS3cIeAT4Cbl2EoB/BF6RrVjxiafw/l8mN5raQW4k/bfGnPsd4I/JtVqcRS58jvgh8BCwNyLajr1pFmb/PPs8e8iF9Fcfe93Tkad7v4Lcl5mvAp3Ag8A6cv88AV4DtJAb3f4m8IHsfcfjo+S+xB2MiPdkwf0a4E+BVnIj3H9M7r+j7wQWkJsAmYA3k/sS8Zzj3espfkZJZSxsI5MkSZLyzxFtSZIkqQAM2pIkSVIBGLQlSZKkAjBoS5IkSQVg0JYkSZIK4EQ7YJW1hoaG1NLSUuwyJEmSNMndddddbSmlxuOdm5RBu6WlhfXr1xe7DEmSJE1yEbHtROdsHZEkSZIKwKAtSZIkFYBBW5IkSSoAg7YkSZJUAAZtSZIkqQAM2pIkSVIBGLQlSZKkAjBoS5IkSQVg0JYkSZIKwKAtSZIkFYBBW5IkSSoAg7YkSZJUAAZtSZIkqQAM2pIkSVIBGLQlSZKkAjBoS5IkSQVg0JYkSVLZGhpOdPYOcGRwuNilPIlBW5IkSWVr54FezvvQ9/j2/buLXcqTGLQlSZJUtlIqdgUnZtCWJElS2RrJ2RFFLeO4DNqSJEkqWykb0g5KL2kbtCVJklS2HNGWJEmSphiDtiRJksqWkyElSZKkgsh6tEuwd8SgLUmSpLI1MqJdejHboC1JkqQy5mRISZIkqQCeGNEuvaRt0JYkSVLZc0RbkiRJyqNE6S47YtCWJElS2XIypCRJklQAo0G7BJO2QVuSJEll64nWkdJL2gZtSZIklT1HtCVJkqQ8cgt2SZIkqYBKcEDboC1JkqTy9cRkyNKL2gZtSZIkla2RyZClF7MN2pIkSSpjLu8nSZIkFZBBW5IkScqjEl50xKAtSZKk8pXSSI926Q1pG7QlSZJUtkZHtEsvZxu0JUmSVL5GJ0MWt4zjMmhLkiSp7LmOtiRJkpRXpTsd0qAtSZKksmXriCRJklQAI+PZJdg5YtCWJElS+XpiRLv0krZBW5IkSWVrdB3t0svZBm1JkiSVvxLM2QZtSZIkla/SXXPEoC1JkqQylkZnQxa1jOMyaEuSJKlspWxM28mQkiRJUj6NrDpSejnboC1JkqTyV4I5u3BBOyKWRcSPIuLhiHgoIt6VHf9gROyKiHuzn6vHvOZPImJTRDwaES8ac/yq7NimiHhfoWqWJElSeSnlyZAVBbz3IPDulNLdEVEL3BURN2fn/j6l9LGxF0fEWuDVwFnAYuD7EbEmO/0p4ApgJ3BnRNyYUnq4gLVLkiSpDIxuWFOCvSMFC9oppT3AnuxxV0Q8Aiw5yUuuAW5IKfUDj0fEJuCi7NymlNIWgIi4IbvWoC1JkjTFjU6GLL2cPTE92hHRAlwA3J4dekdE3B8Rn42IedmxJcCOMS/bmR070fFj3+NtEbE+Ita3trbm+RNIkiSpFD2xBXvpKXjQjoga4OvAH6aUDgH/DJwGnE9uxPvv8vE+KaXrUkrrUkrrGhsb83FLSZIklbjRZbRLMGkXskebiJhBLmRfn1L6BkBKad+Y8/8KfDt7ugtYNublS7NjnOS4JEmSRCmOaRdy1ZEAPgM8klL6+Jjji8Zc9nLgwezxjcCrI2JWRKwAVgN3AHcCqyNiRUTMJDdh8sZC1S1JkqTykVLprjtSyBHtS4HXAw9ExL3ZsT8FXhMR55Mb6d8K/C5ASumhiPgauUmOg8DbU0pDABHxDuC7wHTgsymlhwpYtyRJksrElGwdSSndwvHH8G86yWv+Gvjr4xy/6WSvkyRJ0hQ1lSdDSpIkSYXyxPJ+pRe1DdqSJEkqe6UXsw3akiRJKmMlPBfSoC1JkqTy9cQW7MWt43gM2pIkSSpbo6uOlGDziEFbkiRJZWtkHW1HtCVJkqQ8KuEWbYO2JEmSyp8j2pIkSVIeueqIJEmSVBBZj7aTISVJkqT8cXk/SZIkqQBGl/czaEuSJEn5Z+uIJEmSlEdOhpQkSZIKYDhL2tNKb0DboC1JkqTyNRq0SzBpG7QlSZJUtp4Y0TZoS5IkSXkzNJz7Pd2gLUmSJOXPE60jRS7kOEqwJEmSJGl8hodtHZEkSZLybigb0Z7uZEhJkiQpfxzRliRJkgogy9mOaEuSJEn5NDTshjWSJElS3rlhjSRJklQAI0HbdbQlSZKkPBrZsMbJkJIkSVIeuWGNJEmSVAAjkyFtHZEkSZLyaHRE26AtSZIk5c/ohjWuOiJJkiTlz1BKJblZDRi0JUmSVMaGU2n2Z4NBW5IkSWVseDhRojnboC1JkqTyNTRs64gkSZKUd7aOSJIkSQUwnGwdkSRJkvLO1hFJkiSpAFzeT5IkSSqAlBJRor0jBm1JkiSVrYGhxAxHtCVJkqT8GhwapmJ6aUba0qxKkiRJGoeB4cSM6Y5oS5IkSXk1MDjMDEe0JUmSpPwaHE5UOKItSZIk5dfA0DAV00oz0pZmVZIkSdI4DA4lZto6IkmSJOXX4PCwrSOSJElSvg0MJZf3kyRJkvJtYGjYDWskSZKkfBscSi7vJ0mSJOXbgD3akiRJUv45oi1JkiQVQP/gkMv7SZIkSfnWNzDM7BmlGWlLsypJkiRpHPoGhpg9Y3qxyzgug7YkSZLKUkqJ/sFhZhm0JUmSpPzpHxwGsHVEkiRJyqe+gSEAZlc4oi1JkiTlTd/AyIi2QVuSJEnKm9ERbVtHJEmSpPzpGxwJ2lNsRDsilkXEjyLi4Yh4KCLelR2fHxE3R8TG7Pe87HhExCciYlNE3B8Rzxhzrzdm12+MiDcWqmZJkiSVjydaR0pz7LiQVQ0C704prQWeBbw9ItYC7wN+kFJaDfwgew7wq8Dq7OdtwD9DLpgDHwAuBi4CPjASziVJkjR1PdE6MsVGtFNKe1JKd2ePu4BHgCXANcDns8s+D7wse3wN8IWUcxswNyIWAS8Cbk4pdaSUDgA3A1cVqm5JkiSVh5Hl/WZVTL0R7VER0QJcANwONKWU9mSn9gJN2eMlwI4xL9uZHTvRcUmSJE1hR7KgPXP6FBvRHhERNcDXgT9MKR0aey6llICUp/d5W0Ssj4j1ra2t+bilJEmSStjAUBa0p+KIdkTMIBeyr08pfSM7vC9rCSH7vT87vgtYNublS7NjJzp+lJTSdSmldSmldY2Njfn9IJIkSSo5IyPaM6ZHkSs5vkKuOhLAZ4BHUkofH3PqRmBk5ZA3Av815vgbstVHngV0Zi0m3wWujIh52STIK7NjkiRJmsL6S3x5v4oC3vtS4PXAAxFxb3bsT4Frga9FxFuAbcCrsnM3AVcDm4Be4M0AKaWOiPgwcGd23YdSSh0FrFuSJEllYGR5v1KdDFmwoJ1SugU40Tj+C49zfQLefoJ7fRb4bP6qkyRJUrkr9RHt0oz/kiRJ0in0l/iIdmlWJUmSJJ1C3+AQFdOCiumlGWlLsypJkiTpFA4fGS7ZthEwaEuSJKlM9fQPUjOrkGt7/HIM2pIkSSpL3f2D1Mw2aEuSJEl51d0/SLUj2pIkSVJ+dfcPUmvQliRJkvKru88ebUmSJCnv7NGWJEmSCqDbVUckSZKk/EopGbQlSZKkfOsbGGZoONk6IkmSJOVTV/8AgCPakiRJUj519w0CBm1JkiQpr3r6hwCDtiRJkpRXo60j9mhLkiRJ+WPriCRJklQA3f0GbUmSJCnvekaCtq0jkiRJUv50OaItSZIk5V933yAzpgezKko3zpZuZZIkSdIJjGy/HhHFLuWEDNqSJEkqO919g1SXcNsIGLQlSZJUhkZGtEuZQVuSJEllp7t/kNoSXnEEDNqSJEkqQ45oS5IkSQVgj7YkSZJUALaOSJIkSQVg64gkSZKUZ0PDid4jQ9TMmlHsUk7KoC1JkqSy0p1tv149a3qRKzk5g7YkSZLKSmfvAACVMw3akiRJUt509B4BoHa2rSOSJElS3rR29QPQUl9V5EpOzqAtSZKksrJhzyEAGmpmFbmSkzNoS5IkqaxsaesBYGHd7CJXcnIGbUmSJJWVkTW0p02LYpdyUgZtSZIklZW7th3gwuXzil3GKRm0JUmSVDZSShw+MsTsGaUfY0u/QkmSJCnT3nOEwwNDXNDsiLYkSZKUN7sOHAagscRXHAGDtiRJksrIY/u6ADhtQU2RKzk1g7YkSZLKRnf/IADN80t7sxowaEuSJKmM3LP9INMC5lWV9vbrYNCWJElSGdl3qI/KGdOJKO01tMGgLUmSpDLy8O5DnLVkTrHLGBeDtiRJkspC38AQXf2DrKivLnYp42LQliRJUlkYWXHkvGVzi1zJ+JwyaEfEaRExK3t8WUS8MyLK49NJkiRp0nhw1yEAzl06eVpHvg4MRcQq4DpgGfDlglYlSZIkHeOh3Z0ArCqDNbRhfEF7OKU0CLwc+L8ppT8GFhW2LEmSJOloj+7tonZWBbNnTC92KeMynqA9EBGvAd4IfDs7VvoLF0qSJGlSeXB3J6ubymM0G8YXtN8MXAL8dUrp8YhYAXyxsGVJkiRJT+joOULfwDBnLS6P/myAinFcc0VK6Z0jT7Kw3VfAmiRJkqSj/GxjKwDrWuYVuZLxG8+I9huPc+xNea5DkiRJOqEfbdgPwGVrFhS5kvE74Yh21pf9WmBFRNw45lQt0FHowiRJkqQRt23pYG7VDOZUlc9UwZO1jtwK7AEagL8bc7wLuL+QRUmSJEkjOnqOsPdQHy+/YEmxS3lKThi0U0rbgG3kJkJKkiRJRfGfd+0A4Mq1TUWu5KkZz86Qvx4RGyOiMyIORURXRByaiOIkSZKk7zy4F4DLyyxoj2fVkb8FXppSeqTQxUiSJEljpZS4e/tBzls2lxnTx7OOR+kYT7X7DNmSJEkqhru3HwDgsjWNRa7kqRvPiPb6iPgq8C2gf+RgSukbBatKkiRJAr58e64/+2VlNhESxhe064Be4MoxxxJg0JYkSVJB3bq5jenTghUN1cUu5Sk7ZdBOKb356dw4Ij4LvATYn1I6Ozv2QeB3gNbssj9NKd2UnfsT4C3AEPDOlNJ3s+NXAf8ITAf+LaV07dOpR5IkSeWlvbufPZ19vOTcRcUu5Wk5ZdCOiM+RG8E+Skrpt0/x0n8HPgl84Zjjf59S+tgx77EWeDVwFrAY+H5ErMlOfwq4AtgJ3BkRN6aUHj5V3ZIkSSpv1/1sCwDXnF9+bSMwvtaRb495PBt4ObD7VC9KKf00IlrGWcc1wA0ppX7g8YjYBFyUnduUUtoCEBE3ZNcatCVJkiaxlBKfv3UrAJefWT7bro81ntaRr499HhFfAW75Jd7zHRHxBmA98O6U0gFgCXDbmGt2ZscAdhxz/OLj3TQi3ga8DaC5ufmXKE+SJEnF9uNHW+kbGOZV65YSEcUu52l5OosRrgae7teKfwZOA84nt73735388vFLKV2XUlqXUlrX2Fh+y79IkiQpJ6XEH371XgD++EVnFLmap288Pdpd5Hq0I/u9F3jv03mzlNK+Mff9V55oS9kFLBtz6dLsGCc5LkmSpEnoP+/aSefhAa45fzGNtbOKXc7TNp7Wkdp8vVlELEop7cmevhx4MHt8I/DliPg4ucmQq4E7yIX71RGxglzAfjXw2nzVI0mSpNLzse89CsBHf/2cIlfyyxnPZEgi4teA52ZPf5xS+vbJrs9e8xXgMqAhInYCHwAui4jzyY2MbwV+FyCl9FBEfI3cJMdB4O0ppaHsPu8Avktueb/PppQeGvenkyRJUlm5a1sH+w71c9npjVTNHFdULVmR0pNW7jv6gohrgWcC12eHXgPcmVL60wLX9rStW7curV+/vthlSJIk6Sm6/OM/YdP+bn78nstoKYNNaiLirpTSuuOdG8/XhKuB81NKw9nNPg/cA5Rs0JYkSVL5+fmmNjbt7+aZLfPKImSfynhXHZk75vGcQhQiSZKkqWtoOPG6f7sdgI+98rwiV5Mf4xnR/ihwT0T8iNzkxOcC7ytoVZIkSZpSrv2fRwB45wtXs7y+/EezYXyrjnwlIn5Mrk8b4L0ppb0FrUqSJElTRmfvAP/6s8eZFvCuF64udjl5c8KgHREvAmpTSv+ZLcl3Y3b8FRHRmVK6eaKKlCRJ0uT1hs/dAcDf/Ma5TJ9WnrtAHs/JerT/AvjJcY7/GPhQQaqRJEnSlHLTA3u4b8dBls2v5JXrlp36BWXkZEF7Vkqp9diDKaU2YHI0zkiSJKloDvYe4fevvxuAb/yvS4tcTf6dLGjXRcSTWksiYgZQWbiSJEmSNNkNDg3zm5++DYAPv+zsst5q/UROFrS/AfxrRIyOXkdEDfAv2TlJkiTpafn96+/m0X1dvPicRbz+WcuLXU5BnCxo/xmwD9gWEXdFxF3A40Brdk6SJEl6yt7/zQf43sP7uHD5PD752guKXU7BnHDVkZTSIPC+iPhLYFV2eFNK6fCEVCZJkqRJ54u3beP627czr2oGX/mdZxExeVYZOdZ41tE+DDwwAbVIkiRpErt9Szt//q0HAfjRey5jZsV4NykvT5P700mSJKkk9A0M8ZvX5SY/fvsPns3cqplFrqjwDNqSJEkqqJQSr85C9v+56nTOXjKnyBVNjFO2jgBExBJg+djrU0o/LVRRkiRJmjzedcO93LvjIL969kJ+/7JVp37BJHHKoB0RfwP8JvAwMJQdToBBW5IkSSf1yR9u5Mb7dnP+srl86rXPKHY5E2o8I9ovA05PKfUXuhhJkiRNHl+/aycf+95jVEwLvvTWi5k2bfKuMHI84+nR3gLMKHQhkiRJmjy+8IutvPs/7qNudgW3vPcF1MwaV8fypDKeT9wL3BsRPwBGR7VTSu8sWFWSJEkqSykl/v7mx/jEDzcxt2oG//3O57Bwzuxil1UU4wnaN2Y/kiRJ0gkNDg3zh1+9l2/fv4eLWubz6ddfyLzqyb+M34mMZ8Oaz0dEJdCcUnp0AmqSJElSmdnTeZi3fn49D+0+xOsububD15w95Xqyj3XKHu2IeClwL/Cd7Pn5EeEItyRJkgC4dXMbV378pzy0+xDvv/pM/vrl50z5kA3jax35IHAR8GOAlNK9EbGygDVJkiSpDAwPJz72vUf5px9vpm52Bf/5e5ewrmV+scsqGeMJ2gMppc6Io76VDBeoHkmSJJWBx/Z18ZbP38mOjsOcv2wu173hQhbUTs1JjycynqD9UES8FpgeEauBdwK3FrYsSZIklaIjg8N87HuPct1PtzAt4M9efCZvefYKjhmUFeML2n8AvJ/c0n5fBr4HfKiQRUmSJKn0rN/awe9ffzf7u/o5d+kc/uE3z2dlY02xyypZ4wnar0kpvZ9c2AYgIq4F3lewqiRJklQyDvYe4b1fv5/vPrSPmRXT+JvfOIfffGZzscsqeeMJ2r8REX0ppesBIuKTQGVhy5IkSVIpuOGO7Xz0fzbQeXiAF5+ziL946Vqa6uzFHo9xBW3gxogYBq4CDqaU3lLYsiRJklRMt29p58++9SAb93dTO7uCz73pmTz/jAXFLqusnDBoR8TYtVneCnwL+DnwlxExP6XUUejiJEmSNLEO9h7hgzc+xLfu3Q3AH7xgFe964Woqpp9y+xUd42Qj2ncBCYgxv1+c/STAtbQlSZImiT2dh/nEDzbylTt2AHD5mU38zW+cQ33NrCJXVr5OGLRTSismshBJkiRNvAM9R/jwtx/mG/fsAuCC5rm8+4rTefbqhiJXVv5O2aMdETOA/wU8Nzv0Y+DTKaWBAtYlSZKkAursHeDvv/8YX/jFVoYTPGd1A390xRouaJ5X7NImjfFMhvxnYAbwT9nz12fH3lqooiRJklQYm/Z38+mfbOY/7toJwNlL6viTXz2TS1c5gp1vJ5sMWZFSGgSemVI6b8ypH0bEfYUvTZIkSfmyo6OXj9/8GN/MWkSe0TyX/33FGp6zurHIlU1eJxvRvgN4BjAUEaellDYDRMRKYGgiipMkSdLTl1Lihxv28+Xbt/ODDfsBeN6aRv78JWeyakFtkaub/E4WtEc2rH8P8KOI2JI9bwHeXMiiJEmS9PR19g7wrz/bwg13bqet+wgAV6xt4g8vX81Zi+cUubqp42RBuzEi/ih7/GlgevZ4CLgA+FEhC5MkSdJTc8/2A3zhF9tG20OWzK3kj65Yw5subaFu9owiVzf1nCxoTwdqeGJke+xr/H8NkiRJJaCrb4Cv3rmDL922ja3tvQBcvGI+v//8VTxvjf3XxXSyoL0npfShCatEkiRJ4zI8nPjew/v4xt07+d7D+wComVXB7z3vNH7rWc0snVdV5AoF4+vRliRJUgm4a9sB/mP9Dr517y76BoYBeO6aRl79zGVcddZCpk0zvpWSkwXtF05YFZIkSTqu/V19fPEX2/ive3ezvSPXGnLOkjlcc/5iXn1RMzWzxrMtiorhZFuwd0xkIZIkScrZ1t7Djffu5uZH9nH/zk4AFtbN5neft5I3XNLCkrmVRa5Q4+FXIEmSpBKwaX8X37h7Fzc9sGd0UmPt7Ap+4xlLeeW6pTxrZX2RK9RTZdCWJEkqgpQS9+w4yDfu3slPH2sbbQtZNGc2v33pCl563iLOXzaXCPuuy5VBW5IkaYJ09Q3wPw/s5SePtfKzja0c6hsEYJbULAYAACAASURBVNWCGn770hW84sKlrF1cV+QqlS8GbUmSpALa03mYG+/dzY8e3c9tW56YAnfh8nk8b00jv/6MJS7HN0kZtCVJkvJocGiYWza18YNH9vPzTW1saesBYPaMaVx11kJect4irljbxKyK6ae4k8qdQVuSJOmXMDycuHNrB7dubufnm9pYv+3A6LmVDdW84ZLlXH3OIi5qme8611OMQVuSJOkpSCmxpa2Hmx/ex62b27l1UxuDwwmAuVUzePG5i7hkZT1Xrm1iQd3sIlerYjJoS5IknULn4QF+uGEfP9yQm8R4sHcAgJkV07h0VQPPWd3AC85YwMrGmiJXqlJi0JYkSTrG/kN93LKpjdu2tPOLLe3s6Dg8eu4ZzXN5zupGnn/GAs5dMsd2EJ2QQVuSJE15bd39fP/hffxsYxt3bO2gtat/9NwZC2t5wyXLuWjFfF54RhOVM53EqPExaEuSpCklpcTB3gFuf7ydnzzWxs83PbFZDMAFzXN55YVLOX9ZbuTaYK2ny6AtSZImtcGhYW7d3M492w9y1/YDPLirk46eI6PnVy+o4U2/0sKlqxp47poGl91T3hi0JUnSpHKob4BfbG7nlo1t3Lm1gw17u0bP1c6q4Jylc7igeS6nL6zjuasbmFs1s4jVajIzaEuSpLKVUmJza/foiPUdj3ew6+ATExdb6qv49Wcs4aKW+VxyWj3L66uLWK2mGoO2JEkqCykl9h7q447HO1i/9QD37zzIfTs7j7rmnCVzeNFZCzlv2Ryeu7qRedWOVqt4DNqSJKkkdfcPcsvGVu7efpCHdx/inu0H6DkyNHp+2fxKfu28xZy1uI4Ll8/jnKVz7K9WSTFoS5KkousbGOK2Le3cv7OT+3YcZMPerqNaQOZVzeCZK+Zz3tK5nL1kDpecVk/NLGOMSpt/QyVJ0oTq6R/kF5vbeXB3Jw/s7HxSqK6eOZ0zF9VxxdomLmieawuIypZBW5IkFcy29h7u2naAB3Z18ujeLh7b101b9xObwdTMquD0hbVcsbaJsxbXcfGKeprrq4pYsZQ/BQvaEfFZ4CXA/pTS2dmx+cBXgRZgK/CqlNKBiAjgH4GrgV7gTSmlu7PXvBH4s+y2f5VS+nyhapYkSU/P8HDiwd2dPLjrEBv2HuKRPYe4d8dBBobS6DWNtbM4e0kdqxfUsHZxHRetqGfJ3MoiVi0VViFHtP8d+CTwhTHH3gf8IKV0bUS8L3v+XuBXgdXZz8XAPwMXZ8H8A8A6IAF3RcSNKaUDBaxbkiSdRP/gEPfvzLV9PLwnN0lxc2vPUdcsnVfJC8/IjVKf3zyXC5rn2VOtKadgf+NTSj+NiJZjDl8DXJY9/jzwY3JB+xrgCymlBNwWEXMjYlF27c0ppQ6AiLgZuAr4SqHqliRJOcPDiYf3HOKxfbmWj037u9iwt4udBw4fdd2Zi+p4xYVLOW/pHNYurmPtojluWy4x8T3aTSmlPdnjvUBT9ngJsGPMdTuzYyc6LkmS8mjXwcPZxMRc28eje7vY2t571DVzKmdwWmM1l5/ZxBkLazl7yRxOX1jLjOnTilS1VNqK9v9wUkopItKprxyfiHgb8DaA5ubmfN1WkqRJI6XEpv3dPLznEJv2d7OlrYdt7T08urfrqF7qyhnTWdNUw2suWsaZi+pY01TL2sV11M2eUcTqpfIz0UF7X0QsSintyVpD9mfHdwHLxly3NDu2iydaTUaO//h4N04pXQdcB7Bu3bq8BXhJkspN5+EBNuw5xGP7u9na1sPG/d083tbNjo6jWz6qZk5nRUM1V561kNObajlzUR3nLp1DU93sIlUuTS4THbRvBN4IXJv9/q8xx98RETeQmwzZmYXx7wIfiYh52XVXAn8ywTVLklSS9h3qY8PeLra29fDovi427e/msX1dHOwdOOq6hpqZnLGwjmevamDVglrWNNWwdlEd9TWzilS5NDUUcnm/r5AbjW6IiJ3kVg+5FvhaRLwF2Aa8Krv8JnJL+20it7zfmwFSSh0R8WHgzuy6D41MjJQkaSoYGk48ureLLW250enNrT1s2t/No/u6ODI4fNS1y+ureEbzPNY01bJ6QQ2nL6xl1YIaZs9wYqJUDJFb6GNyWbduXVq/fn2xy5Akadw6eo5w386DPN6a65ve1NrNltYe9nT2HXVdxbTgtMYaVi3I/Zy5qI4VDdW0NFQxq8JALU20iLgrpbTueOdc0FKSpAkyPJx4bH+uxWNr28jIdK5/um/g6NHpRXNms2pBDVeubWJlYw2nNdawuqnG/mmpjBi0JUnKs/bufh7Y1cnGfbmVPba29bC9o5ddB4+ejDh9WrB6QQ1Xrl1IS0M1pzVWc+7SuSyZW8nMCpfMk8qdQVuSpKfhYO8RNrf2sDlbJu/xtm62tfeyrb2XwwNDR127bH4lKxuruez0xtGWj9ULalk4x9FpaTIzaEuSdAK9RwbZ1t7LA7s62by/m8ezkemt7T1PavWonV3BysYarjyrieXzq1jdVMuZi2pZ2VDDtGlRpE8gqZgM2pKkKa1vYIgtrbnJh5v3d7OtvYedBw7zeFsP7T1Hjrp2XtUMmuurueqshTTXV9NSX8WKhmpWLaih1s1cJB3DoC1JmvS6+wfZ0pobkR4J0dvae9jW3sv+rv6jrp1ZMY3l86u4oHlutppHNac11nD2kjnUzPI/m5LGz39jSJLK3vBwYseBXra297Ln4GF2HTzMtvZedh88zJa2HjqOGZmeFtDSUM1Zi+t4cUM1KxtrWNlQzRkLa93ERVLeGLQlSWVhJExv3NfN1vae0X7pkVHqY1XNnM6yebkNXFY2VrMyG51e0VDtEnmSJoRBW5JUMgaGhtl98DBb23tzkw7bcu0dW9pym7cca8ncSpbNr+R5axpZ0VDNsvlVtNRXs2jubOrsmZZUZAZtSdKEOtBzhE2t3ew8kFsKbyRQb+84TFt3/5Oub6iZSfP8Kl61bimrF9TSkk0+XD6/ytU8JJU0g7YkKa+GhxN7DvWNjkZvbc9NPNzRcZit7T30Hjl6jemZ06fRXF/F+cvmsHReFUvnVdI8v4pl83Mresye4bbiksqTQVuS9JSNrC+9ubWb7R297DxwmC2t3ew6eJgdHU/ul55fPZNl8yp5wRkLRpfDWzovF6TnV88swieQpMIzaEuSniSlxN5DfTze1sOeg33sONDLltYetrbnJiAe7B140muWza+kpb6a56xupKW+isVzK1mRTT6smul/biRNPf6bT5KmqEN9A2zPeqR3ZKt35Eake9nW0UtKR18/s2IaKxuquWRl/WhrR0t9NactqGbRnMrifAhJKmEGbUmaxA4fGWJ7Ry8b9h5iS2sPW7KNWra29XCob/BJ1y+bX8nSuVVcvKKe5voqltdX0Ty/iqXzqmzxkKSnyKAtSWWsb2CIPZ19bGntZkdHL7sOHmZ7R+/oZi3HhulZFdNY2VjDpasaaK6vYmVDNUvn5QL1krmVRLiKhyTli0FbkkpcZ+8Am1q72NKa26RlWzb5cEdH75N2PASYUzmDloZqnnf6ApbMrWTJvEpWNlRz5qI6R6UlaQIZtCWpyIaHE/u6+ti8/4nJhtvbc33S246zHF7trAqWza/iopb5LJlXydJ5lSybV0VLQzUt9VVUTJ9WpE8iSRrLoC1JE2DsxMMt2ZJ4uw/2jU5APNbcqhk0z6/i8jObaJ5fNbp1+JqmGmrd8VCSyoJBW5Ly4MjgMHs6D7OlrSe3akd7bhWPkb7pY0elpwWj60g/d01jtopHJSsbajhtQTWzKtykRZLKnUFbksYhpURrVz+Pt/Wwu/MwOzsOs2PMFuJ7Ovue9JqGmlmsaKjirMV1LJ5byfL6KpbNq2JVUw0LamcX4VNIkiaSQVuSMp2HB7Le6B52HjjMrgOH2dLWzZ7OPra19zI0fPTC0tOnBcvmVbKmqZbLz2zKBels2/AVDdXMsFdakqY0g7akKWNoOLHvUB9bWnuybcN7R1fy2N7Ry+GBoSe9ZsncSprnV/GM5nm0ZEG6Ofupr5lVhE8hSSoXBm1Jk8rB3iNsbc+t1jGycseO7GfPob4n7XZYM6uCVQtquPKspmxjlsrRHQ8Xz3W3Q0nS02fQllRW+gaG2NHRy+bW3A6Hezr7csvhZT9HBoePun7G9GDZ/CpWNdXy/DMWsHRebjR6ZWM1LfXVVM500qEkqTAM2pJKTlffAI/s6WJbew/b2nvZ2t7D7oOH2XHgMK1d/U+6fkHtLJbXV3HukkU011exaM5sVjTUsLKxmgbbOyRJRWLQllQUnb0DPLa/i8dbn9ikZVt7L1vbeujqP3rb8NpZFSyZV8n5y+aydF4ly7N1pVc31bLE9g5JUokyaEsqiKHhxN5DfWzc18XWth62d+RW8BhZzePYiYe1sytYMreS565pZHl9rkf6jEW1LK+vZk6lG7RIksqPQVvS0zY0nGjr7uexfV08ureLza097DyQTT48cPhJy+E11s6ipb6Kc5fOYfGcSlY0VLOysZrTF9ZSOWM6EVGkTyJJUv4ZtCWd1JHBYXYfPMzG/d1sbesZ3flwZHm8Y7J0btWOeVVctGI+LQ3VrGyoZmVjDasaa5g2zSAtSZo6DNqS6BsYYk9nHxv2HGJbR25pvK1tuUmIx9vxcOm83NrSl66qZ+m8KpbXV3HGwlpWNhimJUkaYdCWpoj+wSH2HOzjkT2H2NzazbZsjelt7T3sO/TklTya51exakENV6zNrS/dUl/NmqZals6rNExLkjQOBm1pEhkcGmbvoT42t/awcV/XE4G6vZddBw8fdW3FtKC5voozF9Xxq2dX5yYgNlRzxsJammpnG6YlSfolGbSlMjQwNMym/d08ureLLa3dbG7rySYjdj9p58ORzVlecMYCWhqqWdFQxdmL59BQM8swLUlSARm0pRLW2tXPQ7s7eWxfF5v397Cto4cdHYefNDpdNXM6qxbU8KoLl7GisZqW+ipWLahlZUO1YVqSpCIxaEtFNrLe9GN7u3h0Xxeb9nezaX83m/d3P2njlpb6Kk5bUMNlpzeyoqGasxbPYe2iOuoqK1waT5KkEmPQliZId/9gbvOW9h427+/h8bYeHt3XxeNtPU9ab3plQzXPXt3A6gU1nL6wjrWL62ieX8V0R6clSSobBm0pj0YmIz62r4uN+7rZkm0vvrm1m7buI0ddWzu7ghUN1bz8giWsbKxmZUMNZy6qZek8A7UkSZOBQVt6GgaGhtnb2cfd2w+waX8uUG/cn2v7OHYDl+X1VZy1eA5rmmpY01RL8/wqTl9Yy5zKGbZ7SJI0iRm0pZMYHk7sONDLA7s6eWxfN4/sOZS1f/QedV3NrApWNlbz8guWsqaphpaGas5cWMeSeZWOTkuSNEUZtKXM4SNDPLynkwd3HeLh3YfYsK+LDXsO0T84PHpN7ewKTm+q5ZLT6kc3cLmgea6j05Ik6UkM2ppyhoYTm1tza1Bv3NfFg7sPcf/OTtq6j94dsaW+isvXNrF2UW4yYm7t6ZkGakmSNC4GbU1qA0PDbNzXzfptHTy4q5NH93Xz4K7Oo1b5qJtdwZmL6njZ+YtZtaCGs5fM4fSFtcyYPq2IlUuSpHJn0Nak0d0/yIO7OnlwVycP7T7Eg7s62bi/+6hrVjZUc9VZCzln6RzOXjyHMxbVUl/tKLUkSco/g7bKUlffAD/f1M7De3L91PftPEhr19GtH2cvqeOVFy7l9IW1rGuZz1mL6xylliRJE8agrZLX3t3P+m0HeGTPIe7bcZCH9xxi36EnQnXt7ArOWFjLr523mLWL6jhjUS1nLKxztQ9JklRUBm2VlIGhYW7f0sE92w9wz46D3LvjIB09T2z0Upv1U199ziIuaJ7HxSvm01Q3u4gVS5IkHZ9BW0XVe2SQH27Yzx2Pd3DH4x1s2Ns1eq5mVgVrF9dxUct8zlxUxzOWz2Vh3Wz7qSVJUlkwaGtCdfQc4Ycb9nPblnbu2naAx9t6Rs8tm1/JKy9cygXN8/iV0+pZXl9lqJYkSWXLoK2Cau/u5yePtXLLpjZ+vqntqN7qtYvq+M11y/iVVfU8/4wF1M2eUcRKJUmS8sugrbxq6+7nRxv2c9uWDn6xuY3dnX2j585ZModfO28xFy6fz2WnNzJ7xvQiVipJklRYBm390jbt7+LGe3fzvYf3HdVjffaSOq46exEXrZjH89YsoHKmwVqSJE0dBm09ZT39g9z88D5+trGNnzzWOrp1eUPNTF5zUTMvOGMBz17VYLCWJElTmkFbp5RS4v6dnXz/kVy4vnfHwdFz5yyZw6ufuYyrzl7I2UvmFLFKSZKk0mLQ1nENDg3zs41t/M+De/ifB/bS1T8IwPzqmbzs/MU8/4wFXLG2iaqZ/hWSJEk6HlOSRnX3D/LNu3fynYf28vNN7aPHz15Sx4vWLuTqcxdxWmNNESuUJEkqHwbtKe7wkSG+tn4H/33/Hu7Y2gHAtIAXndXEc9c0csXaJhbUuvOiJEnSU2XQnoIGhob57/v38O37d/P9R/YDMGN68OJzF/HScxdx+ZlNVEyfVuQqJUmSyptBe4pIKXHTA3v55j27+P4j+0aPP29NI6+4cCkvPmcR06a5C6MkSVK+GLQnub6BIT79ky185pYtHOrLTWh87ppGrj57Idecv8Ql+CRJkgrEoD1J7ek8zD/9aDPX376N4QSL58zm7c9fxesvWe5KIZIkSRPAxDXJPLCzk0/9aBPfeWgvkFsx5H89bxVXn7OQCFtDJEmSJopBe5K4c2sHH/vuo9z+eG7lkOef3si7Ll/D+cvmFrkySZKkqakoQTsitgJdwBAwmFJaFxHzga8CLcBW4FUppQORG4b9R+BqoBd4U0rp7mLUXYpu39LOtd/ZwD3bc7s1vmrdUt75wtUsnVdV5MokSZKmtmKOaD8/pdQ25vn7gB+klK6NiPdlz98L/CqwOvu5GPjn7PeUtqW1m/d94wHuyEawX3dxM+964WoW1LnmtSRJUikopdaRa4DLssefB35MLmhfA3whpZSA2yJibkQsSintKUqVRdbe3c/ffGcDX1u/E4DXXLSMd195Og01s4pcmSRJksYqVtBOwPciIgGfTildBzSNCc97gabs8RJgx5jX7syOTamgnVLiup9u4W++s4HhBJesrOdD15zF6qbaYpcmSZKk4yhW0H52SmlXRCwAbo6IDWNPppRSFsLHLSLeBrwNoLm5OX+VloAdHb289fPreXRfF0vnVfL/veI8LjmtvthlSZIk6SSKErRTSruy3/sj4pvARcC+kZaQiFgE7M8u3wUsG/PypdmxY+95HXAdwLp1655SSC9ln791K3/5/x5iOMEfXbGGdzx/lTs4SpIklYFpE/2GEVEdEbUjj4ErgQeBG4E3Zpe9Efiv7PGNwBsi51lA51Tozz4yOMwf3nAPH7jxIZrnV/HtP3g273zhakO2JElSmSjGiHYT8M1s85QK4Msppe9ExJ3A1yLiLcA24FXZ9TeRW9pvE7nl/d488SVPrB0dvbz+M7eztb2XX79gCX/7inOpmD7h34kkSZL0S5jwoJ1S2gKcd5zj7cALj3M8AW+fgNJKwo337eY9X7uPgeFhPvLyc3jtxZOr31ySJGmqKKXl/aa8j9/8GJ/4wUYi4Ju/f6m7OkqSJJUxg3aJ+Ifv50L2qgU1fPmtF7vxjCRJUpkzaJeAz9zyOP/w/Y2c1ljNt95+KTWz/MciSZJU7kx0RfaRmx7hup9u4bxlc7n+rRcbsiVJkiYJU10RPbCzk+t+uoXm+VX85+9dwgxXFpEkSZo0THZF8uCuTl76yVuYPWMan3vzMw3ZkiRJk4zprggO9BzhJf/3FgCuf+uzOK2xpsgVSZIkKd8M2hMspcRv/MutAHzk5edw4fJ5Ra5IkiRJhWDQnmA33LmDLa09XLm2yc1oJEmSJjGD9gTqGxjiT77xAACfeM0FRa5GkiRJhWTQnkDX/s8GAN5/9ZnMnjG9yNVIkiSpkAzaE2R4OPHvt24F4K3PWVHcYiRJklRwBu0Jcv3t2wD43eeuJCKKXI0kSZIKzaA9QT7xw00AvOvy1UWuRJIkSRPBoD0BHt3bRWtXPy88YwFVM92MU5IkaSowaE+AkbaR97zo9CJXIkmSpIli0J4AP9/UxsyKaZy5qK7YpUiSJGmCGLQLbHBomM2tPaxzB0hJkqQpxaBdYOu3HQDgstMbi1yJJEmSJpJBu8C+8+BeAK46a1GRK5EkSdJEMmgX2D07DgLQXF9V5EokSZI0kQzaBXbfjoNcvGJ+scuQJEnSBDNoF9Dm1m4ALnQipCRJ0pRj0C6gn29qA+B5a5wIKUmSNNUYtAvo0b1dAJy3bG6RK5EkSdJEM2gX0Ia9XVTPnM7sGdOLXYokSZImmEG7gB7d28WahbXFLkOSJElFYNAukMGhYbr7Bzm9yaAtSZI0FRm0C6St+wgA5yydU+RKJEmSVAwG7QLZ1t4DwNzKmUWuRJIkScVg0C6Qza25oL2ysbrIlUiSJKkYDNoFcvBwrnVkybzKIlciSZKkYjBoF8jOA4epnjmdutkzil2KJEmSisCgXSCPt/ZQV2nIliRJmqoM2gXSNzhEY+2sYpchSZKkIjFoF8g92w+yakFNscuQJElSkRi0C+DwkSEAamZVFLkSSZIkFYtBuwB2dx4GYEWDS/tJkiRNVQbtAmjt6gdg9QK3X5ckSZqqDNoFsHF/N4CTISVJkqYwg3YB9PYPAtBUZ9CWJEmaqgzaBdDW3c/sGdOY4zrakiRJU5ZBuwDu29lJQ80sIqLYpUiSJKlIDNoFkFIipWJXIUmSpGIyaBdAe/cRzm+eW+wyJEmSVEQG7QJo7e6noXpmscuQJElSERm086xvYIiuvkHmGbQlSZKmNIN2nj22rwuA+QZtSZKkKc2gnWd7O/sAOHNRXZErkSRJUjEZtPPsod2H4P9v796j5SrLO45/fwRIMIkJIZEqAQIUQQo0hGiheKEiKAjiWtCCRY1QZVWt4nJ5waIWlRaEXigiImpELgKCiCleMAIRRQkJkBvEQIggaCBBBRKVhMvTP95ncjbDnHNmcmYyOef8PmvtdfZ+9+3dz947eebd78wGdp7woi7XxMzMzMy6yYl2m92/urx+feIYvxXSzMzMbDhzot1mv3xkDS8bN4ottvDLaszMzMyGMyfabbZ81Vr2+Iux3a6GmZmZmXWZE+02euLPTwOw66QxXa6JmZmZmXWbE+02WpH9s/fZYVyXa2JmZmZm3eZEu41WrVkHwM7b+RdHzMzMzIY7J9pttHRl+Wm/yds60TYzMzMb7pxot9GjT5YW7Ylj/FZIMzMzs+HOiXYbrV7zFGNHbonkn/YzMzMzG+6caLfR+meDl43fptvVMDMzM7PNgBPtNlrz1NO85MV+I6SZmZmZOdFuqzVPPcPYUVt2uxpmZmZmthkYNIm2pDdJWiZpuaRTu12fRnabNJqXb++3QpqZmZkZDIrmV0kjgC8ChwIPA/MkzYqIe7pbs+f78jumd7sKZmZmZraZGCwt2q8ClkfEiohYD1wJHN3lOpmZmZmZ9WqwJNo7AA9Vph/OMjMzMzOzzdJgSbT7JelkSfMlzV+9enW3q2NmZmZmw9xgSbR/A+xYmZ6cZRtExEURMT0ipk+aNGmTVs7MzMzMrN5gSbTnAbtL2kXS1sDxwKwu18nMzMzMrFeD4ldHIuIZSf8C3ACMAGZGxN1drpaZmZmZWa8GRaINEBHfB77f7XqYmZmZmTVjsHQdMTMzMzMbVJxom5mZmZl1gBNtMzMzM7MOcKJtZmZmZtYBTrTNzMzMzDrAibaZmZmZWQc40TYzMzMz6wAn2mZmZmZmHeBE28zMzMysA5xom5mZmZl1gBNtMzMzM7MOUER0uw5tJ2k18GCXdj8ReKxL+x4qHMP2cBwHzjEcOMewPRzHgXMM28NxfKGdI2JSoxlDMtHuJknzI2J6t+sxmDmG7eE4DpxjOHCOYXs4jgPnGLaH49gadx0xMzMzM+sAJ9pmZmZmZh3gRLv9Lup2BYYAx7A9HMeBcwwHzjFsD8dx4BzD9nAcW+A+2mZmZmZmHeAWbTMzMzOzDnCi3SaS3iRpmaTlkk7tdn02B5JmSlolaUmlbIKk2ZLuy7/bZrkknZfxWyRpWmWdGbn8fZJmVMr3l7Q41zlPkjbtEXaepB0l3SzpHkl3Szolyx3HJkkaJel2SQszhp/J8l0kzc3jvkrS1lk+MqeX5/wplW19IsuXSXpjpXxY3P+SRki6S9L1Oe0YtkjSA3m/LZA0P8t8P7dA0nhJ10j6paSlkg50DFsjaY+8BmvDk5I+5Dh2QER4GOAAjADuB3YFtgYWAnt1u17dHoDXAtOAJZWys4FTc/xU4PM5fgTwA0DAAcDcLJ8ArMi/2+b4tjnv9lxWue7h3T7mDsTwpcC0HB8L3Avs5Ti2FEMBY3J8K2BuHu+3gOOz/ELgvTn+PuDCHD8euCrH98p7eySwS97zI4bT/Q98GPgmcH1OO4atx/ABYGJdme/n1mL4DeDdOb41MN4xHFA8RwCPADs7ju0f3KLdHq8ClkfEiohYD1wJHN3lOnVdRNwC/L6u+GjKP5Lk37dWyi+J4jZgvKSXAm8EZkfE7yPiD8Bs4E0578URcVuUO/qSyraGjIhYGRF35vgaYCmwA45j0zIWa3NyqxwCeD1wTZbXx7AW22uAQ7Il5mjgyohYFxG/ApZT7v1hcf9Lmgy8GfhqTgvHsF18PzdJ0jhKI87XACJifUQ8jmM4EIcA90fEgziObedEuz12AB6qTD+cZfZC20fEyhx/BNg+x3uLYV/lDzcoH7Ly8ft+lBZZx7EF2eVhAbCK8h/B/cDjEfFMLlI97g2xyvlPANvRemyHmnOBjwHP5fR2OIYbI4AfSbpD0slZ5vu5ebsAq4GvZzemr0oajWM4EMcDV+S449hmTrSta/JTsWaYdwAAC2JJREFUrn/2pgmSxgDfBj4UEU9W5zmO/YuIZyNiKjCZ0nq6Z5erNKhIOhJYFRF3dLsuQ8CrI2IacDjwfkmvrc70/dyvLSldEr8UEfsBf6R0cdjAMWxefq/iLcDV9fMcx/Zwot0evwF2rExPzjJ7oUfzkRL5d1WW9xbDvsonNygfciRtRUmyL4+Ia7PYcdwI+Yj5ZuBAyqPPLXNW9bg3xCrnjwN+R+uxHUoOAt4i6QFKt47XA/+LY9iyiPhN/l0FfIfywc/3c/MeBh6OiLk5fQ0l8XYMN87hwJ0R8WhOO45t5kS7PeYBu6t8A39rymOYWV2u0+ZqFlD7VvIM4LuV8nfmN5sPAJ7Ix1c3AIdJ2ja//XwYcEPOe1LSAdn3852VbQ0ZeWxfA5ZGxH9XZjmOTZI0SdL4HN8GOJTS1/1m4NhcrD6GtdgeC9yULTuzgONVflFjF2B3ypd9hvz9HxGfiIjJETGFcnw3RcQJOIYtkTRa0tjaOOU+XILv56ZFxCPAQ5L2yKJDgHtwDDfW2+jpNgKOY/s1+oakh9YHyjdy76X0/Tyt2/XZHAbKzbsSeJrSCvFPlH6aNwL3AT8GJuSyAr6Y8VsMTK9s5yTKl6aWAydWyqdT/pO6HziffAHTUBqAV1Me3S0CFuRwhOPYUgz3Be7KGC4BPp3lu1KSvOWUx6Yjs3xUTi/P+btWtnVaxmkZlW/QD6f7HziYnl8dcQxbi92ulF9UWQjcXTtO388tx3EqMD/v6esov3bhGLYex9GUJ03jKmWOY5sHvxnSzMzMzKwD3HXEzMzMzKwDnGibmZmZmXWAE20zMzMzsw5wom1mZmZm1gFOtM3MzMzMOsCJttkwJek0SXdLWiRpgaS/6fD+5kia3sLyF0s6trLusqzrLyWdX/tt7E6TNFXSERuxXsPjzVdG79XCdt4i6dT+l9w0JE2XdF6X67C2Q9t9Td4TC/I312vl4yW9r4n1D5Z0fSfq1kmS/rXbdTAbqpxomw1Dkg4EjgSmRcS+wBuAh7pbq36dkHXdF1jHJnj5Qb7VcCrlN57bIiLeHRH3tLD8rIg4a6D7rbzBcUAiYn5EfLAd22rGQOudL9ho9v+6E4AzI2JqRPy5Uj4e6DfRHsScaJt1iBNts+HppcBjEbEOICIei4jfAkj6tKR5kpZIuijf6lVrof0fSfMlLZX0SknXSrpP0hm5zJRscb48l7lG0ovqdy7pMEm/kHSnpKsljWm24hGxHvgYsJOkv8637X1P0sKs83EN9vdBSfdki/iVWTZB0nVZdpukfbP8dEmXSroVuBT4LHBctnIel/ubKel2SXdJOjrX20bSlXnc3wG2qa9HJY7Tc3ytpH/Put8mafsGy79L0vk5frGk8yT9XNKKWot/zvu4pMW5rbMq+zpX0nzgFEn7S/qJpDsk3aCeVy2/J8/5Qknfrp0zSX+fMV0o6ZYs29Bqm7GamftZIemDlfp8SuUpxM8kXSHpIw2ObYqkm/Ic3Chpp8pxXihpLnC2yhsjf5HHd0bdNj6adV8k6TOV7S6TdAnlhRk71q1zSJ67xVn/kZLeDfwD8DlJl9dV9Sxgt7wGzlFxTsZmcS/X3CtzH7v1Efc5kj6f19K9kl5Tv50+zu3UvGYWSfqOylv56q+viZIeqFxH10r6oco9e3aWnwVsk8d2uZq4n8ysBd1+Y44HDx42/QCMobxl8l7gAuB1lXkTKuOXAkfl+Bzg8zl+CvBbSsI+kvLmz+2AKZQ3WR6Uy80EPlJZfzowEbgFGJ3lHyff1lhXx4uBY6vr1s2/DjgOOAb4SqV8XINt/ZaetxaOz79fAP4tx18PLMjx04E7gG1y+l3A+ZVt/Qfw9tq2MoajgQ8DM7N8X+CZ+jrXH0vGqhbfs4FPNlh+w/4zJldTGkn2ApZn+eHAz4EXVc9h7uuCHN8ql5mU08dV6rtdZX9nAB/I8cXADnVxO5ieN0Oentscmef1d7mfV1Kur1HAWMpb5j7S4Nj+D5iR4ycB11WO83pgRE7PAt6Z4+8H1ub4YcBFlLfWbZHrvJZyHT4HHNBgn6MoT29entOXAB+qv+bq1pkCLKlMHwPMBkYA2wO/ptwLB2cd/pZyDe3UT9znAP+V40cAP26w797O7SLyvqV8GDy3wfU1EXigch2tAMZlDB4Edsx5a+uOrc/7yYMHD80PbtE2G4YiYi2wP3AysBq4StK7cvbfSZoraTElAf2ryqqz8u9i4O6IWBmlVXwFPa2GD0XErTl+GeU18lUHUJLEWyUtAGYAO2/EYahSl0OzZfA1EfFEg2UXAZdLejslASbrdSlARNwEbCfpxbXjjOd3Hag6DDg16z6HkrTsREnwLsvtLcp99mc9JTGDkphNaWKd6yLiuSjdT2ot4G8Avh4Rf8r9/76y/FX5dw9gb2B21v2TwOSct7ekn+Y5P4Gec34rcLGk91CSyka+FxHrIuIxYFXW6SDguxHxVESsoSTUjRwIfDPHL+X518rVEfFsjh8EXFFZruawHO4C7gT2BHbPeQ9GxG0N9rkH8KuIuDenv0E5d614NXBFRDwbEY8CP6F8uAB4BSX5Pyoifk3fcQe4Nv/2dv5fcG4ljaN88PlJi8dwY0Q8ERFPAffQ+L5r5n4ysya1pc+emQ0+mcTMAeZkgjVDpVvFBZQWsYcknU5JJGvW5d/nKuO16dq/J1G/q7ppAbMj4m0bW3dJI4B9gKURca+kaZQWwTMk3RgRn61b5c2UROQo4DRJ+/Sziz/2tXvgmIhYVlenlo4hPR0Rtfg8S3P/Jlfj3sxOa8ciyoejAxssczHw1ohYmB+4DgaIiH9W+ZLsm4E7JO3fT32aPYZm1J+D+usIyjGdGRFffl6hNKXB+pvKSso9sx/lSUpfcYee+LUrds/Q0y10VN28fs9Vk/eTmTXJLdpmw5CkPSTtXimaSnmUXPuP+TGVftPHvmDl/u2k8mVLgH8EflY3/zbgIEl/mXUZLenlLdR9K+BMSsv5IkkvA/4UEZcB5wDT6pbfgvKI/GZKN5VxlK4zP6W03iLpYEqf9Scb7HINpftDzQ3AB6QNfdf3y/Jb8niRtDel+8imMhs4UT19qyc0WGYZMKl2biRtJanWcj0WWJmxPaG2gqTdImJuRHya8uRjx/qN9uJW4ChJo/I6OrKX5X4OHJ/jJ1DOSW/bqy5XcwNwUu4DSTtIekk/dVsGTKldf8A7KC3Sfam/Bn5K6bc/QtIkyoe423Pe45QPJmfmddVX3JvxgnObrcx/qPTprh7DA5SnVdD8/ft0nnv6u5/MrDVu0TYbnsYAX1D5ibxngOXAyRHxuKSvUL5A9ggwbyO2vQx4v6SZlMfTX6rOjIjV2Wp6haSRWfxJSl/nvlwuaR2lP/CPgaOzfB/gHEnPAU8D761bbwRwWT5uF3BeHufpwExJi4A/UbqwNHIzPV1FzgQ+B5wLLMok/leURPJLwNclLQWWUroCbBIR8UNJU4H5ktYD36fulyQiYr3KlyfPy1hsmcdxN/ApYC4lmZ5LT1J5Tn4gE3AjsBB4XRP1mSdpFqX7zKOU7giNuiB8gBKzj+a+T+xlk6cA35T0cSq/NhMRP5L0CuAX+blnLfB2Smttb3V7StKJwNUqv2gyD7iwn+P5naRbJS0BfkD5Mu6BlHgE8LGIeETSnrn8o5KOzGVPoiS8jeLerz7O7QzgwkzAV9ATu/8EviXpZOB7zeyD0tVlkaQ7KX3W+7qfzKwF6nlqaWY2MPnI/vqI2LvLVbEukzQmItZmIngL5YPcnd2ul5nZpuQWbTMz64SLVF7MMwr4hpNsMxuO3KJtZmZmZtYB/jKkmZmZmVkHONE2MzMzM+sAJ9pmZmZmZh3gRNvMzMzMrAOcaJuZmZmZdYATbTMzMzOzDvh/VsjeHvkz0gAAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 864x576 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NlfKawySbzg_"
      },
      "source": [
        "## Pre-trained Longformer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GewUQNGmXgbX",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281,
          "referenced_widgets": [
            "e1a76081aea746538ea69d4f4b787614",
            "66ce91fe4ce14eebad559ec6b637acd8",
            "f52f1ce5869c42998181613e178eb563",
            "ee8b154d6c9e4dad95fee4e15203be1a",
            "2d82458910d442ecb4408f10c906a34f",
            "c729425ec35e49c0920459155c59e211",
            "1ac0156886f24c0fb6fbf7fc5258265c",
            "b3436f51426c4d108b47e234f372a478",
            "631f420f4fdb438c9b1f72f14b2af1ad",
            "9b7a5d7c0eae4e5fb3a64fd670e96576",
            "7d0431a261e5447dbe3940339cca5495",
            "7464e0123eb940dfa05356c7a8b70af1",
            "35ff544a0ec64e8fbf915499d5dfafac",
            "a2e24519e5174a63bd875a31889db118",
            "f6741deaeb1740ec94609301c90f130e",
            "29dd956b39174fffa3b1d4958e5d7a23",
            "61b93144426e42cc8c3b263d10ef882c",
            "a21e051ade454c7bbac6f5dec4863d22",
            "984cd5e29d0647a38af81edd59d3e1a4",
            "caa9f61b46b54f5e9d416c196c2afe06",
            "2afdbd3182f54d8388e48701ba8f4e05",
            "a21f8658766b4f4d9b8611176dfe0aa4",
            "c67c9f8abc4943f986eb728c33497d2d",
            "98ebea51b1924544b0072127aef21b9b",
            "7d4138bb08c54c408134434c1bc22f24",
            "03365c120b844e2b9937b8ffeddc6025",
            "4efd055fbd2a485993ca363581b3031a",
            "a38fc080994c48ac9bd3aca34b5eff49",
            "33a33241cfc943bfae87270208e1a5ac",
            "3d31e7dcfbab4bf9a9da773cfbe19dc8",
            "fe2d4619d3fc40ae8da726f18766cce8",
            "fbe5febc21904c32979f6d309050380e"
          ]
        },
        "outputId": "1e65a128-b06c-4948-f3f4-a1b4b02c3bbd"
      },
      "source": [
        "# Import tokenizer and model\n",
        "tokenizer = LongformerTokenizer.from_pretrained(MODEL)\n",
        "model = TFLongformerForQuestionAnswering.from_pretrained(MODEL)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e1a76081aea746538ea69d4f4b787614",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=898823.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "631f420f4fdb438c9b1f72f14b2af1ad",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "61b93144426e42cc8c3b263d10ef882c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=866.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7d4138bb08c54c408134434c1bc22f24",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1739063632.0, style=ProgressStyle(descr…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "All model checkpoint layers were used when initializing TFLongformerForQuestionAnswering.\n",
            "\n",
            "All the layers of TFLongformerForQuestionAnswering were initialized from the model checkpoint at allenai/longformer-large-4096-finetuned-triviaqa.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFLongformerForQuestionAnswering for predictions without further training.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pjUXWrdWxDQV"
      },
      "source": [
        "# Funtion to generate answer for a question and text string\r\n",
        "def get_answer(question, text):\r\n",
        "    inputs = tokenizer.encode_plus(question, text, return_tensors='tf', max_length=MAX_LEN, truncation=True)\r\n",
        "    input_ids = inputs[\"input_ids\"].numpy().tolist()\r\n",
        "    start_scores, end_scores = model(inputs)\r\n",
        "\r\n",
        "    start = tf.math.argmax(start_scores, axis=1).numpy()[0]\r\n",
        "    end = tf.math.argmax(end_scores, axis=1).numpy()[0]\r\n",
        "    \r\n",
        "    all_tokens = tokenizer.convert_ids_to_tokens(input_ids[0])\r\n",
        "    answer = ''.join(all_tokens[start: end + 1]).replace('▁', ' ').strip()\r\n",
        "    answer = answer.replace('[SEP]', '')\r\n",
        "    return answer if ('[CLS]' not in answer) and (len(answer) != 0) else ''"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gr5PWBX6mOKy"
      },
      "source": [
        "## Manual Inference"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CHw1RSt5ZU9R",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "755f9037-6774-4e85-8d5b-a60c890483bf"
      },
      "source": [
        "# Print an example inference from model\r\n",
        "id = 1\r\n",
        "print(get_answer(df_val['question'].iloc[id], df_val['text'].iloc[id]))\r\n",
        "print(df_val['answer'].iloc[id])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Ġpoets\n",
            "poet\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4QdzR4TDb9hb"
      },
      "source": [
        "## Answer Prediction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pn26UFIPcPj1"
      },
      "source": [
        "# Predict answer for a DF with columns 'question' and 'paragraphs'\n",
        "def predict_answers(df):\n",
        "    pred_answers = []\n",
        "    cnt_ans_found = 0\n",
        "    for i in tqdm(range(len(df))):\n",
        "        a = ''\n",
        "        try:\n",
        "            a = get_answer(df['question'].iloc[i], df['text'].iloc[i])\n",
        "            if a != '':\n",
        "                cnt_ans_found += 1\n",
        "        except:\n",
        "            print(f'Error in iteration {i}')\n",
        "        pred_answers.append(a)\n",
        "    return pred_answers, cnt_ans_found"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "03v6CUbGcHWU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "99d5cb29-e5d3-4f8c-e866-eaf940c0c967"
      },
      "source": [
        "# Easy Predictions\n",
        "t0 = time.time()\n",
        "pred_answers_easy, cnt_ans_found_easy = predict_answers(df_val_easy)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 1797/1797 [36:05<00:00,  1.20s/it]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sO-F4xUId17U",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ac9be6a0-25f3-4423-9569-907df0ea9b76"
      },
      "source": [
        "# Medium Predictions\n",
        "t0 = time.time()\n",
        "pred_answers_medium, cnt_ans_found_medium = predict_answers(df_val_medium)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 5681/5681 [1:54:09<00:00,  1.21s/it]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PAHYkOMgxe_H",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a2680e97-a687-404c-83ee-67b026284d04"
      },
      "source": [
        "# Hard Predictions\n",
        "t0 = time.time()\n",
        "pred_answers_hard, cnt_ans_found_hard = predict_answers(df_val_hard)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0it [00:00, ?it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X4nixLgKtJGM",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7e198fea-f6c1-4f17-dc18-b55e7ffc8991"
      },
      "source": [
        "print(f'Easy Questions Answered {cnt_ans_found_easy} / {len(df_val_easy)}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Easy Questions Answered 1674 / 1797\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qoDaeZzud2Jg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "af2d2342-3e75-4bd7-b03f-962d4a016d69"
      },
      "source": [
        "print(f'Medium Questions Answered {cnt_ans_found_medium} / {len(df_val_medium)}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Medium Questions Answered 5297 / 5681\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VUx2pWy6xrza",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "454ff15c-1c31-407a-eeee-7862dd7ca032"
      },
      "source": [
        "print(f'Hard Questions Answered {cnt_ans_found_hard} / {len(df_val_hard)}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Hard Questions Answered 0 / 0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EFXGM_CGEYX5"
      },
      "source": [
        "## Evaluation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yZmMHo1sEaVx"
      },
      "source": [
        "# Evaluation functions taken from HotpotQA\n",
        "def normalize_answer(s):\n",
        "\n",
        "    def remove_articles(text):\n",
        "        return re.sub(r'\\b(a|an|the)\\b', ' ', text)\n",
        "\n",
        "    def white_space_fix(text):\n",
        "        return ' '.join(text.split())\n",
        "\n",
        "    def remove_punc(text):\n",
        "        exclude = set(string.punctuation)\n",
        "        return ''.join(ch for ch in text if ch not in exclude)\n",
        "\n",
        "    def lower(text):\n",
        "        return text.lower()\n",
        "\n",
        "    return white_space_fix(remove_articles(remove_punc(lower(s))))\n",
        "\n",
        "# Exact Match score\n",
        "def exact_match_score(prediction, ground_truth):\n",
        "    return (normalize_answer(prediction) == normalize_answer(ground_truth))\n",
        "\n",
        "# FPR metrics\n",
        "def f1_score(prediction, ground_truth):\n",
        "    normalized_prediction = normalize_answer(prediction)\n",
        "    normalized_ground_truth = normalize_answer(ground_truth)\n",
        "\n",
        "    ZERO_METRIC = (0, 0, 0)\n",
        "\n",
        "    if normalized_prediction in ['yes', 'no', 'noanswer'] and normalized_prediction != normalized_ground_truth:\n",
        "        return ZERO_METRIC\n",
        "    if normalized_ground_truth in ['yes', 'no', 'noanswer'] and normalized_prediction != normalized_ground_truth:\n",
        "        return ZERO_METRIC\n",
        "\n",
        "    prediction_tokens = normalized_prediction.split()\n",
        "    ground_truth_tokens = normalized_ground_truth.split()\n",
        "    common = Counter(prediction_tokens) & Counter(ground_truth_tokens)\n",
        "    num_same = sum(common.values())\n",
        "    if num_same == 0:\n",
        "        return ZERO_METRIC\n",
        "    precision = 1.0 * num_same / len(prediction_tokens)\n",
        "    recall = 1.0 * num_same / len(ground_truth_tokens)\n",
        "    f1 = (2 * precision * recall) / (precision + recall)\n",
        "    return f1, precision, recall"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ALC5Gt4LuQ0z"
      },
      "source": [
        "# Normalize the predictions by removing the seperator\n",
        "pred_answers_easy_n = [' '.join(i.split('Ġ')) for i in pred_answers_easy]\n",
        "pred_answers_medium_n = [' '.join(i.split('Ġ')) for i in pred_answers_medium]\n",
        "pred_answers_hard_n = [' '.join(i.split('Ġ')) for i in pred_answers_hard]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2yoPoKNnso90"
      },
      "source": [
        "# Create a list of true answers for easy and medium\n",
        "true_answers_easy = df_val_easy['answer'].to_list()\n",
        "true_answers_medium = df_val_medium['answer'].to_list()\n",
        "true_answers_hard = df_val_hard['answer'].to_list()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U2TqUTs4r3BT"
      },
      "source": [
        "# Calculate F1 and exact match score EM for each document in both easy and medium sets\n",
        "f1_easy = []\n",
        "em_easy = []\n",
        "p_easy = []\n",
        "r_easy = []\n",
        "f1_medium = []\n",
        "em_medium = []\n",
        "p_medium = []\n",
        "r_medium = []\n",
        "f1_hard = []\n",
        "em_hard = []\n",
        "p_hard = []\n",
        "r_hard = []\n",
        "\n",
        "# Easy\n",
        "for i in range(len(pred_answers_easy_n)):\n",
        "    fpr = f1_score(pred_answers_easy_n[i], true_answers_easy[i])\n",
        "    f1_easy.append(fpr[0])\n",
        "    p_easy.append(fpr[1])\n",
        "    r_easy.append(fpr[2])\n",
        "    em_easy.append(exact_match_score(pred_answers_easy_n[i], true_answers_easy[i]))\n",
        "\n",
        "# Medium\n",
        "for i in range(len(pred_answers_medium_n)):\n",
        "    fpr = f1_score(pred_answers_medium_n[i], true_answers_medium[i])\n",
        "    f1_medium.append(fpr[0])\n",
        "    p_medium.append(fpr[1])\n",
        "    r_medium.append(fpr[2])\n",
        "    em_medium.append(exact_match_score(pred_answers_medium_n[i], true_answers_medium[i]))\n",
        "\n",
        "# Hard\n",
        "for i in range(len(pred_answers_hard_n)):\n",
        "    fpr = f1_score(pred_answers_hard_n[i], true_answers_hard[i])\n",
        "    f1_hard.append(fpr[0])\n",
        "    p_hard.append(fpr[1])\n",
        "    r_hard.append(fpr[2])\n",
        "    em_hard.append(exact_match_score(pred_answers_hard_n[i], true_answers_hard[i]))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Me0s9_tPsfJT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "693777fd-e5cd-4418-8867-ee872a6b82fd"
      },
      "source": [
        "# Trained model scores\n",
        "print(f\"Overall Mean F1: {np.mean(f1_easy+f1_medium+f1_hard)}\")\n",
        "print(f\"Overall Mean Precision: {np.mean(p_easy+p_medium+p_hard)}\")\n",
        "print(f\"Overall Mean Recall: {np.mean(r_easy+r_medium+r_hard)}\")\n",
        "print(f\"Overall Mean EM: {np.mean(em_easy+em_medium+em_hard)}\")\n",
        "\n",
        "print(f\"Easy Mean F1: {np.mean(f1_easy)}\")\n",
        "print(f\"Easy Mean EM: {np.mean(em_easy)}\")\n",
        "\n",
        "print(f\"Medium Mean F1: {np.mean(f1_medium)}\")\n",
        "print(f\"Medium Mean EM: {np.mean(em_medium)}\")\n",
        "\n",
        "# print(f\"Hard Mean F1: {np.mean(f1_hard)}\")\n",
        "# print(f\"Hard Mean EM: {np.mean(em_hard)}\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Overall Mean F1: 0.33522232311264094\n",
            "Overall Mean Precision: 0.36070985183452764\n",
            "Overall Mean Recall: 0.350776040327125\n",
            "Overall Mean EM: 0.2713292324150842\n",
            "Easy Mean F1: 0.3949259480218869\n",
            "Easy Mean EM: 0.3099610461880913\n",
            "Medium Mean F1: 0.31633701877151876\n",
            "Medium Mean EM: 0.2591093117408907\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}