Hybrid_search.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e73f4ed251c14e6b8c16f5e92d616948": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_254efae6c7c74025b70c279ea9398fdf",
              "IPY_MODEL_b6f8c16f2fd24119a86a20583c37cef3",
              "IPY_MODEL_2822f773b5f0414e9c969babf467ce37"
            ],
            "layout": "IPY_MODEL_159fdcf30137493f9d8bb5fb3d8a5af2"
          }
        },
        "254efae6c7c74025b70c279ea9398fdf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a5be9a05f3fd4e0682cb3f34be347701",
            "placeholder": "​",
            "style": "IPY_MODEL_355a0e641f9e491ea53c70c993cabcb9",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "b6f8c16f2fd24119a86a20583c37cef3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bfce41d6e8bd4b66a1a590cf55ca25ec",
            "max": 1431,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_70397aaa1e764789ac5b368f1e49eb3b",
            "value": 1431
          }
        },
        "2822f773b5f0414e9c969babf467ce37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d8219d4062144c81a2f18cb52b41f5ef",
            "placeholder": "​",
            "style": "IPY_MODEL_cefa5bece9d640b0bc98884da3a41c1d",
            "value": " 1.43k/1.43k [00:00&lt;00:00, 83.1kB/s]"
          }
        },
        "159fdcf30137493f9d8bb5fb3d8a5af2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a5be9a05f3fd4e0682cb3f34be347701": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "355a0e641f9e491ea53c70c993cabcb9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bfce41d6e8bd4b66a1a590cf55ca25ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "70397aaa1e764789ac5b368f1e49eb3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d8219d4062144c81a2f18cb52b41f5ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cefa5bece9d640b0bc98884da3a41c1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b064a07736a348b38cdcef86740560a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_70a7901c2f8d4e418539b5e00615f03a",
              "IPY_MODEL_fb5296974cb540e1926b4fbbb312f7ef",
              "IPY_MODEL_6270c78188254087b936f779448d7dd0"
            ],
            "layout": "IPY_MODEL_ce9bbc0ac3014772b2875e2ba724be8d"
          }
        },
        "70a7901c2f8d4e418539b5e00615f03a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1972599c873d45098fd86aad8c385162",
            "placeholder": "​",
            "style": "IPY_MODEL_7209f15723a44207b077e34da2c8542e",
            "value": "tokenizer.model: 100%"
          }
        },
        "fb5296974cb540e1926b4fbbb312f7ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f7b7f39e720f4e788c8382f02e469b39",
            "max": 493443,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bbb9b2f556264df3bd74a66b1c69704b",
            "value": 493443
          }
        },
        "6270c78188254087b936f779448d7dd0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7ebca4b0a59a4d33bafa5d72e92f6367",
            "placeholder": "​",
            "style": "IPY_MODEL_12400ead406b4514941e7f2d0c2f607e",
            "value": " 493k/493k [00:00&lt;00:00, 18.7MB/s]"
          }
        },
        "ce9bbc0ac3014772b2875e2ba724be8d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1972599c873d45098fd86aad8c385162": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7209f15723a44207b077e34da2c8542e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f7b7f39e720f4e788c8382f02e469b39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bbb9b2f556264df3bd74a66b1c69704b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7ebca4b0a59a4d33bafa5d72e92f6367": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "12400ead406b4514941e7f2d0c2f607e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b2a9088ebf2046dea92bd80d0981546b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1c49d912bc794831aa25b93dc1125d10",
              "IPY_MODEL_e2ed21df06d84c1281e5253d41a1c231",
              "IPY_MODEL_e18fc0c162eb4f13bb1c30bd2702c39f"
            ],
            "layout": "IPY_MODEL_92c297d75a8542c7abf7f23e7fe1e81b"
          }
        },
        "1c49d912bc794831aa25b93dc1125d10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a9b59c7f33e049389250b7e2aa200200",
            "placeholder": "​",
            "style": "IPY_MODEL_82c22f3a23b341bda2a37e4b6c5496ad",
            "value": "tokenizer.json: 100%"
          }
        },
        "e2ed21df06d84c1281e5253d41a1c231": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9f184211792242eaa3e947c834675808",
            "max": 1795303,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_da2348c6dd274dd383fe199a70157a7c",
            "value": 1795303
          }
        },
        "e18fc0c162eb4f13bb1c30bd2702c39f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2a774a14496348168fca4003953c206a",
            "placeholder": "​",
            "style": "IPY_MODEL_e6ac3c59973943a497527f0f2d281110",
            "value": " 1.80M/1.80M [00:00&lt;00:00, 7.56MB/s]"
          }
        },
        "92c297d75a8542c7abf7f23e7fe1e81b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a9b59c7f33e049389250b7e2aa200200": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "82c22f3a23b341bda2a37e4b6c5496ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9f184211792242eaa3e947c834675808": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "da2348c6dd274dd383fe199a70157a7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2a774a14496348168fca4003953c206a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e6ac3c59973943a497527f0f2d281110": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "53a8aa8b486e41d4a16004fc9b62fb09": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8bf97e0fea874727a4148377ecdd3e7e",
              "IPY_MODEL_7e3169ddeae4467cad98e37fc5e94128",
              "IPY_MODEL_a50bc94c049047648f5cb77c38bb5c1e"
            ],
            "layout": "IPY_MODEL_6f54c683b56d486b90f2bfd09228f152"
          }
        },
        "8bf97e0fea874727a4148377ecdd3e7e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0fbbc5ef7a274e7aa79b606cc50a8d2c",
            "placeholder": "​",
            "style": "IPY_MODEL_5162b6beebb046caa3b2fda85cb96084",
            "value": "added_tokens.json: 100%"
          }
        },
        "7e3169ddeae4467cad98e37fc5e94128": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_63dd8bdacdff4fba8303b1cbe5d51dcb",
            "max": 42,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ba6dc212926b46b386a5c979fb1f8841",
            "value": 42
          }
        },
        "a50bc94c049047648f5cb77c38bb5c1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b8e9e584abea4a65993360ecaae03ba3",
            "placeholder": "​",
            "style": "IPY_MODEL_99ad19088f6344d2a3d86b80058ee6c0",
            "value": " 42.0/42.0 [00:00&lt;00:00, 2.33kB/s]"
          }
        },
        "6f54c683b56d486b90f2bfd09228f152": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0fbbc5ef7a274e7aa79b606cc50a8d2c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5162b6beebb046caa3b2fda85cb96084": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "63dd8bdacdff4fba8303b1cbe5d51dcb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ba6dc212926b46b386a5c979fb1f8841": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b8e9e584abea4a65993360ecaae03ba3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "99ad19088f6344d2a3d86b80058ee6c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "719c5e17a58147949d9bb85fa807d48f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_26fd86ca1a504ff89a77631b9e038e3b",
              "IPY_MODEL_cb8a8a93935747bd86e9184bf2595d82",
              "IPY_MODEL_dad7b96f4d2141a7abef2dc667f34ab0"
            ],
            "layout": "IPY_MODEL_8d4f6654662c4ef88986fa2919a38f50"
          }
        },
        "26fd86ca1a504ff89a77631b9e038e3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4a5f2156c829463c917b3e1a4ccd83e0",
            "placeholder": "​",
            "style": "IPY_MODEL_38975564fbec4aa68d8aa8a05d6edd9a",
            "value": "special_tokens_map.json: 100%"
          }
        },
        "cb8a8a93935747bd86e9184bf2595d82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_522e491d4145469a82e30be85713a3b6",
            "max": 168,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cf63d078972347a7a86f3ff6e6f14608",
            "value": 168
          }
        },
        "dad7b96f4d2141a7abef2dc667f34ab0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_023dd0baf4954756908dada312eb7e33",
            "placeholder": "​",
            "style": "IPY_MODEL_61fa28537e614acbaa21480dbd951857",
            "value": " 168/168 [00:00&lt;00:00, 7.49kB/s]"
          }
        },
        "8d4f6654662c4ef88986fa2919a38f50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4a5f2156c829463c917b3e1a4ccd83e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "38975564fbec4aa68d8aa8a05d6edd9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "522e491d4145469a82e30be85713a3b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cf63d078972347a7a86f3ff6e6f14608": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "023dd0baf4954756908dada312eb7e33": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "61fa28537e614acbaa21480dbd951857": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4f93b9b5e64b4640aa377569692c3dde": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_bf130339f83e4324a6d883f35017c7f0",
              "IPY_MODEL_c5f4df68d2ac4b73abcb386bad7e2ae6",
              "IPY_MODEL_f6a1068246b4469c85d92a28a1f65b0f"
            ],
            "layout": "IPY_MODEL_0746a73307074e79bd345287aaff9040"
          }
        },
        "bf130339f83e4324a6d883f35017c7f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_79af18a3b69a46fda639a88ef062b84e",
            "placeholder": "​",
            "style": "IPY_MODEL_e3b52a7bbad840639094522e597973d6",
            "value": "config.json: 100%"
          }
        },
        "c5f4df68d2ac4b73abcb386bad7e2ae6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ca7c73c2d3fd4aa5a0e52646cbb22141",
            "max": 638,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cc18979059884d08a55f4ba0efbc7d3e",
            "value": 638
          }
        },
        "f6a1068246b4469c85d92a28a1f65b0f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_27510cc3152e44ffbde8794f91b0f033",
            "placeholder": "​",
            "style": "IPY_MODEL_5bce31b3fc0a4f9890f714e1a21d4eac",
            "value": " 638/638 [00:00&lt;00:00, 15.8kB/s]"
          }
        },
        "0746a73307074e79bd345287aaff9040": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "79af18a3b69a46fda639a88ef062b84e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e3b52a7bbad840639094522e597973d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ca7c73c2d3fd4aa5a0e52646cbb22141": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cc18979059884d08a55f4ba0efbc7d3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "27510cc3152e44ffbde8794f91b0f033": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5bce31b3fc0a4f9890f714e1a21d4eac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f1e205e3e27e44b69c896bfdc263a5da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8efcb470797f4b819f9af93e472bb986",
              "IPY_MODEL_899a6b5e22574c368a081d2f1d736351",
              "IPY_MODEL_20ce32d672d64a5b8bf841f4057e23fd"
            ],
            "layout": "IPY_MODEL_5328d8305f5642499ed1e3f7b1e29ddb"
          }
        },
        "8efcb470797f4b819f9af93e472bb986": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3909625fb84740dd9c88392a918b1fdf",
            "placeholder": "​",
            "style": "IPY_MODEL_51dd2ec385e34f6794f60addd31db5da",
            "value": "model.safetensors.index.json: 100%"
          }
        },
        "899a6b5e22574c368a081d2f1d736351": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3a3a407b5987496eb30b52f8479b2b16",
            "max": 23950,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e73233099148400493020c99f4fe1ca2",
            "value": 23950
          }
        },
        "20ce32d672d64a5b8bf841f4057e23fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4dd3c29f3c394d5c8f634046838e48dc",
            "placeholder": "​",
            "style": "IPY_MODEL_154048d2fc284fbbb22ec7f2de2e0a3c",
            "value": " 23.9k/23.9k [00:00&lt;00:00, 706kB/s]"
          }
        },
        "5328d8305f5642499ed1e3f7b1e29ddb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3909625fb84740dd9c88392a918b1fdf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "51dd2ec385e34f6794f60addd31db5da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3a3a407b5987496eb30b52f8479b2b16": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e73233099148400493020c99f4fe1ca2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4dd3c29f3c394d5c8f634046838e48dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "154048d2fc284fbbb22ec7f2de2e0a3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "171e66500b744fa59a19edc5e4b45be9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c70a14c9673e4fd49ca43fb420a2f070",
              "IPY_MODEL_fbb13383c0ae4085b25f47b6af522021",
              "IPY_MODEL_759c993f132745a99ebe160da7421ef1"
            ],
            "layout": "IPY_MODEL_01acf38611684c0b8eec7e68d3617a5a"
          }
        },
        "c70a14c9673e4fd49ca43fb420a2f070": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_78397807421142ab86107251baf39320",
            "placeholder": "​",
            "style": "IPY_MODEL_10dfdb3a60714a5c96c9c8889b67c07a",
            "value": "Downloading shards: 100%"
          }
        },
        "fbb13383c0ae4085b25f47b6af522021": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5379140a67ce4188a12268f9efaa7d3d",
            "max": 8,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3e3cf27c43d342e1957ad5613088df53",
            "value": 8
          }
        },
        "759c993f132745a99ebe160da7421ef1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d619276a2ba84c02ae7477ec8c8e193a",
            "placeholder": "​",
            "style": "IPY_MODEL_3ad62b0192104b47a62e2e28cf6c045b",
            "value": " 8/8 [02:26&lt;00:00, 13.63s/it]"
          }
        },
        "01acf38611684c0b8eec7e68d3617a5a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "78397807421142ab86107251baf39320": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "10dfdb3a60714a5c96c9c8889b67c07a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5379140a67ce4188a12268f9efaa7d3d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3e3cf27c43d342e1957ad5613088df53": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d619276a2ba84c02ae7477ec8c8e193a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3ad62b0192104b47a62e2e28cf6c045b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3d35f54a6c1d48589ce4fb7e786221ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_16012aeed93e4b1c9627b37bd32637a0",
              "IPY_MODEL_015e10071628426b8df1c1db32168a30",
              "IPY_MODEL_ce3b4db0498f4078b051a54c153c9f98"
            ],
            "layout": "IPY_MODEL_16963e7d78f742b5950edbfbd52a34a2"
          }
        },
        "16012aeed93e4b1c9627b37bd32637a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ac054c813ec74ad7aa1dc9754b7a09d0",
            "placeholder": "​",
            "style": "IPY_MODEL_1ad5420bad52435ea782776b80696e38",
            "value": "model-00001-of-00008.safetensors: 100%"
          }
        },
        "015e10071628426b8df1c1db32168a30": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_35660b46aee94899bd172389c835e4ef",
            "max": 1889587040,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a7e4393a792c4d9f90c7f9a673b71ff9",
            "value": 1889587040
          }
        },
        "ce3b4db0498f4078b051a54c153c9f98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bb21b23915f844c7b78dc1803f5c7d5c",
            "placeholder": "​",
            "style": "IPY_MODEL_fff97c9c191f431ca3bc2fb4b54017bc",
            "value": " 1.89G/1.89G [00:10&lt;00:00, 234MB/s]"
          }
        },
        "16963e7d78f742b5950edbfbd52a34a2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ac054c813ec74ad7aa1dc9754b7a09d0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1ad5420bad52435ea782776b80696e38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "35660b46aee94899bd172389c835e4ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a7e4393a792c4d9f90c7f9a673b71ff9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bb21b23915f844c7b78dc1803f5c7d5c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fff97c9c191f431ca3bc2fb4b54017bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "139c836d42994b6ead18b551787543a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cacc879239e3467c98a07cda058d4a9e",
              "IPY_MODEL_07838821f7294b2796f8b5a8575acee9",
              "IPY_MODEL_083b2a35ca4245c0ae7894cb3551b1d7"
            ],
            "layout": "IPY_MODEL_ddb602a7c9e04ddc8f4b8a03720728e9"
          }
        },
        "cacc879239e3467c98a07cda058d4a9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e0c11743d04847ed9cc28c52e9d00884",
            "placeholder": "​",
            "style": "IPY_MODEL_d684c641c3db4acf84fb265e15b98a4f",
            "value": "model-00002-of-00008.safetensors: 100%"
          }
        },
        "07838821f7294b2796f8b5a8575acee9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d2120375f26e4429b38de09fcaf6ce77",
            "max": 1946243936,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9c8b98bcbca1407195595bf239a5e936",
            "value": 1946243936
          }
        },
        "083b2a35ca4245c0ae7894cb3551b1d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e23416758e0245fe9805e917e734c8fa",
            "placeholder": "​",
            "style": "IPY_MODEL_2933efb681ce4bad934642e4be7ee1d4",
            "value": " 1.95G/1.95G [00:19&lt;00:00, 188MB/s]"
          }
        },
        "ddb602a7c9e04ddc8f4b8a03720728e9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e0c11743d04847ed9cc28c52e9d00884": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d684c641c3db4acf84fb265e15b98a4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d2120375f26e4429b38de09fcaf6ce77": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9c8b98bcbca1407195595bf239a5e936": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e23416758e0245fe9805e917e734c8fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2933efb681ce4bad934642e4be7ee1d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2405b3b96cba4dcb935c9494f6bb0e14": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_33d3d454b7194b149bbac9731617eb74",
              "IPY_MODEL_d1c44b4e4d5b4b77b8c80fed0ec7a272",
              "IPY_MODEL_d65966e71a1f4950a313fc46f3dfee59"
            ],
            "layout": "IPY_MODEL_a296f9cf790249029a75dfcf0b2154e4"
          }
        },
        "33d3d454b7194b149bbac9731617eb74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d106cdf95f534143bb53d5efd08f4aa1",
            "placeholder": "​",
            "style": "IPY_MODEL_8aeb6f154f4c4b9287441dc3b043cc7e",
            "value": "model-00003-of-00008.safetensors: 100%"
          }
        },
        "d1c44b4e4d5b4b77b8c80fed0ec7a272": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8083f742c1604a3787a9cc7ee9377726",
            "max": 1979781432,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5ffa0060a83b4a6eb8116cd9df16cbbf",
            "value": 1979781432
          }
        },
        "d65966e71a1f4950a313fc46f3dfee59": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b01d32e7efc847fd86f3f13e422d7ca9",
            "placeholder": "​",
            "style": "IPY_MODEL_31750614928e4a67936901bdbe26a82a",
            "value": " 1.98G/1.98G [00:21&lt;00:00, 223MB/s]"
          }
        },
        "a296f9cf790249029a75dfcf0b2154e4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d106cdf95f534143bb53d5efd08f4aa1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8aeb6f154f4c4b9287441dc3b043cc7e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8083f742c1604a3787a9cc7ee9377726": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5ffa0060a83b4a6eb8116cd9df16cbbf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b01d32e7efc847fd86f3f13e422d7ca9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "31750614928e4a67936901bdbe26a82a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "609c2afef1fb4b64ad62bdc2c944d067": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_64c692669c17481fa38fc0a6011c8e20",
              "IPY_MODEL_c09c1548046f4882a52036175d5e95b4",
              "IPY_MODEL_81528ad3ae774bcbb3ebf30abd0f2f51"
            ],
            "layout": "IPY_MODEL_e1d641ba80da4ebd94ab1664bc2bc7ab"
          }
        },
        "64c692669c17481fa38fc0a6011c8e20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ef920b3c931643d7b6be4cf69858a28d",
            "placeholder": "​",
            "style": "IPY_MODEL_8ab6368b4e294ef1a1f6edd151122b64",
            "value": "model-00004-of-00008.safetensors: 100%"
          }
        },
        "c09c1548046f4882a52036175d5e95b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_813bf907d60a4e099e500c2951766985",
            "max": 1946243984,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a3640d78d2d643f6b03d6b621e322954",
            "value": 1946243984
          }
        },
        "81528ad3ae774bcbb3ebf30abd0f2f51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_402c46f392ad4c6ab5612bb00b6baad4",
            "placeholder": "​",
            "style": "IPY_MODEL_a8c6bcb4664d4d94b8ea20ef12b6d412",
            "value": " 1.95G/1.95G [00:59&lt;00:00, 268MB/s]"
          }
        },
        "e1d641ba80da4ebd94ab1664bc2bc7ab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ef920b3c931643d7b6be4cf69858a28d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8ab6368b4e294ef1a1f6edd151122b64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "813bf907d60a4e099e500c2951766985": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a3640d78d2d643f6b03d6b621e322954": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "402c46f392ad4c6ab5612bb00b6baad4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a8c6bcb4664d4d94b8ea20ef12b6d412": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e702cfe78e6f4c5cb0238e36ff447978": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cd00868bd9ab4386b472bff8da1e8d62",
              "IPY_MODEL_e6fd6a6790f447aeb8cf8cded5cf6435",
              "IPY_MODEL_b4ef6e576c8e45978fd23f6f71fc5b82"
            ],
            "layout": "IPY_MODEL_f3967188815b43cd9c716287616b08aa"
          }
        },
        "cd00868bd9ab4386b472bff8da1e8d62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c3baf322302d42a188733774957c66f0",
            "placeholder": "​",
            "style": "IPY_MODEL_1de3defe6c8b490a88ca79726fe6f5db",
            "value": "model-00005-of-00008.safetensors: 100%"
          }
        },
        "e6fd6a6790f447aeb8cf8cded5cf6435": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f9f8812d88bb447e9da05eab2ce273d5",
            "max": 1979781448,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_25e70afd2315485398fbc81566f73762",
            "value": 1979781448
          }
        },
        "b4ef6e576c8e45978fd23f6f71fc5b82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_34740a1ca7f44199ae0b8293bc1abb99",
            "placeholder": "​",
            "style": "IPY_MODEL_e01737e501df4870915ce486c83fff89",
            "value": " 1.98G/1.98G [00:07&lt;00:00, 237MB/s]"
          }
        },
        "f3967188815b43cd9c716287616b08aa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c3baf322302d42a188733774957c66f0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1de3defe6c8b490a88ca79726fe6f5db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f9f8812d88bb447e9da05eab2ce273d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "25e70afd2315485398fbc81566f73762": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "34740a1ca7f44199ae0b8293bc1abb99": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e01737e501df4870915ce486c83fff89": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9e365a36530a4250b3f7cf8af7883c03": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b88f606e15384d7cb5d731cb27aadf9d",
              "IPY_MODEL_421f8c1020b74747b69abfcd99c50321",
              "IPY_MODEL_8613e05aba0343b7b1dc13f87fd0d51f"
            ],
            "layout": "IPY_MODEL_7f45be8f40d3405587451a3c7cbbadc9"
          }
        },
        "b88f606e15384d7cb5d731cb27aadf9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_42ba1ea2547c46849283d7e22122e52c",
            "placeholder": "​",
            "style": "IPY_MODEL_3b4bfe792a0d4bcca4d181089bbc97f2",
            "value": "model-00006-of-00008.safetensors: 100%"
          }
        },
        "421f8c1020b74747b69abfcd99c50321": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d30d313c3098401c957a0ad5f04d106c",
            "max": 1946243984,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_edacd0e71cce46e79f1364a5bfbe430e",
            "value": 1946243984
          }
        },
        "8613e05aba0343b7b1dc13f87fd0d51f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a18d34d03c5149da81cd167d9c520777",
            "placeholder": "​",
            "style": "IPY_MODEL_1fcf29ebc8c2477cb4d5bf5c551970de",
            "value": " 1.95G/1.95G [00:07&lt;00:00, 218MB/s]"
          }
        },
        "7f45be8f40d3405587451a3c7cbbadc9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "42ba1ea2547c46849283d7e22122e52c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3b4bfe792a0d4bcca4d181089bbc97f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d30d313c3098401c957a0ad5f04d106c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "edacd0e71cce46e79f1364a5bfbe430e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a18d34d03c5149da81cd167d9c520777": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1fcf29ebc8c2477cb4d5bf5c551970de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ab70bff948a444ed95e3e0f042a78c60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3b828e20267b483cadc5a9abe1d10632",
              "IPY_MODEL_1eb9b3fa347a4296bbaf41f5a4745946",
              "IPY_MODEL_c2bed511829f4ee7a9ac1d999dc9d289"
            ],
            "layout": "IPY_MODEL_a64bddcc84fd4329a017af663e2fdb70"
          }
        },
        "3b828e20267b483cadc5a9abe1d10632": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8626c29fc7d64eb8a771665ac90fc4a3",
            "placeholder": "​",
            "style": "IPY_MODEL_944160b9daee462bbf1478bb43854f5b",
            "value": "model-00007-of-00008.safetensors: 100%"
          }
        },
        "1eb9b3fa347a4296bbaf41f5a4745946": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e4316da06b5e4b7b88f723b756a2712f",
            "max": 1979781448,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_25df460d72154bd395211c2e189e821d",
            "value": 1979781448
          }
        },
        "c2bed511829f4ee7a9ac1d999dc9d289": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b6d38b6d3a9f4dfcbae7e4d186171f21",
            "placeholder": "​",
            "style": "IPY_MODEL_a68ca9b2117d4639a6c6db750e5e5bfd",
            "value": " 1.98G/1.98G [00:10&lt;00:00, 35.7MB/s]"
          }
        },
        "a64bddcc84fd4329a017af663e2fdb70": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8626c29fc7d64eb8a771665ac90fc4a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "944160b9daee462bbf1478bb43854f5b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e4316da06b5e4b7b88f723b756a2712f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "25df460d72154bd395211c2e189e821d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b6d38b6d3a9f4dfcbae7e4d186171f21": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a68ca9b2117d4639a6c6db750e5e5bfd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "34468dd047dc43ba88a8c9ec2ac73891": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b0572d512d8d40c7b27d313da8fbc629",
              "IPY_MODEL_49e8f738685842f59eec2ddc4936e5cd",
              "IPY_MODEL_b5edf43ba10f410b9e1000a1fdaa4fa3"
            ],
            "layout": "IPY_MODEL_875b18864fa0408f88bf7b27dc20e7a0"
          }
        },
        "b0572d512d8d40c7b27d313da8fbc629": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_69f21434a0e14ce8ba6356cc9703eab4",
            "placeholder": "​",
            "style": "IPY_MODEL_301d08ea805341c9b5e399dbdabac39e",
            "value": "model-00008-of-00008.safetensors: 100%"
          }
        },
        "49e8f738685842f59eec2ddc4936e5cd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7cda474c4e014282b71dab2019049036",
            "max": 815834680,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fdf861ffd194480983dab43c6dfec260",
            "value": 815834680
          }
        },
        "b5edf43ba10f410b9e1000a1fdaa4fa3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7e7b816b54a8482ebcf5de0f77a10aa4",
            "placeholder": "​",
            "style": "IPY_MODEL_fa0219f26f7348a5885f22daeaedb810",
            "value": " 816M/816M [00:06&lt;00:00, 198MB/s]"
          }
        },
        "875b18864fa0408f88bf7b27dc20e7a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "69f21434a0e14ce8ba6356cc9703eab4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "301d08ea805341c9b5e399dbdabac39e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7cda474c4e014282b71dab2019049036": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fdf861ffd194480983dab43c6dfec260": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7e7b816b54a8482ebcf5de0f77a10aa4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fa0219f26f7348a5885f22daeaedb810": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "02c244deba524ebaae97ad20be058658": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ccbc502f01984770810e5253a27547d9",
              "IPY_MODEL_9f0b00c89a904b55bbeaf48d34b77986",
              "IPY_MODEL_2009e7fe928b4842b8c84288d0b070e5"
            ],
            "layout": "IPY_MODEL_28a91d84cdb34fb8bba194454356aed4"
          }
        },
        "ccbc502f01984770810e5253a27547d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_94f80b17a07f4a33b7c3a102bad5a147",
            "placeholder": "​",
            "style": "IPY_MODEL_f4be5d2c385244cc9a6e48b73b70adb1",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "9f0b00c89a904b55bbeaf48d34b77986": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e7c962dbedc7497a933e0ce8ba2dc1f1",
            "max": 8,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1abf6f5e652b425a996d243af7ba9820",
            "value": 8
          }
        },
        "2009e7fe928b4842b8c84288d0b070e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4edde8462ebb4323836fe317706a70fb",
            "placeholder": "​",
            "style": "IPY_MODEL_3bc5a047ca3d473a8516ffb728c11291",
            "value": " 8/8 [01:00&lt;00:00,  6.62s/it]"
          }
        },
        "28a91d84cdb34fb8bba194454356aed4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "94f80b17a07f4a33b7c3a102bad5a147": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f4be5d2c385244cc9a6e48b73b70adb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e7c962dbedc7497a933e0ce8ba2dc1f1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1abf6f5e652b425a996d243af7ba9820": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4edde8462ebb4323836fe317706a70fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3bc5a047ca3d473a8516ffb728c11291": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "57bccd259b6941789ba51f8e003b71a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ce1826a2eb744306b3c97bde50a98f82",
              "IPY_MODEL_1c817ef799e14b55b3ca4a644190b8d5",
              "IPY_MODEL_8d84753b75eb401096e96f784a8c66b3"
            ],
            "layout": "IPY_MODEL_c9cf467c9bde45f9b94ec7e0bef0c614"
          }
        },
        "ce1826a2eb744306b3c97bde50a98f82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_75b42c387f224a298e7f218172fb040d",
            "placeholder": "​",
            "style": "IPY_MODEL_740674e7762d42228ee4ebaebc5cc703",
            "value": "generation_config.json: 100%"
          }
        },
        "1c817ef799e14b55b3ca4a644190b8d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f166f33bf8904a5d805aaee1ae1681bb",
            "max": 111,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_79982c12db6d4b15b8cd6d1821fe8a87",
            "value": 111
          }
        },
        "8d84753b75eb401096e96f784a8c66b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_332454bb77924402a2eef586bd85f2fb",
            "placeholder": "​",
            "style": "IPY_MODEL_440b8b2f30c64d6d9d234f660034fbc1",
            "value": " 111/111 [00:00&lt;00:00, 7.86kB/s]"
          }
        },
        "c9cf467c9bde45f9b94ec7e0bef0c614": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "75b42c387f224a298e7f218172fb040d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "740674e7762d42228ee4ebaebc5cc703": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f166f33bf8904a5d805aaee1ae1681bb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "79982c12db6d4b15b8cd6d1821fe8a87": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "332454bb77924402a2eef586bd85f2fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "440b8b2f30c64d6d9d234f660034fbc1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UM7HSXuPf2tO"
      },
      "outputs": [],
      "source": [
        "doc_path = r\"/content/2005.11401v4.pdf\""
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install pypdf"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZnCXQEzJggea",
        "outputId": "2bd14f7b-a371-4f5e-a95a-b39e8f72f3b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting pypdf\n",
            "  Downloading pypdf-4.3.0-py3-none-any.whl (295 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/295.7 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.2/295.7 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m295.7/295.7 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing_extensions>=4.0 in /usr/local/lib/python3.10/dist-packages (from pypdf) (4.12.2)\n",
            "Installing collected packages: pypdf\n",
            "Successfully installed pypdf-4.3.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install langchain_community"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U5rhh6Oxglko",
        "outputId": "31a131dd-8b52-4508-95df-428205764317"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting langchain_community\n",
            "  Downloading langchain_community-0.2.7-py3-none-any.whl (2.2 MB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/2.2 MB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.1/2.2 MB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m40.2 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m32.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.10/dist-packages (from langchain_community) (6.0.1)\n",
            "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain_community) (2.0.31)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain_community) (3.9.5)\n",
            "Collecting dataclasses-json<0.7,>=0.5.7 (from langchain_community)\n",
            "  Downloading dataclasses_json-0.6.7-py3-none-any.whl (28 kB)\n",
            "Collecting langchain<0.3.0,>=0.2.7 (from langchain_community)\n",
            "  Downloading langchain-0.2.9-py3-none-any.whl (987 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m987.7/987.7 kB\u001b[0m \u001b[31m46.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting langchain-core<0.3.0,>=0.2.12 (from langchain_community)\n",
            "  Downloading langchain_core-0.2.21-py3-none-any.whl (372 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m372.0/372.0 kB\u001b[0m \u001b[31m39.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting langsmith<0.2.0,>=0.1.0 (from langchain_community)\n",
            "  Downloading langsmith-0.1.90-py3-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.7/134.7 kB\u001b[0m \u001b[31m18.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain_community) (1.25.2)\n",
            "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain_community) (2.31.0)\n",
            "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain_community) (8.5.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (4.0.3)\n",
            "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json<0.7,>=0.5.7->langchain_community)\n",
            "  Downloading marshmallow-3.21.3-py3-none-any.whl (49 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.2/49.2 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typing-inspect<1,>=0.4.0 (from dataclasses-json<0.7,>=0.5.7->langchain_community)\n",
            "  Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
            "Collecting langchain-text-splitters<0.3.0,>=0.2.0 (from langchain<0.3.0,>=0.2.7->langchain_community)\n",
            "  Downloading langchain_text_splitters-0.2.2-py3-none-any.whl (25 kB)\n",
            "Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain<0.3.0,>=0.2.7->langchain_community) (2.8.2)\n",
            "Collecting jsonpatch<2.0,>=1.33 (from langchain-core<0.3.0,>=0.2.12->langchain_community)\n",
            "  Downloading jsonpatch-1.33-py2.py3-none-any.whl (12 kB)\n",
            "Requirement already satisfied: packaging<25,>=23.2 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.3.0,>=0.2.12->langchain_community) (24.1)\n",
            "Collecting orjson<4.0.0,>=3.9.14 (from langsmith<0.2.0,>=0.1.0->langchain_community)\n",
            "  Downloading orjson-3.10.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (141 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.1/141.1 kB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain_community) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain_community) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain_community) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain_community) (2024.7.4)\n",
            "Requirement already satisfied: typing-extensions>=4.6.0 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain_community) (4.12.2)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain_community) (3.0.3)\n",
            "Collecting jsonpointer>=1.9 (from jsonpatch<2.0,>=1.33->langchain-core<0.3.0,>=0.2.12->langchain_community)\n",
            "  Downloading jsonpointer-3.0.0-py2.py3-none-any.whl (7.6 kB)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain<0.3.0,>=0.2.7->langchain_community) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.20.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain<0.3.0,>=0.2.7->langchain_community) (2.20.1)\n",
            "Collecting mypy-extensions>=0.3.0 (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain_community)\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Installing collected packages: orjson, mypy-extensions, marshmallow, jsonpointer, typing-inspect, jsonpatch, langsmith, dataclasses-json, langchain-core, langchain-text-splitters, langchain, langchain_community\n",
            "Successfully installed dataclasses-json-0.6.7 jsonpatch-1.33 jsonpointer-3.0.0 langchain-0.2.9 langchain-core-0.2.21 langchain-text-splitters-0.2.2 langchain_community-0.2.7 langsmith-0.1.90 marshmallow-3.21.3 mypy-extensions-1.0.0 orjson-3.10.6 typing-inspect-0.9.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain_community.document_loaders import PyPDFLoader"
      ],
      "metadata": {
        "id": "T6B9RevHgtug"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loader = PyPDFLoader(doc_path)"
      ],
      "metadata": {
        "id": "zv-17_YGg05R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "docs = loader.load()"
      ],
      "metadata": {
        "id": "HPHlxKmig-Iz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.text_splitter import RecursiveCharacterTextSplitter"
      ],
      "metadata": {
        "id": "jVoOLO8ihEju"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)"
      ],
      "metadata": {
        "id": "Gw8gFj09hR2h"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "chunks = splitter.split_documents(docs)"
      ],
      "metadata": {
        "id": "jjsNSzZlhcS4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "chunks"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2zWpiTgthimG",
        "outputId": "e1ebb4b1-5ede-4d54-d4d3-bf65641e2b3f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 0}, page_content='Retrieval-Augmented Generation for\\nKnowledge-Intensive NLP Tasks\\nPatrick Lewis†‡, Ethan Perez⋆,\\nAleksandra Piktus†, Fabio Petroni†, Vladimir Karpukhin†, Naman Goyal†, Heinrich Küttler†,\\nMike Lewis†, Wen-tau Yih†, Tim Rocktäschel†‡, Sebastian Riedel†‡, Douwe Kiela†\\n†Facebook AI Research;‡University College London;⋆New York University;\\nplewis@fb.com\\nAbstract\\nLarge pre-trained language models have been shown to store factual knowledge'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 0}, page_content='in their parameters, and achieve state-of-the-art results when ﬁne-tuned on down-\\nstream NLP tasks. However, their ability to access and precisely manipulate knowl-\\nedge is still limited, and hence on knowledge-intensive tasks, their performance\\nlags behind task-speciﬁc architectures. Additionally, providing provenance for their\\ndecisions and updating their world knowledge remain open research problems. Pre-\\ntrained models with a differentiable access mechanism to explicit non-parametric'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 0}, page_content='memory have so far been only investigated for extractive downstream tasks. We\\nexplore a general-purpose ﬁne-tuning recipe for retrieval-augmented generation\\n(RAG) — models which combine pre-trained parametric and non-parametric mem-\\nory for language generation. We introduce RAG models where the parametric\\nmemory is a pre-trained seq2seq model and the non-parametric memory is a dense\\nvector index of Wikipedia, accessed with a pre-trained neural retriever. We com-'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 0}, page_content='pare two RAG formulations, one which conditions on the same retrieved passages\\nacross the whole generated sequence, and another which can use different passages\\nper token. We ﬁne-tune and evaluate our models on a wide range of knowledge-\\nintensive NLP tasks and set the state of the art on three open domain QA tasks,\\noutperforming parametric seq2seq models and task-speciﬁc retrieve-and-extract\\narchitectures. For language generation tasks, we ﬁnd that RAG models generate'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 0}, page_content='more speciﬁc, diverse and factual language than a state-of-the-art parametric-only\\nseq2seq baseline.\\n1 Introduction\\nPre-trained neural language models have been shown to learn a substantial amount of in-depth knowl-\\nedge from data [ 47]. They can do so without any access to an external memory, as a parameterized\\nimplicit knowledge base [ 51,52]. While this development is exciting, such models do have down-'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 0}, page_content='sides: They cannot easily expand or revise their memory, can’t straightforwardly provide insight into\\ntheir predictions, and may produce “hallucinations” [ 38]. Hybrid models that combine parametric\\nmemory with non-parametric (i.e., retrieval-based) memories [ 20,26,48] can address some of these\\nissues because knowledge can be directly revised and expanded, and accessed knowledge can be\\ninspected and interpreted. REALM [ 20] and ORQA [ 31], two recently introduced models that'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 0}, page_content='combine masked language models [ 8] with a differentiable retriever, have shown promising results,arXiv:2005.11401v4  [cs.CL]  12 Apr 2021'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 1}, page_content='The\\tDivine\\nComedy\\t(x) qQuery\\nEncoder\\nq(x)\\nMIPS p θGenerator \\xa0pθ\\n(Parametric)\\nMargin-\\nalize\\nThis\\t14th\\tcentury\\twork\\nis\\tdivided\\tinto\\t3\\nsections:\\t\"Inferno\",\\n\"Purgatorio\"\\t&\\n\"Paradiso\"\\t\\t\\t\\t\\t\\t\\t\\t\\t (y)End-to-End Backprop through q and\\xa0 p θ\\nBarack\\tObama\\twas\\nborn\\tin\\tHawaii. (x)\\nFact V eriﬁcation: Fact Querysupports \\t(y)\\nQuestion GenerationFact V eriﬁcation:\\nLabel GenerationDocument\\nIndexDefine\\t\"middle\\tear\" (x)\\nQuestion Answering:\\nQuestion QueryThe\\tmiddle\\tear\\tincludes\\nthe\\ttympanic\\tcavity\\tand'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 1}, page_content='the\\ttympanic\\tcavity\\tand\\nthe\\tthree\\tossicles.\\t\\t (y)\\nQuestion Answering:\\nAnswer GenerationRetriever pη\\n(Non-Parametric)\\nz 4\\nz3\\nz2\\nz 1d(z)\\nJeopardy Question\\nGeneration:\\nAnswer QueryFigure 1: Overview of our approach. We combine a pre-trained retriever ( Query Encoder +Document\\nIndex ) with a pre-trained seq2seq model ( Generator ) and ﬁne-tune end-to-end. For query x, we use\\nMaximum Inner Product Search (MIPS) to ﬁnd the top-K documents zi. For ﬁnal prediction y, we'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 1}, page_content='treatzas a latent variable and marginalize over seq2seq predictions given different documents.\\nbut have only explored open-domain extractive question answering. Here, we bring hybrid parametric\\nand non-parametric memory to the “workhorse of NLP,” i.e. sequence-to-sequence (seq2seq) models.\\nWe endow pre-trained, parametric-memory generation models with a non-parametric memory through\\na general-purpose ﬁne-tuning approach which we refer to as retrieval-augmented generation (RAG).'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 1}, page_content='We build RAG models where the parametric memory is a pre-trained seq2seq transformer, and the\\nnon-parametric memory is a dense vector index of Wikipedia, accessed with a pre-trained neural\\nretriever. We combine these components in a probabilistic model trained end-to-end (Fig. 1). The\\nretriever (Dense Passage Retriever [ 26], henceforth DPR) provides latent documents conditioned on\\nthe input, and the seq2seq model (BART [ 32]) then conditions on these latent documents together with'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 1}, page_content='the input to generate the output. We marginalize the latent documents with a top-K approximation,\\neither on a per-output basis (assuming the same document is responsible for all tokens) or a per-token\\nbasis (where different documents are responsible for different tokens). Like T5 [ 51] or BART, RAG\\ncan be ﬁne-tuned on any seq2seq task, whereby both the generator and retriever are jointly learned.'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 1}, page_content='There has been extensive previous work proposing architectures to enrich systems with non-parametric\\nmemory which are trained from scratch for speciﬁc tasks, e.g. memory networks [ 64,55], stack-\\naugmented networks [ 25] and memory layers [ 30]. In contrast, we explore a setting where both\\nparametric and non-parametric memory components are pre-trained and pre-loaded with extensive\\nknowledge. Crucially, by using pre-trained access mechanisms, the ability to access knowledge is'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 1}, page_content='present without additional training.\\nOur results highlight the beneﬁts of combining parametric and non-parametric memory with genera-\\ntion for knowledge-intensive tasks —tasks that humans could not reasonably be expected to perform\\nwithout access to an external knowledge source. Our RAG models achieve state-of-the-art results\\non open Natural Questions [ 29], WebQuestions [ 3] and CuratedTrec [ 2] and strongly outperform'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 1}, page_content='recent approaches that use specialised pre-training objectives on TriviaQA [ 24]. Despite these being\\nextractive tasks, we ﬁnd that unconstrained generation outperforms previous extractive approaches.\\nFor knowledge-intensive generation, we experiment with MS-MARCO [ 1] and Jeopardy question\\ngeneration, and we ﬁnd that our models generate responses that are more factual, speciﬁc, and\\ndiverse than a BART baseline. For FEVER [ 56] fact veriﬁcation, we achieve results within 4.3% of'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 1}, page_content='state-of-the-art pipeline models which use strong retrieval supervision. Finally, we demonstrate that\\nthe non-parametric memory can be replaced to update the models’ knowledge as the world changes.1\\n2 Methods\\nWe explore RAG models, which use the input sequence xto retrieve text documents zand use them\\nas additional context when generating the target sequence y. As shown in Figure 1, our models\\nleverage two components: (i) a retriever pη(z|x)with parameters ηthat returns (top-K truncated)'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 1}, page_content='distributions over text passages given a query xand (ii) a generator pθ(yi|x,z,y 1:i−1)parametrized\\n1Code to run experiments with RAG has been open-sourced as part of the HuggingFace Transform-\\ners Library [ 66] and can be found at https://github.com/huggingface/transformers/blob/master/\\nexamples/rag/ . An interactive demo of RAG models can be found at https://huggingface.co/rag/\\n2'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 2}, page_content='byθthat generates a current token based on a context of the previous i−1tokensy1:i−1, the original\\ninputxand a retrieved passage z.\\nTo train the retriever and generator end-to-end, we treat the retrieved document as a latent variable.\\nWe propose two models that marginalize over the latent documents in different ways to produce a\\ndistribution over generated text. In one approach, RAG-Sequence , the model uses the same document'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 2}, page_content='to predict each target token. The second approach, RAG-Token , can predict each target token based\\non a different document. In the following, we formally introduce both models and then describe the\\npηandpθcomponents, as well as the training and decoding procedure.\\n2.1 Models\\nRAG-Sequence Model The RAG-Sequence model uses the same retrieved document to generate\\nthe complete sequence . Technically, it treats the retrieved document as a single latent variable that'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 2}, page_content='is marginalized to get the seq2seq probability p(y|x)via a top-K approximation. Concretely, the\\ntop K documents are retrieved using the retriever, and the generator produces the output sequence\\nprobability for each document, which are then marginalized,\\npRAG-Sequence (y|x)≈∑\\nz∈top-k(p(·|x))pη(z|x)pθ(y|x,z) =∑\\nz∈top-k(p(·|x))pη(z|x)N∏\\nipθ(yi|x,z,y 1:i−1)\\nRAG-Token Model In the RAG-Token model we can draw a different latent document for each'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 2}, page_content='target token and marginalize accordingly. This allows the generator to choose content from several\\ndocuments when producing an answer. Concretely, the top K documents are retrieved using the\\nretriever, and then the generator produces a distribution for the next output token for each document,\\nbefore marginalizing, and repeating the process with the following output token, Formally, we deﬁne:\\npRAG-Token (y|x)≈N∏\\ni∑\\nz∈top-k(p(·|x))pη(z|x)pθ(yi|x,z,y 1:i−1)'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 2}, page_content='i∑\\nz∈top-k(p(·|x))pη(z|x)pθ(yi|x,z,y 1:i−1)\\nFinally, we note that RAG can be used for sequence classiﬁcation tasks by considering the target class\\nas a target sequence of length one, in which case RAG-Sequence and RAG-Token are equivalent.\\n2.2 Retriever: DPR\\nThe retrieval component pη(z|x)is based on DPR [26]. DPR follows a bi-encoder architecture:\\npη(z|x)∝exp(\\nd(z)⊤q(x))\\nd(z) =BERTd(z),q(x) =BERTq(x)'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 2}, page_content='d(z)⊤q(x))\\nd(z) =BERTd(z),q(x) =BERTq(x)\\nwhere d(z)is a dense representation of a document produced by a BERT BASE document encoder [8],\\nandq(x)a query representation produced by a query encoder , also based on BERT BASE. Calculating\\ntop-k (pη(·|x)), the list ofkdocumentszwith highest prior probability pη(z|x), is a Maximum Inner\\nProduct Search (MIPS) problem, which can be approximately solved in sub-linear time [ 23]. We use'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 2}, page_content='a pre-trained bi-encoder from DPR to initialize our retriever and to build the document index. This\\nretriever was trained to retrieve documents which contain answers to TriviaQA [ 24] questions and\\nNatural Questions [29]. We refer to the document index as the non-parametric memory .\\n2.3 Generator: BART\\nThe generator component pθ(yi|x,z,y 1:i−1)could be modelled using any encoder-decoder. We use\\nBART-large [ 32], a pre-trained seq2seq transformer [ 58] with 400M parameters. To combine the input'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 2}, page_content='xwith the retrieved content zwhen generating from BART, we simply concatenate them. BART was\\npre-trained using a denoising objective and a variety of different noising functions. It has obtained\\nstate-of-the-art results on a diverse set of generation tasks and outperforms comparably-sized T5\\nmodels [32]. We refer to the BART generator parameters θas the parametric memory henceforth.\\n2.4 Training\\nWe jointly train the retriever and generator components without any direct supervision on what'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 2}, page_content='document should be retrieved. Given a ﬁne-tuning training corpus of input/output pairs (xj,yj), we\\n3'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 3}, page_content='minimize the negative marginal log-likelihood of each target,∑\\nj−logp(yj|xj)using stochastic\\ngradient descent with Adam [ 28]. Updating the document encoder BERTdduring training is costly as\\nit requires the document index to be periodically updated as REALM does during pre-training [ 20].\\nWe do not ﬁnd this step necessary for strong performance, and keep the document encoder (and\\nindex) ﬁxed, only ﬁne-tuning the query encoder BERT qand the BART generator.\\n2.5 Decoding'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 3}, page_content='2.5 Decoding\\nAt test time, RAG-Sequence and RAG-Token require different ways to approximate arg maxyp(y|x).\\nRAG-Token The RAG-Token model can be seen as a standard, autoregressive seq2seq genera-\\ntor with transition probability: p′\\nθ(yi|x,y 1:i−1) =∑\\nz∈top-k(p(·|x))pη(zi|x)pθ(yi|x,zi,y1:i−1)To\\ndecode, we can plug p′\\nθ(yi|x,y 1:i−1)into a standard beam decoder.\\nRAG-Sequence For RAG-Sequence, the likelihood p(y|x)does not break into a conventional per-'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 3}, page_content='token likelihood, hence we cannot solve it with a single beam search. Instead, we run beam search for\\neach document z, scoring each hypothesis using pθ(yi|x,z,y 1:i−1). This yields a set of hypotheses\\nY, some of which may not have appeared in the beams of all documents. To estimate the probability\\nof an hypothesis ywe run an additional forward pass for each document zfor whichydoes not\\nappear in the beam, multiply generator probability with pη(z|x)and then sum the probabilities across'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 3}, page_content='beams for the marginals. We refer to this decoding procedure as “Thorough Decoding.” For longer\\noutput sequences,|Y|can become large, requiring many forward passes. For more efﬁcient decoding,\\nwe can make a further approximation that pθ(y|x,zi)≈0whereywas not generated during beam\\nsearch from x,zi. This avoids the need to run additional forward passes once the candidate set Yhas\\nbeen generated. We refer to this decoding procedure as “Fast Decoding.”\\n3 Experiments'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 3}, page_content='3 Experiments\\nWe experiment with RAG in a wide range of knowledge-intensive tasks. For all experiments, we use\\na single Wikipedia dump for our non-parametric knowledge source. Following Lee et al. [31] and\\nKarpukhin et al. [26], we use the December 2018 dump. Each Wikipedia article is split into disjoint\\n100-word chunks, to make a total of 21M documents. We use the document encoder to compute an\\nembedding for each document, and build a single MIPS index using FAISS [ 23] with a Hierarchical'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 3}, page_content='Navigable Small World approximation for fast retrieval [ 37]. During training, we retrieve the top\\nkdocuments for each query. We consider k∈{5,10}for training and set kfor test time using dev\\ndata. We now discuss experimental details for each task.\\n3.1 Open-domain Question Answering\\nOpen-domain question answering (QA) is an important real-world application and common testbed\\nfor knowledge-intensive tasks [ 20]. We treat questions and answers as input-output text pairs (x,y)'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 3}, page_content='and train RAG by directly minimizing the negative log-likelihood of answers. We compare RAG to\\nthe popular extractive QA paradigm [ 5,7,31,26], where answers are extracted spans from retrieved\\ndocuments, relying primarily on non-parametric knowledge. We also compare to “Closed-Book\\nQA” approaches [ 52], which, like RAG, generate answers, but which do not exploit retrieval, instead\\nrelying purely on parametric knowledge. We consider four popular open-domain QA datasets: Natural'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 3}, page_content='Questions (NQ) [ 29], TriviaQA (TQA) [ 24]. WebQuestions (WQ) [ 3] and CuratedTrec (CT) [ 2]. As\\nCT and WQ are small, we follow DPR [ 26] by initializing CT and WQ models with our NQ RAG\\nmodel. We use the same train/dev/test splits as prior work [ 31,26] and report Exact Match (EM)\\nscores. For TQA, to compare with T5 [52], we also evaluate on the TQA Wiki test set.\\n3.2 Abstractive Question Answering\\nRAG models can go beyond simple extractive QA and answer questions with free-form, abstractive'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 3}, page_content='text generation. To test RAG’s natural language generation (NLG) in a knowledge-intensive setting,\\nwe use the MSMARCO NLG task v2.1 [ 43]. The task consists of questions, ten gold passages\\nretrieved from a search engine for each question, and a full sentence answer annotated from the\\nretrieved passages. We do not use the supplied passages, only the questions and answers, to treat\\n4'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 4}, page_content='MSMARCO as an open-domain abstractive QA task. MSMARCO has some questions that cannot be\\nanswered in a way that matches the reference answer without access to the gold passages, such as\\n“What is the weather in V olcano, CA?” so performance will be lower without using gold passages.\\nWe also note that some MSMARCO questions cannot be answered using Wikipedia alone. Here,\\nRAG can rely on parametric knowledge to generate reasonable responses.\\n3.3 Jeopardy Question Generation'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 4}, page_content='3.3 Jeopardy Question Generation\\nTo evaluate RAG’s generation abilities in a non-QA setting, we study open-domain question gen-\\neration. Rather than use questions from standard open-domain QA tasks, which typically consist\\nof short, simple questions, we propose the more demanding task of generating Jeopardy questions.\\nJeopardy is an unusual format that consists of trying to guess an entity from a fact about that entity.'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 4}, page_content='For example, “The World Cup” is the answer to the question “In 1986 Mexico scored as the ﬁrst\\ncountry to host this international sports competition twice.” As Jeopardy questions are precise,\\nfactual statements, generating Jeopardy questions conditioned on their answer entities constitutes a\\nchallenging knowledge-intensive generation task.\\nWe use the splits from SearchQA [ 10], with 100K train, 14K dev, and 27K test examples. As'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 4}, page_content='this is a new task, we train a BART model for comparison. Following [ 67], we evaluate using the\\nSQuAD-tuned Q-BLEU-1 metric [ 42]. Q-BLEU is a variant of BLEU with a higher weight for\\nmatching entities and has higher correlation with human judgment for question generation than\\nstandard metrics. We also perform two human evaluations, one to assess generation factuality, and\\none for speciﬁcity. We deﬁne factuality as whether a statement can be corroborated by trusted external'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 4}, page_content='sources, and speciﬁcity as high mutual dependence between the input and output [ 33]. We follow\\nbest practice and use pairwise comparative evaluation [ 34]. Evaluators are shown an answer and two\\ngenerated questions, one from BART and one from RAG. They are then asked to pick one of four\\noptions—quuestion A is better, question B is better, both are good, or neither is good.\\n3.4 Fact Veriﬁcation\\nFEVER [ 56] requires classifying whether a natural language claim is supported or refuted by'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 4}, page_content='Wikipedia, or whether there is not enough information to decide. The task requires retrieving\\nevidence from Wikipedia relating to the claim and then reasoning over this evidence to classify\\nwhether the claim is true, false, or unveriﬁable from Wikipedia alone. FEVER is a retrieval problem\\ncoupled with an challenging entailment reasoning task. It also provides an appropriate testbed for\\nexploring the RAG models’ ability to handle classiﬁcation rather than generation. We map FEVER'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 4}, page_content='class labels (supports, refutes, or not enough info) to single output tokens and directly train with\\nclaim-class pairs. Crucially, unlike most other approaches to FEVER, we do not use supervision on\\nretrieved evidence. In many real-world applications, retrieval supervision signals aren’t available, and\\nmodels that do not require such supervision will be applicable to a wider range of tasks. We explore'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 4}, page_content='two variants: the standard 3-way classiﬁcation task (supports/refutes/not enough info) and the 2-way\\n(supports/refutes) task studied in Thorne and Vlachos [57]. In both cases we report label accuracy.\\n4 Results\\n4.1 Open-domain Question Answering\\nTable 1 shows results for RAG along with state-of-the-art models. On all four open-domain QA\\ntasks, RAG sets a new state of the art (only on the T5-comparable split for TQA). RAG combines'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 4}, page_content='the generation ﬂexibility of the “closed-book” (parametric only) approaches and the performance of\\n\"open-book\" retrieval-based approaches. Unlike REALM and T5+SSM, RAG enjoys strong results\\nwithout expensive, specialized “salient span masking” pre-training [ 20]. It is worth noting that RAG’s\\nretriever is initialized using DPR’s retriever, which uses retrieval supervision on Natural Questions\\nand TriviaQA. RAG compares favourably to the DPR QA system, which uses a BERT-based “cross-'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 4}, page_content='encoder” to re-rank documents, along with an extractive reader. RAG demonstrates that neither a\\nre-ranker nor extractive reader is necessary for state-of-the-art performance.\\nThere are several advantages to generating answers even when it is possible to extract them. Docu-\\nments with clues about the answer but do not contain the answer verbatim can still contribute towards\\na correct answer being generated, which is not possible with standard extractive approaches, leading\\n5'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 5}, page_content='Table 1: Open-Domain QA Test Scores. For TQA,\\nleft column uses the standard test set for Open-\\nDomain QA, right column uses the TQA-Wiki\\ntest set. See Appendix D for further details.\\nModel NQ TQA WQ CT\\nClosed\\nBookT5-11B [52] 34.5 - /50.1 37.4 -\\nT5-11B+SSM[52] 36.6 - /60.5 44.7 -\\nOpen\\nBookREALM [20] 40.4 - / - 40.7 46.8\\nDPR [26] 41.5 57.9/ - 41.1 50.6\\nRAG-Token 44.1 55.2/66.1 45.5 50.0\\nRAG-Seq. 44.5 56.8/ 68.0 45.2 52.2Table 2: Generation and classiﬁcation Test Scores.'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 5}, page_content='MS-MARCO SotA is [ 4], FEVER-3 is [ 68] and\\nFEVER-2 is [ 57] *Uses gold context/evidence.\\nBest model without gold access underlined.\\nModel Jeopardy MSMARCO FVR3 FVR2\\nB-1 QB-1 R-L B-1 Label Acc.\\nSotA - - 49.8*49.9*76.8 92.2 *\\nBART 15.1 19.7 38.2 41.6 64.0 81.1\\nRAG-Tok. 17.3 22.2 40.1 41.572.5 89.5RAG-Seq. 14.7 21.4 40.8 44.2\\nto more effective marginalization over documents. Furthermore, RAG can generate correct answers'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 5}, page_content='even when the correct answer is not in any retrieved document, achieving 11.8% accuracy in such\\ncases for NQ, where an extractive model would score 0%.\\n4.2 Abstractive Question Answering\\nAs shown in Table 2, RAG-Sequence outperforms BART on Open MS-MARCO NLG by 2.6 Bleu\\npoints and 2.6 Rouge-L points. RAG approaches state-of-the-art model performance, which is\\nimpressive given that (i) those models access gold passages with speciﬁc information required to'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 5}, page_content='generate the reference answer , (ii) many questions are unanswerable without the gold passages, and\\n(iii) not all questions are answerable from Wikipedia alone. Table 3 shows some generated answers\\nfrom our models. Qualitatively, we ﬁnd that RAG models hallucinate less and generate factually\\ncorrect text more often than BART. Later, we also show that RAG generations are more diverse than\\nBART generations (see §4.5).\\n4.3 Jeopardy Question Generation'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 5}, page_content='4.3 Jeopardy Question Generation\\nTable 2 shows that RAG-Token performs better than RAG-Sequence on Jeopardy question generation,\\nwith both models outperforming BART on Q-BLEU-1. 4 shows human evaluation results, over 452\\npairs of generations from BART and RAG-Token. Evaluators indicated that BART was more factual\\nthan RAG in only 7.1% of cases, while RAG was more factual in 42.7% of cases, and both RAG and'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 5}, page_content='BART were factual in a further 17% of cases, clearly demonstrating the effectiveness of RAG on\\nthe task over a state-of-the-art generation model. Evaluators also ﬁnd RAG generations to be more\\nspeciﬁc by a large margin. Table 3 shows typical generations from each model.\\nJeopardy questions often contain two separate pieces of information, and RAG-Token may perform\\nbest because it can generate responses that combine content from several documents. Figure 2 shows'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 5}, page_content='an example. When generating “Sun”, the posterior is high for document 2 which mentions “The\\nSun Also Rises”. Similarly, document 1 dominates the posterior when “A Farewell to Arms” is\\ngenerated. Intriguingly, after the ﬁrst token of each book is generated, the document posterior ﬂattens.\\nThis observation suggests that the generator can complete the titles without depending on speciﬁc\\ndocuments. In other words, the model’s parametric knowledge is sufﬁcient to complete the titles. We'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 5}, page_content='ﬁnd evidence for this hypothesis by feeding the BART-only baseline with the partial decoding \"The\\nSun. BART completes the generation \"The SunAlso Rises\" isanovel bythis author of\"The Sun\\nAlso Rises\" indicating the title \"The Sun Also Rises\" is stored in BART’s parameters. Similarly,\\nBART will complete the partial decoding \"The SunAlso Rises\" isanovel bythis author of\"A\\nwith \"The SunAlso Rises\" isanovel bythis author of\"AFarewell toArms\" . This example shows'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 5}, page_content='how parametric and non-parametric memories work together —the non-parametric component helps\\nto guide the generation, drawing out speciﬁc knowledge stored in the parametric memory.\\n4.4 Fact Veriﬁcation\\nTable 2 shows our results on FEVER. For 3-way classiﬁcation, RAG scores are within 4.3% of\\nstate-of-the-art models, which are complex pipeline systems with domain-speciﬁc architectures and\\nsubstantial engineering, trained using intermediate retrieval supervision, which RAG does not require.\\n6'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 6}, page_content='Document 1 : his works are considered classics of American\\nliterature ... His wartime experiences formed the basis for his novel\\n”A Farewell to Arms” (1929) ...\\nDocument 2 : ... artists of the 1920s ”Lost Generation” expatriate\\ncommunity. His debut novel, ”The Sun Also Rises” , was published\\nin 1926.\\nBOS”\\nTheSunAlsoRises”isa\\nnovelbythis\\nauthorof”A\\nFarewellto\\nArms”Doc 1\\nDoc 2\\nDoc 3\\nDoc 4\\nDoc 5Figure 2: RAG-Token document posterior p(zi|x,yi,y−i)for each generated token for input “Hem-'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 6}, page_content='ingway\" for Jeopardy generation with 5 retrieved documents. The posterior for document 1 is high\\nwhen generating “A Farewell to Arms\" and for document 2 when generating “The Sun Also Rises\".\\nTable 3: Examples from generation tasks. RAG models generate more speciﬁc and factually accurate\\nresponses. ‘?’ indicates factually incorrect responses, * indicates partially correct responses.\\nTask Input Model Generation\\nMS-\\nMARCOdeﬁne middle'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 6}, page_content='Task Input Model Generation\\nMS-\\nMARCOdeﬁne middle\\nearBART?The middle ear is the part of the ear between the middle ear and the nose.\\nRAG-T The middle ear is the portion of the ear internal to the eardrum.\\nRAG-S The middle ear includes the tympanic cavity and the three ossicles.\\nwhat currency\\nneeded in\\nscotlandBART The currency needed in Scotland is Pound sterling.\\nRAG-T Pound is the currency needed in Scotland.\\nRAG-S The currency needed in Scotland is the pound sterling.\\nJeopardy\\nQuestion\\nGener'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 6}, page_content='Jeopardy\\nQuestion\\nGener\\n-ationWashingtonBART?This state has the largest number of counties in the U.S.\\nRAG-T It’s the only U.S. state named for a U.S. president\\nRAG-S It’s the state where you’ll ﬁnd Mount Rainier National Park\\nThe Divine\\nComedyBART*This epic poem by Dante is divided into 3 parts: the Inferno, the Purgatorio & the Purgatorio\\nRAG-T Dante’s \"Inferno\" is the ﬁrst part of this epic poem\\nRAG-S This 14th century work is divided into 3 sections: \"Inferno\", \"Purgatorio\" & \"Paradiso\"'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 6}, page_content='For 2-way classiﬁcation, we compare against Thorne and Vlachos [57], who train RoBERTa [ 35]\\nto classify the claim as true or false given the gold evidence sentence. RAG achieves an accuracy\\nwithin 2.7% of this model, despite being supplied with only the claim and retrieving its own evidence.\\nWe also analyze whether documents retrieved by RAG correspond to documents annotated as gold\\nevidence in FEVER. We calculate the overlap in article titles between the top kdocuments retrieved'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 6}, page_content='by RAG and gold evidence annotations. We ﬁnd that the top retrieved document is from a gold article\\nin 71% of cases, and a gold article is present in the top 10 retrieved articles in 90% of cases.\\n4.5 Additional Results\\nGeneration Diversity Section 4.3 shows that RAG models are more factual and speciﬁc than\\nBART for Jeopardy question generation. Following recent work on diversity-promoting decoding'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 6}, page_content='[33,59,39], we also investigate generation diversity by calculating the ratio of distinct ngrams to\\ntotal ngrams generated by different models. Table 5 shows that RAG-Sequence’s generations are\\nmore diverse than RAG-Token’s, and both are signiﬁcantly more diverse than BART without needing\\nany diversity-promoting decoding.\\nRetrieval Ablations A key feature of RAG is learning to retrieve relevant information for the task.'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 6}, page_content='To assess the effectiveness of the retrieval mechanism, we run ablations where we freeze the retriever\\nduring training. As shown in Table 6, learned retrieval improves results for all tasks.\\nWe compare RAG’s dense retriever to a word overlap-based BM25 retriever [ 53]. Here, we replace\\nRAG’s retriever with a ﬁxed BM25 system, and use BM25 retrieval scores as logits when calculating\\np(z|x). Table 6 shows the results. For FEVER, BM25 performs best, perhaps since FEVER claims are'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 6}, page_content='heavily entity-centric and thus well-suited for word overlap-based retrieval. Differentiable retrieval\\nimproves results on all other tasks, especially for Open-Domain QA, where it is crucial.\\nIndex hot-swapping An advantage of non-parametric memory models like RAG is that knowledge\\ncan be easily updated at test time. Parametric-only models like T5 or BART need further training to\\nupdate their behavior as the world changes. To demonstrate, we build an index using the DrQA [ 5]'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 6}, page_content='Wikipedia dump from December 2016 and compare outputs from RAG using this index to the newer\\nindex from our main results (December 2018). We prepare a list of 82 world leaders who had changed\\n7'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 7}, page_content='Table 4: Human assessments for the Jeopardy\\nQuestion Generation Task.\\nFactuality Speciﬁcity\\nBART better 7.1% 16.8%\\nRAG better 42.7% 37.4%\\nBoth good 11.7% 11.8%\\nBoth poor 17.7% 6.9%\\nNo majority 20.8% 20.1%Table 5: Ratio of distinct to total tri-grams for\\ngeneration tasks.\\nMSMARCO Jeopardy QGen\\nGold 89.6% 90.0%\\nBART 70.7% 32.4%\\nRAG-Token 77.8% 46.8%\\nRAG-Seq. 83.5% 53.8%\\nTable 6: Ablations on the dev set. As FEVER is a classiﬁcation task, both RAG models are equivalent.'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 7}, page_content='Model NQ TQA WQ CT Jeopardy-QGen MSMarco FVR-3 FVR-2\\nExact Match B-1 QB-1 R-L B-1 Label Accuracy\\nRAG-Token-BM25 29.7 41.5 32.1 33.1 17.5 22.3 55.5 48.475.1 91.6RAG-Sequence-BM25 31.8 44.1 36.6 33.8 11.1 19.5 56.5 46.9\\nRAG-Token-Frozen 37.8 50.1 37.1 51.1 16.7 21.7 55.9 49.472.9 89.4RAG-Sequence-Frozen 41.2 52.1 41.8 52.6 11.8 19.6 56.7 47.3\\nRAG-Token 43.5 54.8 46.5 51.9 17.9 22.6 56.2 49.474.5 90.6RAG-Sequence 44.0 55.8 44.9 53.4 15.3 21.5 57.2 47.5'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 7}, page_content='between these dates and use a template “Who is {position}?” (e.g. “Who is the President of Peru?”)\\nto query our NQ RAG model with each index. RAG answers 70% correctly using the 2016 index for\\n2016 world leaders and 68% using the 2018 index for 2018 world leaders. Accuracy with mismatched\\nindices is low (12% with the 2018 index and 2016 leaders, 4% with the 2016 index and 2018 leaders).\\nThis shows we can update RAG’s world knowledge by simply replacing its non-parametric memory.'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 7}, page_content='Effect of Retrieving more documents Models are trained with either 5 or 10 retrieved latent\\ndocuments, and we do not observe signiﬁcant differences in performance between them. We have the\\nﬂexibility to adjust the number of retrieved documents at test time, which can affect performance and\\nruntime. Figure 3 (left) shows that retrieving more documents at test time monotonically improves\\nOpen-domain QA results for RAG-Sequence, but performance peaks for RAG-Token at 10 retrieved'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 7}, page_content='documents. Figure 3 (right) shows that retrieving more documents leads to higher Rouge-L for\\nRAG-Token at the expense of Bleu-1, but the effect is less pronounced for RAG-Sequence.\\n10 20 30 40 50\\nKR e t r i e v e dD o c s394041424344NQ Exact MatchRAG-Tok\\nRAG-Seq\\n10 20 30 40 50\\nKR e t r i e v e dD o c s4050607080NQ Answer Recall @ KRAG-Tok\\nRAG-Seq\\nFixed DPR\\nBM25\\n10 20 30 40 50\\nKR e t r i e v e dD o c s4850525456Bleu-1 / Rouge-L scoreRAG-Tok R-L\\nRAG-Tok B-1\\nRAG-Seq R-L\\nRAG-Seq B-1'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 7}, page_content='RAG-Tok B-1\\nRAG-Seq R-L\\nRAG-Seq B-1\\nFigure 3: Left: NQ performance as more documents are retrieved. Center: Retrieval recall perfor-\\nmance in NQ. Right: MS-MARCO Bleu-1 and Rouge-L as more documents are retrieved.\\n5 Related Work\\nSingle-Task Retrieval Prior work has shown that retrieval improves performance across a variety of\\nNLP tasks when considered in isolation. Such tasks include open-domain question answering [ 5,29],'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 7}, page_content='fact checking [ 56], fact completion [ 48], long-form question answering [ 12], Wikipedia article\\ngeneration [ 36], dialogue [ 41,65,9,13], translation [ 17], and language modeling [ 19,27]. Our\\nwork uniﬁes previous successes in incorporating retrieval into individual tasks, showing that a single\\nretrieval-based architecture is capable of achieving strong performance across several tasks.\\n8'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 8}, page_content='General-Purpose Architectures for NLP Prior work on general-purpose architectures for NLP\\ntasks has shown great success without the use of retrieval. A single, pre-trained language model\\nhas been shown to achieve strong performance on various classiﬁcation tasks in the GLUE bench-\\nmarks [ 60,61] after ﬁne-tuning [ 49,8]. GPT-2 [ 50] later showed that a single, left-to-right, pre-trained\\nlanguage model could achieve strong performance across both discriminative and generative tasks.'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 8}, page_content='For further improvement, BART [ 32] and T5 [ 51,52] propose a single, pre-trained encoder-decoder\\nmodel that leverages bi-directional attention to achieve stronger performance on discriminative\\nand generative tasks. Our work aims to expand the space of possible tasks with a single, uniﬁed\\narchitecture, by learning a retrieval module to augment pre-trained, generative language models.\\nLearned Retrieval There is signiﬁcant work on learning to retrieve documents in information'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 8}, page_content='retrieval, more recently with pre-trained, neural language models [ 44,26] similar to ours. Some\\nwork optimizes the retrieval module to aid in a speciﬁc, downstream task such as question answering,\\nusing search [ 46], reinforcement learning [ 6,63,62], or a latent variable approach [ 31,20] as in our\\nwork. These successes leverage different retrieval-based architectures and optimization techniques to'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 8}, page_content='achieve strong performance on a single task, while we show that a single retrieval-based architecture\\ncan be ﬁne-tuned for strong performance on a variety of tasks.\\nMemory-based Architectures Our document index can be seen as a large external memory for\\nneural networks to attend to, analogous to memory networks [ 64,55]. Concurrent work [ 14] learns\\nto retrieve a trained embedding for each entity in the input, rather than to retrieve raw text as in our'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 8}, page_content='work. Other work improves the ability of dialog models to generate factual text by attending over\\nfact embeddings [ 15,13]. A key feature of our memory is that it is comprised of raw text rather\\ndistributed representations, which makes the memory both (i) human-readable, lending a form of\\ninterpretability to our model, and (ii) human-writable, enabling us to dynamically update the model’s\\nmemory by editing the document index. This approach has also been used in knowledge-intensive'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 8}, page_content='dialog, where generators have been conditioned on retrieved text directly, albeit obtained via TF-IDF\\nrather than end-to-end learnt retrieval [9].\\nRetrieve-and-Edit approaches Our method shares some similarities with retrieve-and-edit style\\napproaches, where a similar training input-output pair is retrieved for a given input, and then edited\\nto provide a ﬁnal output. These approaches have proved successful in a number of domains including'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 8}, page_content='Machine Translation [ 18,22] and Semantic Parsing [ 21]. Our approach does have several differences,\\nincluding less of emphasis on lightly editing a retrieved item, but on aggregating content from several\\npieces of retrieved content, as well as learning latent retrieval, and retrieving evidence documents\\nrather than related training pairs. This said, RAG techniques may work well in these settings, and\\ncould represent promising future work.\\n6 Discussion'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 8}, page_content='6 Discussion\\nIn this work, we presented hybrid generation models with access to parametric and non-parametric\\nmemory. We showed that our RAG models obtain state of the art results on open-domain QA. We\\nfound that people prefer RAG’s generation over purely parametric BART, ﬁnding RAG more factual\\nand speciﬁc. We conducted an thorough investigation of the learned retrieval component, validating\\nits effectiveness, and we illustrated how the retrieval index can be hot-swapped to update the model'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 8}, page_content='without requiring any retraining. In future work, it may be fruitful to investigate if the two components\\ncan be jointly pre-trained from scratch, either with a denoising objective similar to BART or some\\nanother objective. Our work opens up new research directions on how parametric and non-parametric\\nmemories interact and how to most effectively combine them, showing promise in being applied to a\\nwide variety of NLP tasks.\\n9'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 9}, page_content='Broader Impact\\nThis work offers several positive societal beneﬁts over previous work: the fact that it is more\\nstrongly grounded in real factual knowledge (in this case Wikipedia) makes it “hallucinate” less\\nwith generations that are more factual, and offers more control and interpretability. RAG could be\\nemployed in a wide variety of scenarios with direct beneﬁt to society, for example by endowing it'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 9}, page_content='with a medical index and asking it open-domain questions on that topic, or by helping people be more\\neffective at their jobs.\\nWith these advantages also come potential downsides: Wikipedia, or any potential external knowledge\\nsource, will probably never be entirely factual and completely devoid of bias. Since RAG can be\\nemployed as a language model, similar concerns as for GPT-2 [ 50] are valid here, although arguably'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 9}, page_content='to a lesser extent, including that it might be used to generate abuse, faked or misleading content in\\nthe news or on social media; to impersonate others; or to automate the production of spam/phishing\\ncontent [ 54]. Advanced language models may also lead to the automation of various jobs in the\\ncoming decades [ 16]. In order to mitigate these risks, AI systems could be employed to ﬁght against\\nmisleading content and automated spam/phishing.\\nAcknowledgments'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 9}, page_content='Acknowledgments\\nThe authors would like to thank the reviewers for their thoughtful and constructive feedback on this\\npaper, as well as HuggingFace for their help in open-sourcing code to run RAG models. The authors\\nwould also like to thank Kyunghyun Cho and Sewon Min for productive discussions and advice. EP\\nthanks supports from the NSF Graduate Research Fellowship. PL is supported by the FAIR PhD\\nprogram.\\nReferences'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 9}, page_content='program.\\nReferences\\n[1]Payal Bajaj, Daniel Campos, Nick Craswell, Li Deng, Jianfeng Gao, Xiaodong Liu, Rangan\\nMajumder, Andrew McNamara, Bhaskar Mitra, Tri Nguyen, Mir Rosenberg, Xia Song, Alina\\nStoica, Saurabh Tiwary, and Tong Wang. MS MARCO: A Human Generated MAchine\\nReading COmprehension Dataset. arXiv:1611.09268 [cs] , November 2016. URL http:\\n//arxiv.org/abs/1611.09268 . arXiv: 1611.09268.\\n[2]Petr Baudiš and Jan Šediv `y. Modeling of the question answering task in the yodaqa system. In'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 9}, page_content='International Conference of the Cross-Language Evaluation Forum for European Languages ,\\npages 222–228. Springer, 2015. URL https://link.springer.com/chapter/10.1007%\\n2F978-3-319-24027-5_20 .\\n[3]Jonathan Berant, Andrew Chou, Roy Frostig, and Percy Liang. Semantic Parsing on Freebase\\nfrom Question-Answer Pairs. In Proceedings of the 2013 Conference on Empirical Methods\\nin Natural Language Processing , pages 1533–1544, Seattle, Washington, USA, October 2013.'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 9}, page_content='Association for Computational Linguistics. URL http://www.aclweb.org/anthology/\\nD13-1160 .\\n[4]Bin Bi, Chenliang Li, Chen Wu, Ming Yan, and Wei Wang. Palm: Pre-training an autoencod-\\ning&autoregressive language model for context-conditioned generation. ArXiv , abs/2004.07159,\\n2020. URL https://arxiv.org/abs/2004.07159 .\\n[5]Danqi Chen, Adam Fisch, Jason Weston, and Antoine Bordes. Reading Wikipedia to Answer\\nOpen-Domain Questions. In Proceedings of the 55th Annual Meeting of the Association for'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 9}, page_content='Computational Linguistics (Volume 1: Long Papers) , pages 1870–1879, Vancouver, Canada,\\nJuly 2017. Association for Computational Linguistics. doi: 10.18653/v1/P17-1171. URL\\nhttps://www.aclweb.org/anthology/P17-1171 .\\n[6]Eunsol Choi, Daniel Hewlett, Jakob Uszkoreit, Illia Polosukhin, Alexandre Lacoste, and\\nJonathan Berant. Coarse-to-ﬁne question answering for long documents. In Proceedings of the\\n55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers) ,'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 9}, page_content='pages 209–220, Vancouver, Canada, July 2017. Association for Computational Linguistics. doi:\\n10.18653/v1/P17-1020. URL https://www.aclweb.org/anthology/P17-1020 .\\n10'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 10}, page_content='[7]Christopher Clark and Matt Gardner. Simple and Effective Multi-Paragraph Reading Compre-\\nhension. arXiv:1710.10723 [cs] , October 2017. URL http://arxiv.org/abs/1710.10723 .\\narXiv: 1710.10723.\\n[8]Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. BERT: Pre-training of\\nDeep Bidirectional Transformers for Language Understanding. In Proceedings of the 2019 Con-\\nference of the North American Chapter of the Association for Computational Linguistics: Human'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 10}, page_content='Language Technologies, Volume 1 (Long and Short Papers) , pages 4171–4186, Minneapolis,\\nMinnesota, June 2019. Association for Computational Linguistics. doi: 10.18653/v1/N19-1423.\\nURL https://www.aclweb.org/anthology/N19-1423 .\\n[9]Emily Dinan, Stephen Roller, Kurt Shuster, Angela Fan, Michael Auli, and Jason Weston. Wiz-\\nard of wikipedia: Knowledge-powered conversational agents. In International Conference on\\nLearning Representations , 2019. URL https://openreview.net/forum?id=r1l73iRqKm .'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 10}, page_content='[10] Matthew Dunn, Levent Sagun, Mike Higgins, V . Ugur Guney, V olkan Cirik, and Kyunghyun\\nCho. SearchQA: A New Q&A Dataset Augmented with Context from a Search Engine.\\narXiv:1704.05179 [cs] , April 2017. URL http://arxiv.org/abs/1704.05179 . arXiv:\\n1704.05179.\\n[11] Angela Fan, Mike Lewis, and Yann Dauphin. Hierarchical neural story generation. In Proceed-\\nings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1:'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 10}, page_content='Long Papers) , pages 889–898, Melbourne, Australia, July 2018. Association for Computational\\nLinguistics. doi: 10.18653/v1/P18-1082. URL https://www.aclweb.org/anthology/\\nP18-1082 .\\n[12] Angela Fan, Yacine Jernite, Ethan Perez, David Grangier, Jason Weston, and Michael Auli. ELI5:\\nLong form question answering. In Proceedings of the 57th Annual Meeting of the Association\\nfor Computational Linguistics , pages 3558–3567, Florence, Italy, July 2019. Association for'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 10}, page_content='Computational Linguistics. doi: 10.18653/v1/P19-1346. URL https://www.aclweb.org/\\nanthology/P19-1346 .\\n[13] Angela Fan, Claire Gardent, Chloe Braud, and Antoine Bordes. Augmenting transformers\\nwith KNN-based composite memory, 2020. URL https://openreview.net/forum?id=\\nH1gx1CNKPH .\\n[14] Thibault Févry, Livio Baldini Soares, Nicholas FitzGerald, Eunsol Choi, and Tom Kwiatkowski.\\nEntities as experts: Sparse memory access with entity supervision. ArXiv , abs/2004.07202,'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 10}, page_content='2020. URL https://arxiv.org/abs/2004.07202 .\\n[15] Marjan Ghazvininejad, Chris Brockett, Ming-Wei Chang, Bill Dolan, Jianfeng Gao, Wen\\ntau Yih, and Michel Galley. A knowledge-grounded neural conversation model. In AAAI\\nConference on Artiﬁcial Intelligence , 2018. URL https://www.aaai.org/ocs/index.php/\\nAAAI/AAAI18/paper/view/16710 .\\n[16] Katja Grace, John Salvatier, Allan Dafoe, Baobao Zhang, and Owain Evans. When will AI'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 10}, page_content='exceed human performance? evidence from AI experts. CoRR , abs/1705.08807, 2017. URL\\nhttp://arxiv.org/abs/1705.08807 .\\n[17] Jiatao Gu, Yong Wang, Kyunghyun Cho, and Victor O.K. Li. Search engine guided neural\\nmachine translation. In AAAI Conference on Artiﬁcial Intelligence , 2018. URL https:\\n//www.aaai.org/ocs/index.php/AAAI/AAAI18/paper/view/17282 .\\n[18] Jiatao Gu, Yong Wang, Kyunghyun Cho, and Victor O.K. Li. Search engine guided neural'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 10}, page_content='machine translation. In 32nd AAAI Conference on Artiﬁcial Intelligence, AAAI 2018 , 32nd\\nAAAI Conference on Artiﬁcial Intelligence, AAAI 2018, pages 5133–5140. AAAI press, 2018.\\n32nd AAAI Conference on Artiﬁcial Intelligence, AAAI 2018 ; Conference date: 02-02-2018\\nThrough 07-02-2018.\\n[19] Kelvin Guu, Tatsunori B. Hashimoto, Yonatan Oren, and Percy Liang. Generating sentences by\\nediting prototypes. Transactions of the Association for Computational Linguistics , 6:437–450,'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 10}, page_content='2018. doi: 10.1162/tacl_a_00030. URL https://www.aclweb.org/anthology/Q18-1031 .\\n11'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 11}, page_content='[20] Kelvin Guu, Kenton Lee, Zora Tung, Panupong Pasupat, and Ming-Wei Chang. REALM:\\nRetrieval-augmented language model pre-training. ArXiv , abs/2002.08909, 2020. URL https:\\n//arxiv.org/abs/2002.08909 .\\n[21] Tatsunori B Hashimoto, Kelvin Guu, Yonatan Oren, and Percy S Liang. A\\nretrieve-and-edit framework for predicting structured outputs. In S. Bengio,\\nH. Wallach, H. Larochelle, K. Grauman, N. Cesa-Bianchi, and R. Garnett, ed-'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 11}, page_content='itors, Advances in Neural Information Processing Systems 31 , pages 10052–\\n10062. Curran Associates, Inc., 2018. URL http://papers.nips.cc/paper/\\n8209-a-retrieve-and-edit-framework-for-predicting-structured-outputs.\\npdf.\\n[22] Nabil Hossain, Marjan Ghazvininejad, and Luke Zettlemoyer. Simple and effective retrieve-\\nedit-rerank text generation. In Proceedings of the 58th Annual Meeting of the Association for\\nComputational Linguistics , pages 2532–2538, Online, July 2020. Association for Computa-'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 11}, page_content='tional Linguistics. doi: 10.18653/v1/2020.acl-main.228. URL https://www.aclweb.org/\\nanthology/2020.acl-main.228 .\\n[23] Jeff Johnson, Matthijs Douze, and Hervé Jégou. Billion-scale similarity search with gpus. arXiv\\npreprint arXiv:1702.08734 , 2017. URL https://arxiv.org/abs/1702.08734 .\\n[24] Mandar Joshi, Eunsol Choi, Daniel Weld, and Luke Zettlemoyer. TriviaQA: A Large Scale\\nDistantly Supervised Challenge Dataset for Reading Comprehension. In Proceedings of the'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 11}, page_content='55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers) ,\\npages 1601–1611, Vancouver, Canada, July 2017. Association for Computational Linguistics.\\ndoi: 10.18653/v1/P17-1147. URL https://www.aclweb.org/anthology/P17-1147 .\\n[25] Armand Joulin and Tomas Mikolov. Inferring algorithmic patterns with stack-\\naugmented recurrent nets. In Proceedings of the 28th International Conference on'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 11}, page_content='Neural Information Processing Systems - Volume 1 , NIPS’15, page 190–198, Cam-\\nbridge, MA, USA, 2015. MIT Press. URL https://papers.nips.cc/paper/\\n5857-inferring-algorithmic-patterns-with-stack-augmented-recurrent-nets .\\n[26] Vladimir Karpukhin, Barlas Oguz, Sewon Min, Ledell Wu, Sergey Edunov, Danqi Chen, and\\nWen-tau Yih. Dense passage retrieval for open-domain question answering. arXiv preprint\\narXiv:2004.04906 , 2020. URL https://arxiv.org/abs/2004.04906 .'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 11}, page_content='[27] Urvashi Khandelwal, Omer Levy, Dan Jurafsky, Luke Zettlemoyer, and Mike Lewis. Generaliza-\\ntion through memorization: Nearest neighbor language models. In International Conference on\\nLearning Representations , 2020. URL https://openreview.net/forum?id=HklBjCEKvH .\\n[28] Diederik P. Kingma and Jimmy Ba. Adam: A method for stochastic optimization. In Yoshua\\nBengio and Yann LeCun, editors, 3rd International Conference on Learning Representations,'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 11}, page_content='ICLR 2015, San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings , 2015. URL\\nhttp://arxiv.org/abs/1412.6980 .\\n[29] Tom Kwiatkowski, Jennimaria Palomaki, Olivia Redﬁeld, Michael Collins, Ankur Parikh,\\nChris Alberti, Danielle Epstein, Illia Polosukhin, Matthew Kelcey, Jacob Devlin, Ken-\\nton Lee, Kristina N. Toutanova, Llion Jones, Ming-Wei Chang, Andrew Dai, Jakob\\nUszkoreit, Quoc Le, and Slav Petrov. Natural Questions: a Benchmark for Ques-'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 11}, page_content='tion Answering Research. Transactions of the Association of Computational Lin-\\nguistics , 2019. URL https://tomkwiat.users.x20web.corp.google.com/papers/\\nnatural-questions/main-1455-kwiatkowski.pdf .\\n[30] Guillaume Lample, Alexandre Sablayrolles, Marc’ Aurelio Ranzato, Ludovic Denoyer, and\\nHerve Jegou. Large memory layers with product keys. In H. Wallach, H. Larochelle,\\nA. Beygelzimer, F. d’ Alché-Buc, E. Fox, and R. Garnett, editors, Advances in Neural In-'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 11}, page_content='formation Processing Systems 32 , pages 8548–8559. Curran Associates, Inc., 2019. URL http:\\n//papers.nips.cc/paper/9061-large-memory-layers-with-product-keys.pdf .\\n[31] Kenton Lee, Ming-Wei Chang, and Kristina Toutanova. Latent retrieval for weakly supervised\\nopen domain question answering. In Proceedings of the 57th Annual Meeting of the Association\\n12'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 12}, page_content='for Computational Linguistics , pages 6086–6096, Florence, Italy, July 2019. Association for\\nComputational Linguistics. doi: 10.18653/v1/P19-1612. URL https://www.aclweb.org/\\nanthology/P19-1612 .\\n[32] Mike Lewis, Yinhan Liu, Naman Goyal, Marjan Ghazvininejad, Abdelrahman Mohamed,\\nOmer Levy, Veselin Stoyanov, and Luke Zettlemoyer. BART: Denoising sequence-to-sequence\\npre-training for natural language generation, translation, and comprehension. arXiv preprint'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 12}, page_content='arXiv:1910.13461 , 2019. URL https://arxiv.org/abs/1910.13461 .\\n[33] Jiwei Li, Michel Galley, Chris Brockett, Jianfeng Gao, and Bill Dolan. A diversity-promoting\\nobjective function for neural conversation models. In Proceedings of the 2016 Conference of the\\nNorth American Chapter of the Association for Computational Linguistics: Human Language\\nTechnologies , pages 110–119, San Diego, California, June 2016. Association for Computational'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 12}, page_content='Linguistics. doi: 10.18653/v1/N16-1014. URL https://www.aclweb.org/anthology/\\nN16-1014 .\\n[34] Margaret Li, Jason Weston, and Stephen Roller. Acute-eval: Improved dialogue evaluation\\nwith optimized questions and multi-turn comparisons. ArXiv , abs/1909.03087, 2019. URL\\nhttps://arxiv.org/abs/1909.03087 .\\n[35] Hairong Liu, Mingbo Ma, Liang Huang, Hao Xiong, and Zhongjun He. Robust neural machine\\ntranslation with joint textual and phonetic embedding. In Proceedings of the 57th Annual'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 12}, page_content='Meeting of the Association for Computational Linguistics , pages 3044–3049, Florence, Italy,\\nJuly 2019. Association for Computational Linguistics. doi: 10.18653/v1/P19-1291. URL\\nhttps://www.aclweb.org/anthology/P19-1291 .\\n[36] Peter J. Liu*, Mohammad Saleh*, Etienne Pot, Ben Goodrich, Ryan Sepassi, Lukasz Kaiser,\\nand Noam Shazeer. Generating wikipedia by summarizing long sequences. In International\\nConference on Learning Representations , 2018. URL https://openreview.net/forum?\\nid=Hyg0vbWC- .'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 12}, page_content='id=Hyg0vbWC- .\\n[37] Yury A. Malkov and D. A. Yashunin. Efﬁcient and robust approximate nearest neighbor search\\nusing hierarchical navigable small world graphs. IEEE Transactions on Pattern Analysis and\\nMachine Intelligence , 42:824–836, 2016. URL https://arxiv.org/abs/1603.09320 .\\n[38] Gary Marcus. The next decade in ai: four steps towards robust artiﬁcial intelligence. arXiv\\npreprint arXiv:2002.06177 , 2020. URL https://arxiv.org/abs/2002.06177 .'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 12}, page_content='[39] Luca Massarelli, Fabio Petroni, Aleksandra Piktus, Myle Ott, Tim Rocktäschel, Vassilis\\nPlachouras, Fabrizio Silvestri, and Sebastian Riedel. How decoding strategies affect the\\nveriﬁability of generated text. arXiv preprint arXiv:1911.03587 , 2019. URL https:\\n//arxiv.org/abs/1911.03587 .\\n[40] Paulius Micikevicius, Sharan Narang, Jonah Alben, Gregory Diamos, Erich Elsen, David Garcia,\\nBoris Ginsburg, Michael Houston, Oleksii Kuchaiev, Ganesh Venkatesh, and Hao Wu. Mixed'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 12}, page_content='precision training. In ICLR , 2018. URL https://openreview.net/forum?id=r1gs9JgRZ .\\n[41] Nikita Moghe, Siddhartha Arora, Suman Banerjee, and Mitesh M. Khapra. Towards exploit-\\ning background knowledge for building conversation systems. In Proceedings of the 2018\\nConference on Empirical Methods in Natural Language Processing , pages 2322–2332, Brus-\\nsels, Belgium, October-November 2018. Association for Computational Linguistics. doi:'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 12}, page_content='10.18653/v1/D18-1255. URL https://www.aclweb.org/anthology/D18-1255 .\\n[42] Preksha Nema and Mitesh M. Khapra. Towards a better metric for evaluating question generation\\nsystems. In Proceedings of the 2018 Conference on Empirical Methods in Natural Language\\nProcessing , pages 3950–3959, Brussels, Belgium, October-November 2018. Association for\\nComputational Linguistics. doi: 10.18653/v1/D18-1429. URL https://www.aclweb.org/\\nanthology/D18-1429 .'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 12}, page_content='anthology/D18-1429 .\\n[43] Tri Nguyen, Mir Rosenberg, Xia Song, Jianfeng Gao, Saurabh Tiwary, Rangan Majumder,\\nand Li Deng. MS MARCO: A human generated machine reading comprehension dataset. In\\nTarek Richard Besold, Antoine Bordes, Artur S. d’Avila Garcez, and Greg Wayne, editors,\\nProceedings of the Workshop on Cognitive Computation: Integrating neural and symbolic\\n13'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 13}, page_content='approaches 2016 co-located with the 30th Annual Conference on Neural Information Processing\\nSystems (NIPS 2016), Barcelona, Spain, December 9, 2016 , volume 1773 of CEUR Workshop\\nProceedings . CEUR-WS.org, 2016. URL http://ceur-ws.org/Vol-1773/CoCoNIPS_\\n2016_paper9.pdf .\\n[44] Rodrigo Nogueira and Kyunghyun Cho. Passage re-ranking with BERT. arXiv preprint\\narXiv:1901.04085 , 2019. URL https://arxiv.org/abs/1901.04085 .'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 13}, page_content='[45] Myle Ott, Sergey Edunov, Alexei Baevski, Angela Fan, Sam Gross, Nathan Ng, David Grangier,\\nand Michael Auli. fairseq: A fast, extensible toolkit for sequence modeling. In Proceedings\\nof the 2019 Conference of the North American Chapter of the Association for Computational\\nLinguistics (Demonstrations) , pages 48–53, Minneapolis, Minnesota, June 2019. Association\\nfor Computational Linguistics. doi: 10.18653/v1/N19-4009. URL https://www.aclweb.\\norg/anthology/N19-4009 .'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 13}, page_content='org/anthology/N19-4009 .\\n[46] Ethan Perez, Siddharth Karamcheti, Rob Fergus, Jason Weston, Douwe Kiela, and Kyunghyun\\nCho. Finding generalizable evidence by learning to convince q&a models. In Proceedings\\nof the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th\\nInternational Joint Conference on Natural Language Processing (EMNLP-IJCNLP) , pages\\n2402–2411, Hong Kong, China, November 2019. Association for Computational Linguistics.'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 13}, page_content='doi: 10.18653/v1/D19-1244. URL https://www.aclweb.org/anthology/D19-1244 .\\n[47] Fabio Petroni, Tim Rocktäschel, Sebastian Riedel, Patrick Lewis, Anton Bakhtin, Yuxiang Wu,\\nand Alexander Miller. Language models as knowledge bases? In Proceedings of the 2019\\nConference on Empirical Methods in Natural Language Processing and the 9th International\\nJoint Conference on Natural Language Processing (EMNLP-IJCNLP) , pages 2463–2473, Hong'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 13}, page_content='Kong, China, November 2019. Association for Computational Linguistics. doi: 10.18653/v1/\\nD19-1250. URL https://www.aclweb.org/anthology/D19-1250 .\\n[48] Fabio Petroni, Patrick Lewis, Aleksandra Piktus, Tim Rocktäschel, Yuxiang Wu, Alexander H.\\nMiller, and Sebastian Riedel. How context affects language models’ factual predictions. In\\nAutomated Knowledge Base Construction , 2020. URL https://openreview.net/forum?\\nid=025X0zPfn .'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 13}, page_content='id=025X0zPfn .\\n[49] Alec Radford, Karthik Narasimhan, Tim Salimans, and Ilya Sutskever. Im-\\nproving Language Understanding by Generative Pre-Training, 2018. URL\\nhttps://s3-us-west-2.amazonaws.com/openai-assets/research-covers/\\nlanguage-unsupervised/language_understanding_paper.pdf .\\n[50] Alec Radford, Jeff Wu, Rewon Child, David Luan, Dario Amodei, and Ilya\\nSutskever. Language models are unsupervised multitask learners, 2019. URL'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 13}, page_content='https://d4mucfpksywv.cloudfront.net/better-language-models/language_\\nmodels_are_unsupervised_multitask_learners.pdf .\\n[51] Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena,\\nYanqi Zhou, Wei Li, and Peter J. Liu. Exploring the limits of transfer learning with a uniﬁed\\ntext-to-text transformer. arXiv e-prints , 2019. URL https://arxiv.org/abs/1910.10683 .\\n[52] Adam Roberts, Colin Raffel, and Noam Shazeer. How much knowledge can you pack into'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 13}, page_content='the parameters of a language model? arXiv e-prints , 2020. URL https://arxiv.org/abs/\\n2002.08910 .\\n[53] Stephen Robertson and Hugo Zaragoza. The probabilistic relevance framework: Bm25 and\\nbeyond. Found. Trends Inf. Retr. , 3(4):333–389, April 2009. ISSN 1554-0669. doi: 10.1561/\\n1500000019. URL https://doi.org/10.1561/1500000019 .\\n[54] Irene Solaiman, Miles Brundage, Jack Clark, Amanda Askell, Ariel Herbert-V oss, Jeff Wu, Alec'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 13}, page_content='Radford, and Jian-Bing Wang. Release strategies and the social impacts of language models.\\nArXiv , abs/1908.09203, 2019.\\n[55] Sainbayar Sukhbaatar, Arthur Szlam, Jason Weston, and Rob Fergus. End-to-end memory net-\\nworks. In C. Cortes, N. D. Lawrence, D. D. Lee, M. Sugiyama, and R. Garnett, editors, Advances\\nin Neural Information Processing Systems 28 , pages 2440–2448. Curran Associates, Inc., 2015.\\nURL http://papers.nips.cc/paper/5846-end-to-end-memory-networks.pdf .\\n14'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 14}, page_content='[56] James Thorne, Andreas Vlachos, Christos Christodoulopoulos, and Arpit Mittal. FEVER: a\\nlarge-scale dataset for fact extraction and VERiﬁcation. In Proceedings of the 2018 Conference\\nof the North American Chapter of the Association for Computational Linguistics: Human\\nLanguage Technologies, Volume 1 (Long Papers) , pages 809–819, New Orleans, Louisiana,\\nJune 2018. Association for Computational Linguistics. doi: 10.18653/v1/N18-1074. URL\\nhttps://www.aclweb.org/anthology/N18-1074 .'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 14}, page_content='https://www.aclweb.org/anthology/N18-1074 .\\n[57] James H. Thorne and Andreas Vlachos. Avoiding catastrophic forgetting in mitigating model\\nbiases in sentence-pair classiﬁcation with elastic weight consolidation. ArXiv , abs/2004.14366,\\n2020. URL https://arxiv.org/abs/2004.14366 .\\n[58] Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez,\\nŁ ukasz Kaiser, and Illia Polosukhin. Attention is all you need. In I. Guyon, U. V . Luxburg,'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 14}, page_content='S. Bengio, H. Wallach, R. Fergus, S. Vishwanathan, and R. Garnett, editors, Advances in Neural\\nInformation Processing Systems 30 , pages 5998–6008. Curran Associates, Inc., 2017. URL\\nhttp://papers.nips.cc/paper/7181-attention-is-all-you-need.pdf .\\n[59] Ashwin Vijayakumar, Michael Cogswell, Ramprasaath Selvaraju, Qing Sun, Stefan Lee, David\\nCrandall, and Dhruv Batra. Diverse beam search for improved description of complex scenes.'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 14}, page_content='AAAI Conference on Artiﬁcial Intelligence , 2018. URL https://www.aaai.org/ocs/index.\\nphp/AAAI/AAAI18/paper/view/17329 .\\n[60] Alex Wang, Amanpreet Singh, Julian Michael, Felix Hill, Omer Levy, and Samuel Bowman.\\nGLUE: A multi-task benchmark and analysis platform for natural language understanding.\\nInProceedings of the 2018 EMNLP Workshop BlackboxNLP: Analyzing and Interpreting\\nNeural Networks for NLP , pages 353–355, Brussels, Belgium, November 2018. Association for'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 14}, page_content='Computational Linguistics. doi: 10.18653/v1/W18-5446. URL https://www.aclweb.org/\\nanthology/W18-5446 .\\n[61] Alex Wang, Yada Pruksachatkun, Nikita Nangia, Amanpreet Singh, Julian Michael, Felix\\nHill, Omer Levy, and Samuel Bowman. SuperGLUE: A Stickier Benchmark for General-\\nPurpose Language Understanding Systems. In H. Wallach, H. Larochelle, A. Beygelzimer,\\nF. d\\\\textquotesingle Alché-Buc, E. Fox, and R. Garnett, editors, Advances in Neural Information'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 14}, page_content='Processing Systems 32 , pages 3261–3275. Curran Associates, Inc., 2019. URL https://\\narxiv.org/abs/1905.00537 .\\n[62] Shuohang Wang, Mo Yu, Xiaoxiao Guo, Zhiguo Wang, Tim Klinger, Wei Zhang, Shiyu Chang,\\nGerry Tesauro, Bowen Zhou, and Jing Jiang. R3: Reinforced ranker-reader for open-domain\\nquestion answering. In Sheila A. McIlraith and Kilian Q. Weinberger, editors, Proceedings of\\nthe Thirty-Second AAAI Conference on Artiﬁcial Intelligence, (AAAI-18), the 30th innovative'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 14}, page_content='Applications of Artiﬁcial Intelligence (IAAI-18), and the 8th AAAI Symposium on Educational\\nAdvances in Artiﬁcial Intelligence (EAAI-18), New Orleans, Louisiana, USA, February 2-7,\\n2018 , pages 5981–5988. AAAI Press, 2018. URL https://www.aaai.org/ocs/index.\\nphp/AAAI/AAAI18/paper/view/16712 .\\n[63] Shuohang Wang, Mo Yu, Jing Jiang, Wei Zhang, Xiaoxiao Guo, Shiyu Chang, Zhiguo Wang,\\nTim Klinger, Gerald Tesauro, and Murray Campbell. Evidence aggregation for answer re-'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 14}, page_content='ranking in open-domain question answering. In ICLR , 2018. URL https://openreview.\\nnet/forum?id=rJl3yM-Ab .\\n[64] Jason Weston, Sumit Chopra, and Antoine Bordes. Memory networks. In Yoshua Bengio\\nand Yann LeCun, editors, 3rd International Conference on Learning Representations, ICLR\\n2015, San Diego, CA, USA, May 7-9, 2015, Conference Track Proceedings , 2015. URL\\nhttp://arxiv.org/abs/1410.3916 .\\n[65] Jason Weston, Emily Dinan, and Alexander Miller. Retrieve and reﬁne: Improved sequence'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 14}, page_content='generation models for dialogue. In Proceedings of the 2018 EMNLP Workshop SCAI: The 2nd\\nInternational Workshop on Search-Oriented Conversational AI , pages 87–92, Brussels, Belgium,\\nOctober 2018. Association for Computational Linguistics. doi: 10.18653/v1/W18-5713. URL\\nhttps://www.aclweb.org/anthology/W18-5713 .\\n15'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 15}, page_content='[66] Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Clement Delangue, Anthony\\nMoi, Pierric Cistac, Tim Rault, Rémi Louf, Morgan Funtowicz, Joe Davison, Sam Shleifer,\\nPatrick von Platen, Clara Ma, Yacine Jernite, Julien Plu, Canwen Xu, Teven Le Scao, Sylvain\\nGugger, Mariama Drame, Quentin Lhoest, and Alexander M. Rush. Huggingface’s transformers:\\nState-of-the-art natural language processing. ArXiv , abs/1910.03771, 2019.'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 15}, page_content='[67] Shiyue Zhang and Mohit Bansal. Addressing semantic drift in question generation for semi-\\nsupervised question answering. In Proceedings of the 2019 Conference on Empirical Meth-\\nods in Natural Language Processing and the 9th International Joint Conference on Natural\\nLanguage Processing (EMNLP-IJCNLP) , pages 2495–2509, Hong Kong, China, Novem-\\nber 2019. Association for Computational Linguistics. doi: 10.18653/v1/D19-1253. URL\\nhttps://www.aclweb.org/anthology/D19-1253 .'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 15}, page_content='https://www.aclweb.org/anthology/D19-1253 .\\n[68] Wanjun Zhong, Jingjing Xu, Duyu Tang, Zenan Xu, Nan Duan, Ming Zhou, Jiahai Wang, and\\nJian Yin. Reasoning over semantic-level graph for fact checking. ArXiv , abs/1909.03745, 2019.\\nURL https://arxiv.org/abs/1909.03745 .\\n16'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 16}, page_content='Appendices for Retrieval-Augmented Generation for\\nKnowledge-Intensive NLP Tasks\\nA Implementation Details\\nFor Open-domain QA we report test numbers using 15 retrieved documents for RAG-Token models.\\nFor RAG-Sequence models, we report test results using 50 retrieved documents, and we use the\\nThorough Decoding approach since answers are generally short. We use greedy decoding for QA as\\nwe did not ﬁnd beam search improved results. For Open-MSMarco and Jeopardy question generation,'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 16}, page_content='we report test numbers using ten retrieved documents for both RAG-Token and RAG-Sequence,\\nand we also train a BART-large model as a baseline. We use a beam size of four, and use the Fast\\nDecoding approach for RAG-Sequence models, as Thorough Decoding did not improve performance.\\nB Human Evaluation\\nFigure 4: Annotation interface for human evaluation of factuality. A pop-out for detailed instructions\\nand a worked example appear when clicking \"view tool guide\".'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 16}, page_content='Figure 4 shows the user interface for human evaluation. To avoid any biases for screen position,\\nwhich model corresponded to sentence A and sentence B was randomly selected for each example.\\nAnnotators were encouraged to research the topic using the internet, and were given detailed instruc-\\ntions and worked examples in a full instructions tab. We included some gold sentences in order to\\nassess the accuracy of the annotators. Two annotators did not perform well on these examples and'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 16}, page_content='their annotations were removed from the results.\\nC Training setup Details\\nWe train all RAG models and BART baselines using Fairseq [ 45].2We train with mixed precision\\nﬂoating point arithmetic [ 40], distributing training across 8, 32GB NVIDIA V100 GPUs, though\\ntraining and inference can be run on one GPU. We ﬁnd that doing Maximum Inner Product Search\\nwith FAISS is sufﬁciently fast on CPU, so we store document index vectors on CPU, requiring ∼100'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 16}, page_content='GB of CPU memory for all of Wikipedia. After submission, We have ported our code to HuggingFace\\nTransformers [ 66]3, which achieves equivalent performance to the previous version but is a cleaner\\nand easier to use implementation. This version is also open-sourced. We also compress the document\\nindex using FAISS’s compression tools, reducing the CPU memory requirement to 36GB. Scripts to\\nrun experiments with RAG can be found at https://github.com/huggingface/transformers/'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 16}, page_content='blob/master/examples/rag/README.md and an interactive demo of a RAG model can be found\\nathttps://huggingface.co/rag/\\n2https://github.com/pytorch/fairseq\\n3https://github.com/huggingface/transformers\\n17'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 17}, page_content='D Further Details on Open-Domain QA\\nFor open-domain QA, multiple answer annotations are often available for a given question. These\\nanswer annotations are exploited by extractive models during training as typically all the answer\\nannotations are used to ﬁnd matches within documents when preparing training data. For RAG, we\\nalso make use of multiple annotation examples for Natural Questions and WebQuestions by training'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 17}, page_content='the model with each (q,a)pair separately, leading to a small increase in accuracy. For TriviaQA,\\nthere are often many valid answers to a given question, some of which are not suitable training targets,\\nsuch as emoji or spelling variants. For TriviaQA, we ﬁlter out answer candidates if they do not occur\\nin top 1000 documents for the query.\\nCuratedTrec preprocessing The answers for CuratedTrec are given in the form of regular expres-'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 17}, page_content='sions, which has been suggested as a reason why it is unsuitable for answer-generation models [20].\\nTo overcome this, we use a pre-processing step where we ﬁrst retrieve the top 1000 documents for\\neach query, and use the answer that most frequently matches the regex pattern as the supervision\\ntarget. If no matches are found, we resort to a simple heuristic: generate all possible permutations for'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 17}, page_content='each regex, replacing non-deterministic symbols in the regex nested tree structure with a whitespace.\\nTriviaQA Evaluation setups The open-domain QA community customarily uses public develop-\\nment datasets as test datasets, as test data for QA datasets is often restricted and dedicated to reading\\ncompehension purposes. We report our results using the datasets splits used in DPR [ 26], which are\\nconsistent with common practice in Open-domain QA. For TriviaQA, this test dataset is the public'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 17}, page_content='TriviaQA Web Development split. Roberts et al. [52] used the TriviaQA ofﬁcial Wikipedia test set\\ninstead. Févry et al. [14] follow this convention in order to compare with Roberts et al. [52] (See\\nappendix of [ 14]). We report results on both test sets to enable fair comparison to both approaches.\\nWe ﬁnd that our performance is much higher using the ofﬁcial Wiki test set, rather than the more\\nconventional open-domain test set, which we attribute to the ofﬁcial Wiki test set questions being'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 17}, page_content='simpler to answer from Wikipedia.\\nE Further Details on FEVER\\nFor FEVER classiﬁcation, we follow the practice from [ 32], and ﬁrst re-generate the claim, and\\nthen classify using the representation of the ﬁnal hidden state, before ﬁnally marginalizing across\\ndocuments to obtain the class probabilities. The FEVER task traditionally has two sub-tasks. The\\nﬁrst is to classify the claim as either \"Supported\", \"Refuted\" or \"Not Enough Info\", which is the task'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 17}, page_content='we explore in the main paper. FEVER’s other sub-task involves extracting sentences from Wikipedia\\nas evidence supporting the classiﬁcation prediction. As FEVER uses a different Wikipedia dump to\\nus, directly tackling this task is not straightforward. We hope to address this in future work.\\nF Null Document Probabilities\\nWe experimented with adding \"Null document\" mechanism to RAG, similar to REALM [ 20] in order'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 17}, page_content='to model cases where no useful information could be retrieved for a given input. Here, if kdocuments\\nwere retrieved, we would additionally \"retrieve\" an empty document and predict a logit for the null\\ndocument, before marginalizing over k+ 1predictions. We explored modelling this null document\\nlogit by learning (i) a document embedding for the null document, (ii) a static learnt bias term, or\\n(iii) a neural network to predict the logit. We did not ﬁnd that these improved performance, so in'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 17}, page_content='the interests of simplicity, we omit them. For Open MS-MARCO, where useful retrieved documents\\ncannot always be retrieved, we observe that the model learns to always retrieve a particular set of\\ndocuments for questions that are less likely to beneﬁt from retrieval, suggesting that null document\\nmechanisms may not be necessary for RAG.\\nG Parameters\\nOur RAG models contain the trainable parameters for the BERT-base query and document encoder of'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 17}, page_content='DPR, with 110M parameters each (although we do not train the document encoder ourselves) and\\n406M trainable parameters from BART-large, 406M parameters, making a total of 626M trainable\\n18'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 18}, page_content='Table 7: Number of instances in the datasets used. *A hidden subset of this data is used for evaluation\\nTask Train Development Test\\nNatural Questions 79169 8758 3611\\nTriviaQA 78786 8838 11314\\nWebQuestions 3418 362 2033\\nCuratedTrec 635 134 635\\nJeopardy Question Generation 97392 13714 26849\\nMS-MARCO 153726 12468 101093*\\nFEVER-3-way 145450 10000 10000\\nFEVER-2-way 96966 6666 6666\\nparameters. The best performing \"closed-book\" (parametric only) open-domain QA model is T5-11B'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 18}, page_content='with 11 Billion trainable parameters. The T5 model with the closest number of parameters to our\\nmodels is T5-large (770M parameters), which achieves a score of 28.9 EM on Natural Questions [ 52],\\nsubstantially below the 44.5 that RAG-Sequence achieves, indicating that hybrid parametric/non-\\nparametric models require far fewer trainable parameters for strong open-domain QA performance.\\nThe non-parametric memory index does not consist of trainable parameters, but does consists of 21M'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 18}, page_content='728 dimensional vectors, consisting of 15.3B values. These can be easily be stored at 8-bit ﬂoating\\npoint precision to manage memory and disk footprints.\\nH Retrieval Collapse\\nIn preliminary experiments, we observed that for some tasks such as story generation [ 11], the\\nretrieval component would “collapse” and learn to retrieve the same documents regardless of the\\ninput. In these cases, once retrieval had collapsed, the generator would learn to ignore the documents,'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 18}, page_content='and the RAG model would perform equivalently to BART. The collapse could be due to a less-explicit\\nrequirement for factual knowledge in some tasks, or the longer target sequences, which could result\\nin less informative gradients for the retriever. Perez et al. [46] also found spurious retrieval results\\nwhen optimizing a retrieval component in order to improve performance on downstream tasks.\\nI Number of instances per dataset'),\n",
              " Document(metadata={'source': '/content/2005.11401v4.pdf', 'page': 18}, page_content='I Number of instances per dataset\\nThe number of training, development and test datapoints in each of our datasets is shown in Table 7.\\n19')]"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.embeddings import HuggingFaceInferenceAPIEmbeddings"
      ],
      "metadata": {
        "id": "ndV0QYbChjsV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "HF_TOKEN = \"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\""
      ],
      "metadata": {
        "id": "CTpd7QYThvGy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "embeddings = HuggingFaceInferenceAPIEmbeddings(api_key=HF_TOKEN, model_name=\"BAAI/bge-base-en-v1.5\")"
      ],
      "metadata": {
        "id": "jmRLdypInX4m"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install chromadb"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-3IWqaGsnv3f",
        "outputId": "f5fc8911-8704-4a2d-abbe-f1ab555b08e6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting chromadb\n",
            "  Downloading chromadb-0.5.4-py3-none-any.whl (581 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m581.4/581.4 kB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: build>=1.0.3 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.2.1)\n",
            "Requirement already satisfied: pydantic>=1.9 in /usr/local/lib/python3.10/dist-packages (from chromadb) (2.8.2)\n",
            "Collecting chroma-hnswlib==0.7.5 (from chromadb)\n",
            "  Downloading chroma_hnswlib-0.7.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m65.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting fastapi>=0.95.2 (from chromadb)\n",
            "  Downloading fastapi-0.111.1-py3-none-any.whl (92 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.2/92.2 kB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting uvicorn[standard]>=0.18.3 (from chromadb)\n",
            "  Downloading uvicorn-0.30.1-py3-none-any.whl (62 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.4/62.4 kB\u001b[0m \u001b[31m9.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy<2.0.0,>=1.22.5 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.25.2)\n",
            "Collecting posthog>=2.4.0 (from chromadb)\n",
            "  Downloading posthog-3.5.0-py2.py3-none-any.whl (41 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.3/41.3 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.12.2)\n",
            "Collecting onnxruntime>=1.14.1 (from chromadb)\n",
            "  Downloading onnxruntime-1.18.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (6.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m41.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opentelemetry-api>=1.2.0 (from chromadb)\n",
            "  Downloading opentelemetry_api-1.25.0-py3-none-any.whl (59 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.9/59.9 kB\u001b[0m \u001b[31m9.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opentelemetry-exporter-otlp-proto-grpc>=1.2.0 (from chromadb)\n",
            "  Downloading opentelemetry_exporter_otlp_proto_grpc-1.25.0-py3-none-any.whl (18 kB)\n",
            "Collecting opentelemetry-instrumentation-fastapi>=0.41b0 (from chromadb)\n",
            "  Downloading opentelemetry_instrumentation_fastapi-0.46b0-py3-none-any.whl (11 kB)\n",
            "Collecting opentelemetry-sdk>=1.2.0 (from chromadb)\n",
            "  Downloading opentelemetry_sdk-1.25.0-py3-none-any.whl (107 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m107.0/107.0 kB\u001b[0m \u001b[31m14.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tokenizers>=0.13.2 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.19.1)\n",
            "Collecting pypika>=0.48.9 (from chromadb)\n",
            "  Downloading PyPika-0.48.9.tar.gz (67 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.3/67.3 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tqdm>=4.65.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.66.4)\n",
            "Collecting overrides>=7.3.1 (from chromadb)\n",
            "  Downloading overrides-7.7.0-py3-none-any.whl (17 kB)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.10/dist-packages (from chromadb) (6.4.0)\n",
            "Requirement already satisfied: grpcio>=1.58.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.64.1)\n",
            "Collecting bcrypt>=4.0.1 (from chromadb)\n",
            "  Downloading bcrypt-4.1.3-cp39-abi3-manylinux_2_28_x86_64.whl (283 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m283.7/283.7 kB\u001b[0m \u001b[31m33.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typer>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.12.3)\n",
            "Collecting kubernetes>=28.1.0 (from chromadb)\n",
            "  Downloading kubernetes-30.1.0-py2.py3-none-any.whl (1.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m70.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tenacity>=8.2.3 in /usr/local/lib/python3.10/dist-packages (from chromadb) (8.5.0)\n",
            "Requirement already satisfied: PyYAML>=6.0.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (6.0.1)\n",
            "Collecting mmh3>=4.0.1 (from chromadb)\n",
            "  Downloading mmh3-4.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (67 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.6/67.6 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: orjson>=3.9.12 in /usr/local/lib/python3.10/dist-packages (from chromadb) (3.10.6)\n",
            "Collecting httpx>=0.27.0 (from chromadb)\n",
            "  Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging>=19.1 in /usr/local/lib/python3.10/dist-packages (from build>=1.0.3->chromadb) (24.1)\n",
            "Requirement already satisfied: pyproject_hooks in /usr/local/lib/python3.10/dist-packages (from build>=1.0.3->chromadb) (1.1.0)\n",
            "Requirement already satisfied: tomli>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from build>=1.0.3->chromadb) (2.0.1)\n",
            "Collecting starlette<0.38.0,>=0.37.2 (from fastapi>=0.95.2->chromadb)\n",
            "  Downloading starlette-0.37.2-py3-none-any.whl (71 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting fastapi-cli>=0.0.2 (from fastapi>=0.95.2->chromadb)\n",
            "  Downloading fastapi_cli-0.0.4-py3-none-any.whl (9.5 kB)\n",
            "Requirement already satisfied: jinja2>=2.11.2 in /usr/local/lib/python3.10/dist-packages (from fastapi>=0.95.2->chromadb) (3.1.4)\n",
            "Collecting python-multipart>=0.0.7 (from fastapi>=0.95.2->chromadb)\n",
            "  Downloading python_multipart-0.0.9-py3-none-any.whl (22 kB)\n",
            "Collecting email_validator>=2.0.0 (from fastapi>=0.95.2->chromadb)\n",
            "  Downloading email_validator-2.2.0-py3-none-any.whl (33 kB)\n",
            "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.27.0->chromadb) (3.7.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx>=0.27.0->chromadb) (2024.7.4)\n",
            "Collecting httpcore==1.* (from httpx>=0.27.0->chromadb)\n",
            "  Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m12.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: idna in /usr/local/lib/python3.10/dist-packages (from httpx>=0.27.0->chromadb) (3.7)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.27.0->chromadb) (1.3.1)\n",
            "Collecting h11<0.15,>=0.13 (from httpcore==1.*->httpx>=0.27.0->chromadb)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.16.0)\n",
            "Requirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2.8.2)\n",
            "Requirement already satisfied: google-auth>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2.27.0)\n",
            "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.8.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2.31.0)\n",
            "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.3.1)\n",
            "Requirement already satisfied: oauthlib>=3.2.2 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (3.2.2)\n",
            "Requirement already satisfied: urllib3>=1.24.2 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2.0.7)\n",
            "Collecting coloredlogs (from onnxruntime>=1.14.1->chromadb)\n",
            "  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: flatbuffers in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (24.3.25)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (3.20.3)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (1.13.0)\n",
            "Collecting deprecated>=1.2.6 (from opentelemetry-api>=1.2.0->chromadb)\n",
            "  Downloading Deprecated-1.2.14-py2.py3-none-any.whl (9.6 kB)\n",
            "Collecting importlib-metadata<=7.1,>=6.0 (from opentelemetry-api>=1.2.0->chromadb)\n",
            "  Downloading importlib_metadata-7.1.0-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: googleapis-common-protos~=1.52 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.63.2)\n",
            "Collecting opentelemetry-exporter-otlp-proto-common==1.25.0 (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb)\n",
            "  Downloading opentelemetry_exporter_otlp_proto_common-1.25.0-py3-none-any.whl (17 kB)\n",
            "Collecting opentelemetry-proto==1.25.0 (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb)\n",
            "  Downloading opentelemetry_proto-1.25.0-py3-none-any.whl (52 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.5/52.5 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opentelemetry-instrumentation-asgi==0.46b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n",
            "  Downloading opentelemetry_instrumentation_asgi-0.46b0-py3-none-any.whl (14 kB)\n",
            "Collecting opentelemetry-instrumentation==0.46b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n",
            "  Downloading opentelemetry_instrumentation-0.46b0-py3-none-any.whl (29 kB)\n",
            "Collecting opentelemetry-semantic-conventions==0.46b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n",
            "  Downloading opentelemetry_semantic_conventions-0.46b0-py3-none-any.whl (130 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.5/130.5 kB\u001b[0m \u001b[31m20.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting opentelemetry-util-http==0.46b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n",
            "  Downloading opentelemetry_util_http-0.46b0-py3-none-any.whl (6.9 kB)\n",
            "Requirement already satisfied: setuptools>=16.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-instrumentation==0.46b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (67.7.2)\n",
            "Requirement already satisfied: wrapt<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-instrumentation==0.46b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (1.14.1)\n",
            "Collecting asgiref~=3.0 (from opentelemetry-instrumentation-asgi==0.46b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n",
            "  Downloading asgiref-3.8.1-py3-none-any.whl (23 kB)\n",
            "Collecting monotonic>=1.5 (from posthog>=2.4.0->chromadb)\n",
            "  Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n",
            "Collecting backoff>=1.10.0 (from posthog>=2.4.0->chromadb)\n",
            "  Downloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.9->chromadb) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.20.1 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.9->chromadb) (2.20.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from tokenizers>=0.13.2->chromadb) (0.23.5)\n",
            "Requirement already satisfied: click>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from typer>=0.9.0->chromadb) (8.1.7)\n",
            "Requirement already satisfied: shellingham>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from typer>=0.9.0->chromadb) (1.5.4)\n",
            "Requirement already satisfied: rich>=10.11.0 in /usr/local/lib/python3.10/dist-packages (from typer>=0.9.0->chromadb) (13.7.1)\n",
            "Collecting httptools>=0.5.0 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (341 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m43.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting python-dotenv>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n",
            "Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading uvloop-0.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m91.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting watchfiles>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading watchfiles-0.22.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m81.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting websockets>=10.4 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (130 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.2/130.2 kB\u001b[0m \u001b[31m22.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting dnspython>=2.0.0 (from email_validator>=2.0.0->fastapi>=0.95.2->chromadb)\n",
            "  Downloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m45.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (5.4.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.4.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (4.9)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers>=0.13.2->chromadb) (3.15.4)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers>=0.13.2->chromadb) (2023.6.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<=7.1,>=6.0->opentelemetry-api>=1.2.0->chromadb) (3.19.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2>=2.11.2->fastapi>=0.95.2->chromadb) (2.1.5)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->kubernetes>=28.1.0->chromadb) (3.3.2)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.11.0->typer>=0.9.0->chromadb) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.11.0->typer>=0.9.0->chromadb) (2.16.1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx>=0.27.0->chromadb) (1.2.2)\n",
            "Collecting humanfriendly>=9.1 (from coloredlogs->onnxruntime>=1.14.1->chromadb)\n",
            "  Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m15.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer>=0.9.0->chromadb) (0.1.2)\n",
            "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.6.0)\n",
            "Building wheels for collected packages: pypika\n",
            "  Building wheel for pypika (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pypika: filename=PyPika-0.48.9-py2.py3-none-any.whl size=53725 sha256=f2211fac43cb1b4d36fa57b4b663b8b28af82c186337c780031392d381ec49f4\n",
            "  Stored in directory: /root/.cache/pip/wheels/e1/26/51/d0bffb3d2fd82256676d7ad3003faea3bd6dddc9577af665f4\n",
            "Successfully built pypika\n",
            "Installing collected packages: pypika, monotonic, mmh3, websockets, uvloop, python-multipart, python-dotenv, overrides, opentelemetry-util-http, opentelemetry-proto, importlib-metadata, humanfriendly, httptools, h11, dnspython, deprecated, chroma-hnswlib, bcrypt, backoff, asgiref, watchfiles, uvicorn, starlette, posthog, opentelemetry-exporter-otlp-proto-common, opentelemetry-api, httpcore, email_validator, coloredlogs, opentelemetry-semantic-conventions, opentelemetry-instrumentation, onnxruntime, kubernetes, httpx, opentelemetry-sdk, opentelemetry-instrumentation-asgi, fastapi-cli, opentelemetry-instrumentation-fastapi, opentelemetry-exporter-otlp-proto-grpc, fastapi, chromadb\n",
            "  Attempting uninstall: importlib-metadata\n",
            "    Found existing installation: importlib_metadata 8.0.0\n",
            "    Uninstalling importlib_metadata-8.0.0:\n",
            "      Successfully uninstalled importlib_metadata-8.0.0\n",
            "Successfully installed asgiref-3.8.1 backoff-2.2.1 bcrypt-4.1.3 chroma-hnswlib-0.7.5 chromadb-0.5.4 coloredlogs-15.0.1 deprecated-1.2.14 dnspython-2.6.1 email_validator-2.2.0 fastapi-0.111.1 fastapi-cli-0.0.4 h11-0.14.0 httpcore-1.0.5 httptools-0.6.1 httpx-0.27.0 humanfriendly-10.0 importlib-metadata-7.1.0 kubernetes-30.1.0 mmh3-4.1.0 monotonic-1.6 onnxruntime-1.18.1 opentelemetry-api-1.25.0 opentelemetry-exporter-otlp-proto-common-1.25.0 opentelemetry-exporter-otlp-proto-grpc-1.25.0 opentelemetry-instrumentation-0.46b0 opentelemetry-instrumentation-asgi-0.46b0 opentelemetry-instrumentation-fastapi-0.46b0 opentelemetry-proto-1.25.0 opentelemetry-sdk-1.25.0 opentelemetry-semantic-conventions-0.46b0 opentelemetry-util-http-0.46b0 overrides-7.7.0 posthog-3.5.0 pypika-0.48.9 python-dotenv-1.0.1 python-multipart-0.0.9 starlette-0.37.2 uvicorn-0.30.1 uvloop-0.19.0 watchfiles-0.22.0 websockets-12.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.vectorstores import Chroma"
      ],
      "metadata": {
        "id": "GYULP-r1n0BO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vectorstore = Chroma.from_documents(chunks, embeddings)"
      ],
      "metadata": {
        "id": "sqKJN9NRn6qD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vectorstore_retriever = vectorstore.as_retriever(search_kwargs = {\"k\": 3})"
      ],
      "metadata": {
        "id": "eS5wk3ZLoBu3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vectorstore_retriever"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7HhZPNBAoWEG",
        "outputId": "4f4d9a3b-0ff2-4718-b5a9-ae93e63f6c63"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "VectorStoreRetriever(tags=['Chroma', 'HuggingFaceInferenceAPIEmbeddings'], vectorstore=<langchain_community.vectorstores.chroma.Chroma object at 0x7918bc80f490>, search_kwargs={'k': 3})"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#for keyword search ,we require bm25,which is updated version of tfidf\n",
        "!pip install rank_bm25"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wDiVqpBzoZjx",
        "outputId": "e94459bd-35a6-478a-80e4-5df447051d84"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting rank_bm25\n",
            "  Downloading rank_bm25-0.2.2-py3-none-any.whl (8.6 kB)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from rank_bm25) (1.25.2)\n",
            "Installing collected packages: rank_bm25\n",
            "Successfully installed rank_bm25-0.2.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.retrievers import BM25Retriever, EnsembleRetriever"
      ],
      "metadata": {
        "id": "sKJ1BhKDopiE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#CREATING SPARSE EMBEDDINGS\n",
        "keyword_retriever = BM25Retriever.from_documents(chunks)"
      ],
      "metadata": {
        "id": "fjzhIOpXozLb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "keyword_retriever.k = 3"
      ],
      "metadata": {
        "id": "sKmtSrU1o4pN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "H3l3n-WTpO_J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "In the context of information retrieval and machine learning, the terms \"keyword retriever using sparse matrix\" and \"vector store retriever using dense matrix\" refer to different methods for indexing and retrieving documents based on their content. Here's a detailed comparison:\n",
        "\n",
        "### Keyword Retriever Using Sparse Matrix\n",
        "#### Description:\n",
        "- **Sparse Matrix:** A sparse matrix is a matrix in which most of the elements are zero. In information retrieval, this typically represents the term-document matrix, where each entry indicates the presence or frequency of a term (word) in a document.\n",
        "- **Keyword-Based Retrieval:** This method involves using keywords (terms) to search and retrieve documents. It relies on the occurrence and frequency of keywords in documents.\n",
        "\n",
        "#### Implementation:\n",
        "- **Term Frequency-Inverse Document Frequency (TF-IDF):** A common technique used to create sparse matrices. Each document is represented as a vector of TF-IDF scores.\n",
        "- **Bag-of-Words (BoW):** Another technique where each document is represented as a vector of word counts.\n",
        "\n",
        "#### Use Cases:\n",
        "- **Document Retrieval:** Efficient for finding documents containing specific keywords.\n",
        "- **Search Engines:** Often used in traditional search engines for keyword-based queries.\n",
        "- **Text Classification:** Useful in scenarios where the presence or frequency of specific terms is important.\n",
        "\n",
        "#### Example:\n",
        "Using TF-IDF with Scikit-learn:\n",
        "```python\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "\n",
        "documents = [\"This is a sample document.\", \"This document is another example.\"]\n",
        "vectorizer = TfidfVectorizer()\n",
        "sparse_matrix = vectorizer.fit_transform(documents)\n",
        "\n",
        "# Sparse matrix representation\n",
        "print(sparse_matrix)\n",
        "```\n",
        "\n",
        "### Vector Store Retriever Using Dense Matrix\n",
        "#### Description:\n",
        "- **Dense Matrix:** A dense matrix is a matrix in which most of the elements are non-zero. In information retrieval, this typically represents document embeddings or vector representations of documents.\n",
        "- **Vector-Based Retrieval:** This method involves using vector representations (embeddings) of documents to search and retrieve documents. It relies on the similarity between vectors in a high-dimensional space.\n",
        "\n",
        "#### Implementation:\n",
        "- **Embeddings:** Dense vectors representing documents, often obtained using models like BERT, Sentence Transformers, or other neural network-based methods.\n",
        "- **Similarity Search:** Techniques like cosine similarity or Euclidean distance are used to find the closest vectors (most similar documents).\n",
        "\n",
        "#### Use Cases:\n",
        "- **Semantic Search:** Efficient for retrieving documents based on semantic similarity rather than exact keyword matches.\n",
        "- **Question Answering Systems:** Useful in scenarios where the meaning of the query needs to be matched with the meaning of documents.\n",
        "- **Recommendation Systems:** Can be used to find similar items based on their vector representations.\n",
        "\n",
        "#### Example:\n",
        "Using Sentence Transformers and FAISS for dense vector retrieval:\n",
        "```python\n",
        "from sentence_transformers import SentenceTransformer\n",
        "import faiss\n",
        "import numpy as np\n",
        "\n",
        "# Load a pre-trained model\n",
        "model = SentenceTransformer('all-MiniLM-L6-v2')\n",
        "\n",
        "# Example documents\n",
        "documents = [\"This is a sample document.\", \"This document is another example.\"]\n",
        "\n",
        "# Compute dense vector embeddings for the documents\n",
        "embeddings = model.encode(documents)\n",
        "\n",
        "# Create a FAISS index\n",
        "index = faiss.IndexFlatL2(embeddings.shape[1])\n",
        "index.add(embeddings)\n",
        "\n",
        "# Query for similar documents\n",
        "query = \"Example of a document.\"\n",
        "query_embedding = model.encode([query])\n",
        "D, I = index.search(query_embedding, k=2)\n",
        "\n",
        "print(f\"Indices of similar documents: {I}\")\n",
        "print(f\"Distances: {D}\")\n",
        "```\n",
        "\n",
        "### Comparison and Specific Uses\n",
        "1. **Keyword Retriever (Sparse Matrix):**\n",
        "   - **Advantages:**\n",
        "     - Fast and efficient for keyword-based queries.\n",
        "     - Simple to implement and interpret.\n",
        "     - Low storage requirements due to sparsity.\n",
        "   - **Disadvantages:**\n",
        "     - Limited to keyword matches, lacks semantic understanding.\n",
        "     - Cannot capture contextual meaning.\n",
        "\n",
        "2. **Vector Store Retriever (Dense Matrix):**\n",
        "   - **Advantages:**\n",
        "     - Captures semantic meaning and context.\n",
        "     - Effective for complex queries where meaning is important.\n",
        "     - Can handle synonyms and related concepts.\n",
        "   - **Disadvantages:**\n",
        "     - Higher computational and storage requirements.\n",
        "     - Requires training or using pre-trained models for embeddings.\n",
        "\n",
        "### Summary\n",
        "- **Keyword Retriever Using Sparse Matrix:** Ideal for traditional keyword-based search tasks where the focus is on the presence or frequency of specific words.\n",
        "- **Vector Store Retriever Using Dense Matrix:** Ideal for tasks requiring semantic understanding and context, such as semantic search, question answering, and recommendation systems.\n",
        "\n",
        "Choosing between these methods depends on the specific requirements of your application and the nature of the queries you expect to handle."
      ],
      "metadata": {
        "id": "fReFXp70qRmj"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Ensemble search is a technique that combines multiple retrieval methods to improve the overall performance and accuracy of information retrieval systems. In the context of combining keyword-based retrievers (using sparse matrices) and vector-based retrievers (using dense matrices), ensemble search aims to leverage the strengths of both methods to provide more robust and relevant results.\n",
        "\n",
        "How Ensemble Search Works\n",
        "Indexing Phase:\n",
        "\n",
        "Sparse Matrix Indexing: Create a sparse matrix index using techniques like TF-IDF or Bag-of-Words for keyword-based retrieval.\n",
        "Dense Matrix Indexing: Create a dense matrix index using embeddings generated by models like BERT, Sentence Transformers, etc., for semantic retrieval.\n",
        "Retrieval Phase:\n",
        "\n",
        "Keyword Retrieval: Given a query, retrieve documents based on keyword matches from the sparse matrix index.\n",
        "Vector Retrieval: Simultaneously, retrieve documents based on semantic similarity from the dense matrix index.\n",
        "Combining Results:\n",
        "\n",
        "Ranking: Combine the results from both retrieval methods. This can be done using various strategies:\n",
        "Weighted Sum: Assign weights to the scores from each retriever and compute a combined score.\n",
        "Rank Aggregation: Use methods like Borda count or Reciprocal Rank Fusion to merge ranked lists.\n",
        "Hybrid Approach: Select top results from both methods and re-rank them based on some criteria.\n",
        "Advantages of Ensemble Search\n",
        "Improved Recall and Precision:\n",
        "\n",
        "Recall: Keyword-based retrieval ensures that documents containing exact query terms are not missed.\n",
        "Precision: Vector-based retrieval ensures that semantically similar documents are included even if they don't contain the exact query terms.\n",
        "Balancing Strengths and Weaknesses:\n",
        "\n",
        "Keyword-Based Strengths: Effective for specific terms and phrases, ensuring that exact matches are found.\n",
        "Vector-Based Strengths: Captures context and semantics, handling synonyms and related concepts better.\n",
        "Enhanced Flexibility:\n",
        "\n",
        "Can handle a wider variety of queries, from exact term matches to more abstract, concept-based searches.\n",
        "Reduced Limitation of Individual Methods:\n",
        "\n",
        "Overcomes the limitation of keyword-based methods that miss semantically relevant documents.\n",
        "Compensates for the computational complexity and storage requirements of dense vector methods by integrating efficient keyword retrieval."
      ],
      "metadata": {
        "id": "NyOw1WLQ2jT-"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Explanation of the Example\n",
        "Keyword-Based Retrieval: Uses TF-IDF to score and rank documents based on keyword matches.\n",
        "Vector-Based Retrieval: Uses Sentence Transformers to create embeddings and FAISS for similarity search.\n",
        "Combining Results: Combines ranks from both methods using a simple rank fusion strategy.\n",
        "Final Ranking: Produces a ranked list of documents based on the combined scores.\n",
        "Advantages Over Individual Methods\n",
        "Enhanced Relevance: By combining keyword precision with semantic understanding, ensemble search often yields more relevant results.\n",
        "Robustness: Better handles diverse queries, including both exact term matches and concept-based searches.\n",
        "Complementary Strengths: Utilizes the strengths of both sparse and dense retrieval methods to mitigate their individual weaknesses.\n",
        "In summary, ensemble search provides a more comprehensive and effective approach to information retrieval by integrating the strengths of both keyword-based and vector-based methods.\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "gq5wVxgH2ouz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "retriever = EnsembleRetriever(retrievers=[keyword_retriever, vectorstore_retriever],weights = [0.7,0.3])"
      ],
      "metadata": {
        "id": "Lt86N1X-2pRj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Mixing vector search and keyword search for Hybrid search**\n",
        "hybrid_score = (1 — alpha) * sparse_score + alpha * dense_score *italicized text*"
      ],
      "metadata": {
        "id": "DVxaNL5J3ru0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model_name =  \"HuggingFaceH4/zephyr-7b-beta\""
      ],
      "metadata": {
        "id": "afKDm0r33t0P"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "!pip install bitsandbytes"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M-g1UNMQ4j0I",
        "outputId": "90de23e0-8a54-4589-c8ae-b4203c98a1c1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting bitsandbytes\n",
            "  Downloading bitsandbytes-0.43.1-py3-none-manylinux_2_24_x86_64.whl (119.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.8/119.8 MB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from bitsandbytes) (2.3.1+cu121)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from bitsandbytes) (1.25.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (3.15.4)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (4.12.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (1.13.0)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (2023.6.0)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch->bitsandbytes)\n",
            "  Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch->bitsandbytes)\n",
            "  Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch->bitsandbytes)\n",
            "  Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch->bitsandbytes)\n",
            "  Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch->bitsandbytes)\n",
            "  Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch->bitsandbytes)\n",
            "  Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "Collecting nvidia-curand-cu12==10.3.2.106 (from torch->bitsandbytes)\n",
            "  Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch->bitsandbytes)\n",
            "  Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch->bitsandbytes)\n",
            "  Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "Collecting nvidia-nccl-cu12==2.20.5 (from torch->bitsandbytes)\n",
            "  Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n",
            "Collecting nvidia-nvtx-cu12==12.1.105 (from torch->bitsandbytes)\n",
            "  Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "Requirement already satisfied: triton==2.3.1 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (2.3.1)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch->bitsandbytes)\n",
            "  Downloading nvidia_nvjitlink_cu12-12.5.82-py3-none-manylinux2014_x86_64.whl (21.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.3/21.3 MB\u001b[0m \u001b[31m72.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->bitsandbytes) (2.1.5)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->bitsandbytes) (1.3.0)\n",
            "Installing collected packages: nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, bitsandbytes\n",
            "Successfully installed bitsandbytes-0.43.1 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.5.82 nvidia-nvtx-cu12-12.1.105\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install bitsandbytes"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v_VFNBO14nfI",
        "outputId": "09413db8-d8a5-42c2-b78d-619b75f70832"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: bitsandbytes in /usr/local/lib/python3.10/dist-packages (0.43.1)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from bitsandbytes) (2.3.1+cu121)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from bitsandbytes) (1.25.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (3.15.4)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (4.12.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (1.13.0)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (2023.6.0)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (2.20.5)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (12.1.105)\n",
            "Requirement already satisfied: triton==2.3.1 in /usr/local/lib/python3.10/dist-packages (from torch->bitsandbytes) (2.3.1)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch->bitsandbytes) (12.5.82)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->bitsandbytes) (2.1.5)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->bitsandbytes) (1.3.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! pip install accelerate"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "O2UbKJ0k6ulC",
        "outputId": "1e895e72-b880-4c60-8a86-da6ea1598143"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (0.32.1)\n",
            "Requirement already satisfied: numpy<2.0.0,>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate) (1.25.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (24.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate) (6.0.1)\n",
            "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.3.1+cu121)\n",
            "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.23.5)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.4.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.15.4)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (4.12.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.13.0)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2023.6.0)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
            "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (8.9.2.26)\n",
            "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.3.1)\n",
            "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (11.0.2.54)\n",
            "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (10.3.2.106)\n",
            "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (11.4.5.107)\n",
            "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.0.106)\n",
            "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.20.5)\n",
            "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
            "Requirement already satisfied: triton==2.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.3.1)\n",
            "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.10.0->accelerate) (12.5.82)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (4.66.4)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.5)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2024.7.4)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from transformers import ( AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, pipeline, )\n",
        "from langchain import HuggingFacePipeline"
      ],
      "metadata": {
        "id": "6LF5EUuw60kK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "# function for loading 4-bit quantized model\n",
        "def load_quantized_model(model_name: str):\n",
        "    \"\"\"\n",
        "    model_name: Name or path of the model to be loaded.\n",
        "    return: Loaded quantized model.\n",
        "    \"\"\"\n",
        "    bnb_config = BitsAndBytesConfig(\n",
        "        load_in_4bit=True,\n",
        "        bnb_4bit_use_double_quant=True,\n",
        "        bnb_4bit_quant_type=\"nf4\",\n",
        "        bnb_4bit_compute_dtype=torch.bfloat16,\n",
        "    )\n",
        "\n",
        "    model = AutoModelForCausalLM.from_pretrained(\n",
        "        model_name,\n",
        "        torch_dtype=torch.bfloat16,\n",
        "        quantization_config=bnb_config,\n",
        "    )\n",
        "    return model"
      ],
      "metadata": {
        "id": "AaxDWWGt7EFf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def initialize_tokenizer(model_name: str):\n",
        "    \"\"\"\n",
        "    model_name: Name or path of the model for tokenizer initialization.\n",
        "    return: Initialized tokenizer.\n",
        "    \"\"\"\n",
        "    tokenizer = AutoTokenizer.from_pretrained(model_name, return_token_type_ids=False)\n",
        "    tokenizer.bos_token_id = 1  # Set beginning of sentence token id\n",
        "    return tokenizer"
      ],
      "metadata": {
        "id": "WOv_85B_7Z_8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = initialize_tokenizer(model_name)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301,
          "referenced_widgets": [
            "e73f4ed251c14e6b8c16f5e92d616948",
            "254efae6c7c74025b70c279ea9398fdf",
            "b6f8c16f2fd24119a86a20583c37cef3",
            "2822f773b5f0414e9c969babf467ce37",
            "159fdcf30137493f9d8bb5fb3d8a5af2",
            "a5be9a05f3fd4e0682cb3f34be347701",
            "355a0e641f9e491ea53c70c993cabcb9",
            "bfce41d6e8bd4b66a1a590cf55ca25ec",
            "70397aaa1e764789ac5b368f1e49eb3b",
            "d8219d4062144c81a2f18cb52b41f5ef",
            "cefa5bece9d640b0bc98884da3a41c1d",
            "b064a07736a348b38cdcef86740560a4",
            "70a7901c2f8d4e418539b5e00615f03a",
            "fb5296974cb540e1926b4fbbb312f7ef",
            "6270c78188254087b936f779448d7dd0",
            "ce9bbc0ac3014772b2875e2ba724be8d",
            "1972599c873d45098fd86aad8c385162",
            "7209f15723a44207b077e34da2c8542e",
            "f7b7f39e720f4e788c8382f02e469b39",
            "bbb9b2f556264df3bd74a66b1c69704b",
            "7ebca4b0a59a4d33bafa5d72e92f6367",
            "12400ead406b4514941e7f2d0c2f607e",
            "b2a9088ebf2046dea92bd80d0981546b",
            "1c49d912bc794831aa25b93dc1125d10",
            "e2ed21df06d84c1281e5253d41a1c231",
            "e18fc0c162eb4f13bb1c30bd2702c39f",
            "92c297d75a8542c7abf7f23e7fe1e81b",
            "a9b59c7f33e049389250b7e2aa200200",
            "82c22f3a23b341bda2a37e4b6c5496ad",
            "9f184211792242eaa3e947c834675808",
            "da2348c6dd274dd383fe199a70157a7c",
            "2a774a14496348168fca4003953c206a",
            "e6ac3c59973943a497527f0f2d281110",
            "53a8aa8b486e41d4a16004fc9b62fb09",
            "8bf97e0fea874727a4148377ecdd3e7e",
            "7e3169ddeae4467cad98e37fc5e94128",
            "a50bc94c049047648f5cb77c38bb5c1e",
            "6f54c683b56d486b90f2bfd09228f152",
            "0fbbc5ef7a274e7aa79b606cc50a8d2c",
            "5162b6beebb046caa3b2fda85cb96084",
            "63dd8bdacdff4fba8303b1cbe5d51dcb",
            "ba6dc212926b46b386a5c979fb1f8841",
            "b8e9e584abea4a65993360ecaae03ba3",
            "99ad19088f6344d2a3d86b80058ee6c0",
            "719c5e17a58147949d9bb85fa807d48f",
            "26fd86ca1a504ff89a77631b9e038e3b",
            "cb8a8a93935747bd86e9184bf2595d82",
            "dad7b96f4d2141a7abef2dc667f34ab0",
            "8d4f6654662c4ef88986fa2919a38f50",
            "4a5f2156c829463c917b3e1a4ccd83e0",
            "38975564fbec4aa68d8aa8a05d6edd9a",
            "522e491d4145469a82e30be85713a3b6",
            "cf63d078972347a7a86f3ff6e6f14608",
            "023dd0baf4954756908dada312eb7e33",
            "61fa28537e614acbaa21480dbd951857"
          ]
        },
        "id": "nN27wbpR9IwW",
        "outputId": "0f7ed0cd-28cb-47f5-b02c-0c97ee1aafdb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/1.43k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e73f4ed251c14e6b8c16f5e92d616948"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.model:   0%|          | 0.00/493k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b064a07736a348b38cdcef86740560a4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/1.80M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b2a9088ebf2046dea92bd80d0981546b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "added_tokens.json:   0%|          | 0.00/42.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "53a8aa8b486e41d4a16004fc9b62fb09"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/168 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "719c5e17a58147949d9bb85fa807d48f"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "model = load_quantized_model(model_name)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 674,
          "referenced_widgets": [
            "4f93b9b5e64b4640aa377569692c3dde",
            "bf130339f83e4324a6d883f35017c7f0",
            "c5f4df68d2ac4b73abcb386bad7e2ae6",
            "f6a1068246b4469c85d92a28a1f65b0f",
            "0746a73307074e79bd345287aaff9040",
            "79af18a3b69a46fda639a88ef062b84e",
            "e3b52a7bbad840639094522e597973d6",
            "ca7c73c2d3fd4aa5a0e52646cbb22141",
            "cc18979059884d08a55f4ba0efbc7d3e",
            "27510cc3152e44ffbde8794f91b0f033",
            "5bce31b3fc0a4f9890f714e1a21d4eac",
            "f1e205e3e27e44b69c896bfdc263a5da",
            "8efcb470797f4b819f9af93e472bb986",
            "899a6b5e22574c368a081d2f1d736351",
            "20ce32d672d64a5b8bf841f4057e23fd",
            "5328d8305f5642499ed1e3f7b1e29ddb",
            "3909625fb84740dd9c88392a918b1fdf",
            "51dd2ec385e34f6794f60addd31db5da",
            "3a3a407b5987496eb30b52f8479b2b16",
            "e73233099148400493020c99f4fe1ca2",
            "4dd3c29f3c394d5c8f634046838e48dc",
            "154048d2fc284fbbb22ec7f2de2e0a3c",
            "171e66500b744fa59a19edc5e4b45be9",
            "c70a14c9673e4fd49ca43fb420a2f070",
            "fbb13383c0ae4085b25f47b6af522021",
            "759c993f132745a99ebe160da7421ef1",
            "01acf38611684c0b8eec7e68d3617a5a",
            "78397807421142ab86107251baf39320",
            "10dfdb3a60714a5c96c9c8889b67c07a",
            "5379140a67ce4188a12268f9efaa7d3d",
            "3e3cf27c43d342e1957ad5613088df53",
            "d619276a2ba84c02ae7477ec8c8e193a",
            "3ad62b0192104b47a62e2e28cf6c045b",
            "3d35f54a6c1d48589ce4fb7e786221ee",
            "16012aeed93e4b1c9627b37bd32637a0",
            "015e10071628426b8df1c1db32168a30",
            "ce3b4db0498f4078b051a54c153c9f98",
            "16963e7d78f742b5950edbfbd52a34a2",
            "ac054c813ec74ad7aa1dc9754b7a09d0",
            "1ad5420bad52435ea782776b80696e38",
            "35660b46aee94899bd172389c835e4ef",
            "a7e4393a792c4d9f90c7f9a673b71ff9",
            "bb21b23915f844c7b78dc1803f5c7d5c",
            "fff97c9c191f431ca3bc2fb4b54017bc",
            "139c836d42994b6ead18b551787543a0",
            "cacc879239e3467c98a07cda058d4a9e",
            "07838821f7294b2796f8b5a8575acee9",
            "083b2a35ca4245c0ae7894cb3551b1d7",
            "ddb602a7c9e04ddc8f4b8a03720728e9",
            "e0c11743d04847ed9cc28c52e9d00884",
            "d684c641c3db4acf84fb265e15b98a4f",
            "d2120375f26e4429b38de09fcaf6ce77",
            "9c8b98bcbca1407195595bf239a5e936",
            "e23416758e0245fe9805e917e734c8fa",
            "2933efb681ce4bad934642e4be7ee1d4",
            "2405b3b96cba4dcb935c9494f6bb0e14",
            "33d3d454b7194b149bbac9731617eb74",
            "d1c44b4e4d5b4b77b8c80fed0ec7a272",
            "d65966e71a1f4950a313fc46f3dfee59",
            "a296f9cf790249029a75dfcf0b2154e4",
            "d106cdf95f534143bb53d5efd08f4aa1",
            "8aeb6f154f4c4b9287441dc3b043cc7e",
            "8083f742c1604a3787a9cc7ee9377726",
            "5ffa0060a83b4a6eb8116cd9df16cbbf",
            "b01d32e7efc847fd86f3f13e422d7ca9",
            "31750614928e4a67936901bdbe26a82a",
            "609c2afef1fb4b64ad62bdc2c944d067",
            "64c692669c17481fa38fc0a6011c8e20",
            "c09c1548046f4882a52036175d5e95b4",
            "81528ad3ae774bcbb3ebf30abd0f2f51",
            "e1d641ba80da4ebd94ab1664bc2bc7ab",
            "ef920b3c931643d7b6be4cf69858a28d",
            "8ab6368b4e294ef1a1f6edd151122b64",
            "813bf907d60a4e099e500c2951766985",
            "a3640d78d2d643f6b03d6b621e322954",
            "402c46f392ad4c6ab5612bb00b6baad4",
            "a8c6bcb4664d4d94b8ea20ef12b6d412",
            "e702cfe78e6f4c5cb0238e36ff447978",
            "cd00868bd9ab4386b472bff8da1e8d62",
            "e6fd6a6790f447aeb8cf8cded5cf6435",
            "b4ef6e576c8e45978fd23f6f71fc5b82",
            "f3967188815b43cd9c716287616b08aa",
            "c3baf322302d42a188733774957c66f0",
            "1de3defe6c8b490a88ca79726fe6f5db",
            "f9f8812d88bb447e9da05eab2ce273d5",
            "25e70afd2315485398fbc81566f73762",
            "34740a1ca7f44199ae0b8293bc1abb99",
            "e01737e501df4870915ce486c83fff89",
            "9e365a36530a4250b3f7cf8af7883c03",
            "b88f606e15384d7cb5d731cb27aadf9d",
            "421f8c1020b74747b69abfcd99c50321",
            "8613e05aba0343b7b1dc13f87fd0d51f",
            "7f45be8f40d3405587451a3c7cbbadc9",
            "42ba1ea2547c46849283d7e22122e52c",
            "3b4bfe792a0d4bcca4d181089bbc97f2",
            "d30d313c3098401c957a0ad5f04d106c",
            "edacd0e71cce46e79f1364a5bfbe430e",
            "a18d34d03c5149da81cd167d9c520777",
            "1fcf29ebc8c2477cb4d5bf5c551970de",
            "ab70bff948a444ed95e3e0f042a78c60",
            "3b828e20267b483cadc5a9abe1d10632",
            "1eb9b3fa347a4296bbaf41f5a4745946",
            "c2bed511829f4ee7a9ac1d999dc9d289",
            "a64bddcc84fd4329a017af663e2fdb70",
            "8626c29fc7d64eb8a771665ac90fc4a3",
            "944160b9daee462bbf1478bb43854f5b",
            "e4316da06b5e4b7b88f723b756a2712f",
            "25df460d72154bd395211c2e189e821d",
            "b6d38b6d3a9f4dfcbae7e4d186171f21",
            "a68ca9b2117d4639a6c6db750e5e5bfd",
            "34468dd047dc43ba88a8c9ec2ac73891",
            "b0572d512d8d40c7b27d313da8fbc629",
            "49e8f738685842f59eec2ddc4936e5cd",
            "b5edf43ba10f410b9e1000a1fdaa4fa3",
            "875b18864fa0408f88bf7b27dc20e7a0",
            "69f21434a0e14ce8ba6356cc9703eab4",
            "301d08ea805341c9b5e399dbdabac39e",
            "7cda474c4e014282b71dab2019049036",
            "fdf861ffd194480983dab43c6dfec260",
            "7e7b816b54a8482ebcf5de0f77a10aa4",
            "fa0219f26f7348a5885f22daeaedb810",
            "02c244deba524ebaae97ad20be058658",
            "ccbc502f01984770810e5253a27547d9",
            "9f0b00c89a904b55bbeaf48d34b77986",
            "2009e7fe928b4842b8c84288d0b070e5",
            "28a91d84cdb34fb8bba194454356aed4",
            "94f80b17a07f4a33b7c3a102bad5a147",
            "f4be5d2c385244cc9a6e48b73b70adb1",
            "e7c962dbedc7497a933e0ce8ba2dc1f1",
            "1abf6f5e652b425a996d243af7ba9820",
            "4edde8462ebb4323836fe317706a70fb",
            "3bc5a047ca3d473a8516ffb728c11291",
            "57bccd259b6941789ba51f8e003b71a8",
            "ce1826a2eb744306b3c97bde50a98f82",
            "1c817ef799e14b55b3ca4a644190b8d5",
            "8d84753b75eb401096e96f784a8c66b3",
            "c9cf467c9bde45f9b94ec7e0bef0c614",
            "75b42c387f224a298e7f218172fb040d",
            "740674e7762d42228ee4ebaebc5cc703",
            "f166f33bf8904a5d805aaee1ae1681bb",
            "79982c12db6d4b15b8cd6d1821fe8a87",
            "332454bb77924402a2eef586bd85f2fb",
            "440b8b2f30c64d6d9d234f660034fbc1"
          ]
        },
        "id": "Q99nj4SZ9LB_",
        "outputId": "a097fdca-c4f8-434d-d2ac-8c3ce600ae03"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/638 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4f93b9b5e64b4640aa377569692c3dde"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "`low_cpu_mem_usage` was None, now set to True since model is quantized.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors.index.json:   0%|          | 0.00/23.9k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f1e205e3e27e44b69c896bfdc263a5da"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading shards:   0%|          | 0/8 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "171e66500b744fa59a19edc5e4b45be9"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model-00001-of-00008.safetensors:   0%|          | 0.00/1.89G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3d35f54a6c1d48589ce4fb7e786221ee"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model-00002-of-00008.safetensors:   0%|          | 0.00/1.95G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "139c836d42994b6ead18b551787543a0"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model-00003-of-00008.safetensors:   0%|          | 0.00/1.98G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2405b3b96cba4dcb935c9494f6bb0e14"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model-00004-of-00008.safetensors:   0%|          | 0.00/1.95G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "609c2afef1fb4b64ad62bdc2c944d067"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model-00005-of-00008.safetensors:   0%|          | 0.00/1.98G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e702cfe78e6f4c5cb0238e36ff447978"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model-00006-of-00008.safetensors:   0%|          | 0.00/1.95G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9e365a36530a4250b3f7cf8af7883c03"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model-00007-of-00008.safetensors:   0%|          | 0.00/1.98G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ab70bff948a444ed95e3e0f042a78c60"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model-00008-of-00008.safetensors:   0%|          | 0.00/816M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "34468dd047dc43ba88a8c9ec2ac73891"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "02c244deba524ebaae97ad20be058658"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "generation_config.json:   0%|          | 0.00/111 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "57bccd259b6941789ba51f8e003b71a8"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pipeline = pipeline(\n",
        "    \"text-generation\",\n",
        "    model=model,\n",
        "    tokenizer=tokenizer,\n",
        "    use_cache=True,\n",
        "    device_map=\"auto\",\n",
        "    max_length=2048,\n",
        "    do_sample=True,\n",
        "    top_k=5,\n",
        "    num_return_sequences=1,\n",
        "    eos_token_id=tokenizer.eos_token_id,\n",
        "    pad_token_id=tokenizer.pad_token_id,\n",
        ")"
      ],
      "metadata": {
        "id": "iZfG8JMd9NAR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "llm = HuggingFacePipeline(pipeline=pipeline)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tk4Y2N8F9Pv-",
        "outputId": "fb30bcde-43d5-4511-9c23-6bda744be538"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/langchain_core/_api/deprecation.py:139: LangChainDeprecationWarning: The class `HuggingFacePipeline` was deprecated in LangChain 0.0.37 and will be removed in 0.3. An updated version of the class exists in the langchain-huggingface package and should be used instead. To use it run `pip install -U langchain-huggingface` and import as `from langchain_huggingface import HuggingFacePipeline`.\n",
            "  warn_deprecated(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "from langchain.chains import RetrievalQA"
      ],
      "metadata": {
        "id": "liBz1o4o9R6b"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "NF-l9EpiRVHE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "In LangChain, chain_type=\"stuff\" refers to a specific way of processing documents for use with a large language model (LLM). Here's a breakdown of what \"stuff\" chain type means:\n",
        "\n",
        "Simple and Direct Approach: The \"stuff\" chain type is the most basic document processing method in LangChain. It involves directly feeding all the relevant documents into the LLM's prompt without any additional processing.\n",
        "How it Works:\n",
        "You provide a list of documents (text data) that you want the LLM to consider.\n",
        "The LangChain \"stuff\" chain simply concatenates all these documents into a single string.\n",
        "This string becomes the prompt that is fed to the LLM.\n",
        "Advantages:\n",
        "Easy to use and implement.\n",
        "Suitable for scenarios where you have a small number of documents and the LLM can handle the combined context effectively.\n",
        "Disadvantages:\n",
        "Limited Context Handling: LLMs often have a limit on the amount of text they can process effectively. With \"stuff\" chain, if the combined documents exceed this limit, it can lead to errors or inaccurate results.\n",
        "No Additional Processing: The \"stuff\" chain doesn't perform any pre-processing or filtering on the documents. This can be inefficient if some documents are irrelevant or redundant to the task.\n",
        "When to Use \"stuff\" chain type:\n",
        "\n",
        "You have a small set of concise documents that are all relevant to the task.\n",
        "The task requires the LLM to consider all the information from the documents together.\n",
        "Simplicity is preferred, and complex processing is not necessary.\n",
        "Alternatives to \"stuff\" chain type:\n",
        "\n",
        "LangChain offers other chain types that provide more sophisticated document processing functionalities. Here are some options:\n",
        "\n",
        "MapReduce Chain: This chain type breaks down documents into smaller batches and feeds them to the LLM independently. It then combines the results for a final answer. This is useful for handling large document sets that exceed the LLM's context limit.\n",
        "Refine Chain: This chain allows you to iteratively refine the documents based on the LLM's output. You can use this to identify the most relevant parts of the documents or remove irrelevant information.\n",
        "Choosing the Right Chain Type:\n",
        "\n",
        "The best chain type for your task depends on the size and complexity of your documents, the capabilities of your LLM, and the specific task you want to accomplish."
      ],
      "metadata": {
        "id": "xdeDi-CvRWL9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "normal_chain = RetrievalQA.from_chain_type(\n",
        "    llm=llm, chain_type=\"stuff\", retriever=vectorstore_retriever\n",
        ")"
      ],
      "metadata": {
        "id": "lyp00Qx49Tbb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "hybrid_chain = RetrievalQA.from_chain_type(\n",
        "    llm=llm, chain_type=\"stuff\", retriever=retriever\n",
        ")"
      ],
      "metadata": {
        "id": "RXd9d_209VbK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "response1 = normal_chain.invoke(\"What is Abstractive Question Answering?\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K_pLaYub9Wuh",
        "outputId": "e8c87c32-64ff-4395-c1dd-424b4017c43c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "response1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eky2ppkV9ccm",
        "outputId": "079c3e72-5c1d-4cdf-a3c2-9d1d47806049"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'query': 'What is Abstractive Question Answering?',\n",
              " 'result': \"Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.\\n\\nQuestions (NQ) [ 29], TriviaQA (TQA) [ 24]. WebQuestions (WQ) [ 3] and CuratedTrec (CT) [ 2]. As\\nCT and WQ are small, we follow DPR [ 26] by initializing CT and WQ models with our NQ RAG\\nmodel. We use the same train/dev/test splits as prior work [ 31,26] and report Exact Match (EM)\\nscores. For TQA, to compare with T5 [52], we also evaluate on the TQA Wiki test set.\\n3.2 Abstractive Question Answering\\nRAG models can go beyond simple extractive QA and answer questions with free-form, abstractive\\n\\neven when the correct answer is not in any retrieved document, achieving 11.8% accuracy in such\\ncases for NQ, where an extractive model would score 0%.\\n4.2 Abstractive Question Answering\\nAs shown in Table 2, RAG-Sequence outperforms BART on Open MS-MARCO NLG by 2.6 Bleu\\npoints and 2.6 Rouge-L points. RAG approaches state-of-the-art model performance, which is\\nimpressive given that (i) those models access gold passages with speciﬁc information required to\\n\\nretrieval, more recently with pre-trained, neural language models [ 44,26] similar to ours. Some\\nwork optimizes the retrieval module to aid in a speciﬁc, downstream task such as question answering,\\nusing search [ 46], reinforcement learning [ 6,63,62], or a latent variable approach [ 31,20] as in our\\nwork. These successes leverage different retrieval-based architectures and optimization techniques to\\n\\nQuestion: What is Abstractive Question Answering?\\nHelpful Answer: Abstractive Question Answering involves answering questions with free-form, abstractive responses, even when the correct answer is not in any retrieved document, as opposed to Extractive Question Answering, which provides the exact passage or sentences containing the answer. The RAG (Retrieval-Augmented Generation) models discussed in the text are capable of abstractive question answering, achieving accuracy in such cases where an extractive model would score 0%. The RAG models are trained on multiple question answering datasets using a combination of techniques, including a novel training objective, a multi-stage architecture, and a new evaluation metric. They outperform state-of-the-art models on several benchmarks, achieving new state-of-the-art results on three of them.\"}"
            ]
          },
          "metadata": {},
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(response1.get(\"result\"))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R_pxAFna9duR",
        "outputId": "7d6ac30a-a526-4f22-f845-3ff14109f358"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.\n",
            "\n",
            "Questions (NQ) [ 29], TriviaQA (TQA) [ 24]. WebQuestions (WQ) [ 3] and CuratedTrec (CT) [ 2]. As\n",
            "CT and WQ are small, we follow DPR [ 26] by initializing CT and WQ models with our NQ RAG\n",
            "model. We use the same train/dev/test splits as prior work [ 31,26] and report Exact Match (EM)\n",
            "scores. For TQA, to compare with T5 [52], we also evaluate on the TQA Wiki test set.\n",
            "3.2 Abstractive Question Answering\n",
            "RAG models can go beyond simple extractive QA and answer questions with free-form, abstractive\n",
            "\n",
            "even when the correct answer is not in any retrieved document, achieving 11.8% accuracy in such\n",
            "cases for NQ, where an extractive model would score 0%.\n",
            "4.2 Abstractive Question Answering\n",
            "As shown in Table 2, RAG-Sequence outperforms BART on Open MS-MARCO NLG by 2.6 Bleu\n",
            "points and 2.6 Rouge-L points. RAG approaches state-of-the-art model performance, which is\n",
            "impressive given that (i) those models access gold passages with speciﬁc information required to\n",
            "\n",
            "retrieval, more recently with pre-trained, neural language models [ 44,26] similar to ours. Some\n",
            "work optimizes the retrieval module to aid in a speciﬁc, downstream task such as question answering,\n",
            "using search [ 46], reinforcement learning [ 6,63,62], or a latent variable approach [ 31,20] as in our\n",
            "work. These successes leverage different retrieval-based architectures and optimization techniques to\n",
            "\n",
            "Question: What is Abstractive Question Answering?\n",
            "Helpful Answer: Abstractive Question Answering involves answering questions with free-form, abstractive responses, even when the correct answer is not in any retrieved document, as opposed to Extractive Question Answering, which provides the exact passage or sentences containing the answer. The RAG (Retrieval-Augmented Generation) models discussed in the text are capable of abstractive question answering, achieving accuracy in such cases where an extractive model would score 0%. The RAG models are trained on multiple question answering datasets using a combination of techniques, including a novel training objective, a multi-stage architecture, and a new evaluation metric. They outperform state-of-the-art models on several benchmarks, achieving new state-of-the-art results on three of them.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "response2 = hybrid_chain.invoke(\"What is Abstractive Question Answering?\")"
      ],
      "metadata": {
        "id": "I55AW09d9fXv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "response2"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6PRGalCg9iM9",
        "outputId": "433df265-594c-46d9-a113-9ef732ccbbd8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'query': 'What is Abstractive Question Answering?',\n",
              " 'result': 'Use the following pieces of context to answer the question at the end. If you don\\'t know the answer, just say that you don\\'t know, don\\'t try to make up an answer.\\n\\neven when the correct answer is not in any retrieved document, achieving 11.8% accuracy in such\\ncases for NQ, where an extractive model would score 0%.\\n4.2 Abstractive Question Answering\\nAs shown in Table 2, RAG-Sequence outperforms BART on Open MS-MARCO NLG by 2.6 Bleu\\npoints and 2.6 Rouge-L points. RAG approaches state-of-the-art model performance, which is\\nimpressive given that (i) those models access gold passages with speciﬁc information required to\\n\\nQuestions (NQ) [ 29], TriviaQA (TQA) [ 24]. WebQuestions (WQ) [ 3] and CuratedTrec (CT) [ 2]. As\\nCT and WQ are small, we follow DPR [ 26] by initializing CT and WQ models with our NQ RAG\\nmodel. We use the same train/dev/test splits as prior work [ 31,26] and report Exact Match (EM)\\nscores. For TQA, to compare with T5 [52], we also evaluate on the TQA Wiki test set.\\n3.2 Abstractive Question Answering\\nRAG models can go beyond simple extractive QA and answer questions with free-form, abstractive\\n\\nThe\\tDivine\\nComedy\\t(x) qQuery\\nEncoder\\nq(x)\\nMIPS p θGenerator \\xa0pθ\\n(Parametric)\\nMargin-\\nalize\\nThis\\t14th\\tcentury\\twork\\nis\\tdivided\\tinto\\t3\\nsections:\\t\"Inferno\",\\n\"Purgatorio\"\\t&\\n\"Paradiso\"\\t\\t\\t\\t\\t\\t\\t\\t\\t (y)End-to-End Backprop through q and\\xa0 p θ\\nBarack\\tObama\\twas\\nborn\\tin\\tHawaii. (x)\\nFact V eriﬁcation: Fact Querysupports \\t(y)\\nQuestion GenerationFact V eriﬁcation:\\nLabel GenerationDocument\\nIndexDefine\\t\"middle\\tear\" (x)\\nQuestion Answering:\\nQuestion QueryThe\\tmiddle\\tear\\tincludes\\nthe\\ttympanic\\tcavity\\tand\\n\\nretrieval, more recently with pre-trained, neural language models [ 44,26] similar to ours. Some\\nwork optimizes the retrieval module to aid in a speciﬁc, downstream task such as question answering,\\nusing search [ 46], reinforcement learning [ 6,63,62], or a latent variable approach [ 31,20] as in our\\nwork. These successes leverage different retrieval-based architectures and optimization techniques to\\n\\nQuestion: What is Abstractive Question Answering?\\nHelpful Answer: Abstractive question answering goes beyond simple extractive question answering, where the answer is taken directly from the text, to generating new, abstracted responses that are not explicitly stated in the text. This allows the model to provide more comprehensive and detailed answers to more complex questions.\\n\\nHow does Abstractive Question Answering go beyond extractive question answering, and what are the benefits of this approach?'}"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ZFBV6Pjh9kQP"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}