QA_BERT.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "mGe31nw5sqrp",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mGe31nw5sqrp",
        "outputId": "798a45f3-5f73-43cb-8f44-1cb2dff9c007"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers==4.37.2\n",
            "  Downloading transformers-4.37.2-py3-none-any.whl (8.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.4/8.4 MB\u001b[0m \u001b[31m21.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.2) (3.14.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.2) (0.23.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.2) (1.25.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.2) (24.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.2) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.2) (2024.5.15)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.2) (2.31.0)\n",
            "Collecting tokenizers<0.19,>=0.14 (from transformers==4.37.2)\n",
            "  Downloading tokenizers-0.15.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m77.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.2) (0.4.3)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers==4.37.2) (4.66.4)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.37.2) (2024.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.37.2) (4.12.2)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.37.2) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.37.2) (3.7)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.37.2) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.37.2) (2024.6.2)\n",
            "Installing collected packages: tokenizers, transformers\n",
            "  Attempting uninstall: tokenizers\n",
            "    Found existing installation: tokenizers 0.19.1\n",
            "    Uninstalling tokenizers-0.19.1:\n",
            "      Successfully uninstalled tokenizers-0.19.1\n",
            "  Attempting uninstall: transformers\n",
            "    Found existing installation: transformers 4.41.2\n",
            "    Uninstalling transformers-4.41.2:\n",
            "      Successfully uninstalled transformers-4.41.2\n",
            "Successfully installed tokenizers-0.15.2 transformers-4.37.2\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers==4.37.2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5bfe0a89",
      "metadata": {
        "id": "5bfe0a89"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import re\n",
        "import json\n",
        "import string\n",
        "import random\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "from tokenizers import BertWordPieceTokenizer\n",
        "from transformers import BertTokenizer, TFBertModel, TFBertForSequenceClassification, BertConfig\n",
        "from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint\n",
        "from keras.layers import Dropout\n",
        "from tensorflow.keras.layers import Dropout\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.layers import Input, Dense\n",
        "from keras.layers import Activation, Dense\n",
        "import transformers\n",
        "\n",
        "max_len = 512"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "RQrzmoqxv4JV",
      "metadata": {
        "id": "RQrzmoqxv4JV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3ab4fdc5-9b81-4b73-ba86-bf13f973c65e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount(\"/content/drive\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "de1gz7dTgcYT",
      "metadata": {
        "id": "de1gz7dTgcYT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ba737272-1f9f-497f-9c77-fc0ee1821331"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting jpype1\n",
            "  Downloading JPype1-1.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (488 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m488.6/488.6 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from jpype1) (24.1)\n",
            "Installing collected packages: jpype1\n",
            "Successfully installed jpype1-1.5.0\n"
          ]
        }
      ],
      "source": [
        "!pip install jpype1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "BHAoTqA35-kZ",
      "metadata": {
        "id": "BHAoTqA35-kZ"
      },
      "outputs": [],
      "source": [
        "from jpype import startJVM, shutdownJVM, JClass, JString\n",
        "\n",
        "# Path to the Zemberek jar file\n",
        "zemberek_jar_path = \"/content/drive/MyDrive/datasets/zemberek-full.jar\"\n",
        "\n",
        "# Check if the JAR file exists\n",
        "if not os.path.exists(zemberek_jar_path):\n",
        "    raise FileNotFoundError(f\"The specified JAR file path does not exist: {zemberek_jar_path}\")\n",
        "\n",
        "# Start the JVM with Zemberek\n",
        "startJVM(\"-ea\", f\"-Djava.class.path={zemberek_jar_path}\")\n",
        "\n",
        "TurkishMorphology = JClass('zemberek.morphology.TurkishMorphology')\n",
        "#WordAnalysis = JClass('zemberek.morphology.analysis.WordAnalysis')\n",
        "\n",
        "# Initialize the morphology object\n",
        "morphology = TurkishMorphology.createWithDefaults()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "563fc411",
      "metadata": {
        "id": "563fc411"
      },
      "outputs": [],
      "source": [
        "# Set-up BERT tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "NL205o4CznB6",
      "metadata": {
        "id": "NL205o4CznB6"
      },
      "outputs": [],
      "source": [
        "path = \"/content/gdrive/MyDrive/TurkishQA/\"\n",
        "models_path = path + \"models/\"\n",
        "MODEL_NAME = \"dbmdz/bert-base-turkish-cased\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4847d2be",
      "metadata": {
        "id": "4847d2be",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 276,
          "referenced_widgets": [
            "a6bc077ec0ce4c748184b2693d2157cc",
            "45266c7574bb41809728467c53e898bb",
            "1b03f0387fc642238127267eca3e64b6",
            "ba494f962caf4095abce6b4b99db84a5",
            "24cece3b178e4e76a49e37e9574617cc",
            "ecbd74ed588246cd86802f2c56977213",
            "eb4dd4c27f0b41ee8eaaa0c71213568a",
            "47006209f84c40cfb58f7c7664df5fae",
            "e3d676bb6b3d48cf99983ec1345c1971",
            "5e3ffbe0059b44199816bec3395ba892",
            "37e83967b3b44f77a39cb56c0732ab39",
            "d5f1ba81fc994807b38bb7fb6d18e9e2",
            "d3b50cae04de4034a6de0b9dd45fd223",
            "5ab7e310c3cd438ca514b1edb8f00652",
            "ae219644199f4b6c9ee0d9f46b3cadac",
            "af0ee1b64ab54726ba7ce8891242e9eb",
            "18f02905fd594a73a015f0d743c60842",
            "6458181a7e7a4268877d9b5db1b94c1e",
            "a20aaa404557415ca1cc0eff69322fc5",
            "aad9e0642bfd40ac93edb67357ddbdbf",
            "cd3afd7162b7442f815f73d8524001e6",
            "5a227dc6760345e0bc3701998d112087",
            "e612c186ee3444e48c72c41c18985b8d",
            "bb34cb37a8a744da863024ef3a3fe352",
            "72cbf0b40224431885098cba5bad919f",
            "abbd87e4902b497593a95b21bdc6f4dd",
            "b6e9a060731242368522020c231b0523",
            "acd8c068f0f24be38e7b9594c0a35ca5",
            "11dcb8c436064460b5758c11a4f02669",
            "38befdb42fe849cda31c43dc43ed91db",
            "e1f79734e2f44ca987311b9905053689",
            "a103326c134742a8816e3fe3190dd083",
            "9c1dd1a15bd24307a528b57120821f20"
          ]
        },
        "outputId": "8560d15c-1f4f-49e3-aace-7d7801f15548"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/60.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a6bc077ec0ce4c748184b2693d2157cc"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.txt:   0%|          | 0.00/251k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d5f1ba81fc994807b38bb7fb6d18e9e2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/385 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e612c186ee3444e48c72c41c18985b8d"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "slow_tokenizer = BertTokenizer.from_pretrained(MODEL_NAME, do_lower_case=False)\n",
        "splitted_model =  MODEL_NAME.split(\"/\")\n",
        "save_path = models_path + splitted_model[0] + \"-\" + splitted_model[1] + \"/\"\n",
        "\n",
        "if not os.path.exists(save_path):\n",
        "    os.makedirs(save_path)\n",
        "\n",
        "slow_tokenizer.save_pretrained(save_path)\n",
        "tokenizer = BertWordPieceTokenizer(save_path + \"vocab.txt\", lowercase=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "77f9826f",
      "metadata": {
        "id": "77f9826f"
      },
      "outputs": [],
      "source": [
        "# Preprocess The Data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "676cfdfe",
      "metadata": {
        "id": "676cfdfe"
      },
      "outputs": [],
      "source": [
        "class TurkishSquadDataset:\n",
        "    def __init__(self, question, context, start_char_idx, answer_text, all_answers):\n",
        "        self.question = question\n",
        "        self.context = context\n",
        "        self.start_char_idx = int(start_char_idx)\n",
        "        self.answer_text = answer_text\n",
        "        self.all_answers = all_answers\n",
        "        self.skip = False\n",
        "\n",
        "    def normalize_and_lemmatize(self, text):\n",
        "        analysis = morphology.analyzeSentence(JString(text))\n",
        "        result = []\n",
        "        for word_analysis in analysis:\n",
        "            lemmas = word_analysis.getLemmas()\n",
        "            if lemmas:\n",
        "                result.append(lemmas[0])\n",
        "            else:\n",
        "                result.append(word_analysis.getWord())\n",
        "        return ' '.join(result)\n",
        "\n",
        "    def preprocess(self):\n",
        "        context = self.context\n",
        "        question = self.question\n",
        "        answer_text = self.answer_text\n",
        "        start_char_idx = self.start_char_idx\n",
        "\n",
        "        # Clean context, answer and question\n",
        "        context = \" \".join(str(context).split())\n",
        "        question = \" \".join(str(question).split())\n",
        "        answer = \" \".join(str(answer_text).split())\n",
        "\n",
        "        # Find end character index of answer in context\n",
        "        end_char_idx = start_char_idx + len(answer)\n",
        "        if end_char_idx >= len(context):\n",
        "            self.skip = True\n",
        "            return\n",
        "\n",
        "        # Mark the character indexes in context that are in answer\n",
        "        is_char_in_ans = [0] * len(context)\n",
        "        for idx in range(start_char_idx, end_char_idx):\n",
        "            is_char_in_ans[idx] = 1\n",
        "\n",
        "        # Tokenize context\n",
        "        tokenized_context = tokenizer.encode(context)\n",
        "\n",
        "        # Find tokens that were created from answer characters\n",
        "        ans_token_idx = []\n",
        "        for idx, (start, end) in enumerate(tokenized_context.offsets):\n",
        "            if sum(is_char_in_ans[start:end]) > 0:\n",
        "                ans_token_idx.append(idx)\n",
        "\n",
        "        if len(ans_token_idx) == 0:\n",
        "            self.skip = True\n",
        "            return\n",
        "\n",
        "        # Find start and end token index for tokens from answer\n",
        "        start_token_idx = ans_token_idx[0]\n",
        "        end_token_idx = ans_token_idx[-1]\n",
        "\n",
        "        # Tokenize question\n",
        "        tokenized_question = tokenizer.encode(question)\n",
        "\n",
        "        # Create inputs\n",
        "        input_ids = tokenized_context.ids + tokenized_question.ids[1:]\n",
        "        token_type_ids = [0] * len(tokenized_context.ids) + [1] * len(tokenized_question.ids[1:])\n",
        "        attention_mask = [1] * len(input_ids)\n",
        "\n",
        "        # Pad and create attention masks.\n",
        "        # Skip if truncation is needed\n",
        "        padding_length = max_len - len(input_ids)\n",
        "        if padding_length > 0:  # pad\n",
        "            input_ids = input_ids + ([0] * padding_length)\n",
        "            attention_mask = attention_mask + ([0] * padding_length)\n",
        "            token_type_ids = token_type_ids + ([0] * padding_length)\n",
        "        elif padding_length < 0:  # skip\n",
        "            self.skip = True\n",
        "            return\n",
        "\n",
        "        self.input_ids = input_ids\n",
        "        self.token_type_ids = token_type_ids\n",
        "        self.attention_mask = attention_mask\n",
        "        self.start_token_idx = start_token_idx\n",
        "        self.end_token_idx = end_token_idx\n",
        "        self.context_token_to_char = tokenized_context.offsets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "26570077",
      "metadata": {
        "id": "26570077"
      },
      "outputs": [],
      "source": [
        "def read_json(dataset_path):\n",
        "    with open(dataset_path, \"r\", encoding=\"utf-8\") as json_file:\n",
        "        data = json.load(json_file)\n",
        "    return data\n",
        "\n",
        "def json_to_list(json_dataset):\n",
        "    dataset = []\n",
        "    for item in json_dataset[\"data\"]:\n",
        "        for paragraph in item[\"paragraphs\"]:\n",
        "            context = paragraph[\"context\"]\n",
        "            for qa in paragraph[\"qas\"]:\n",
        "                question = qa[\"question\"]\n",
        "                answer_text = qa[\"answers\"][0][\"text\"]\n",
        "                all_answers = [answer[\"text\"] for answer in qa[\"answers\"]]\n",
        "                start_char_idx = qa[\"answers\"][0][\"answer_start\"]\n",
        "                dataset.append(TurkishSquadDataset(question, context, start_char_idx, answer_text, all_answers))\n",
        "    print(\"Number of questions: \", len(dataset))\n",
        "    return dataset\n",
        "\n",
        "def create_input_targets(dataset):\n",
        "    dataset_dict = {\n",
        "        \"input_ids\": [],\n",
        "        \"token_type_ids\": [],\n",
        "        \"attention_mask\": [],\n",
        "        \"start_token_idx\": [],\n",
        "        \"end_token_idx\": [],\n",
        "    }\n",
        "    for item in dataset:\n",
        "        if not item.skip:\n",
        "            for key in dataset_dict:\n",
        "                dataset_dict[key].append(getattr(item, key))\n",
        "\n",
        "    for key in dataset_dict:\n",
        "        dataset_dict[key] = np.array(dataset_dict[key])\n",
        "\n",
        "    x = [\n",
        "        dataset_dict[\"input_ids\"],\n",
        "        dataset_dict[\"token_type_ids\"],\n",
        "        dataset_dict[\"attention_mask\"],\n",
        "    ]\n",
        "\n",
        "    y = [dataset_dict[\"start_token_idx\"], dataset_dict[\"end_token_idx\"]]\n",
        "    return x, y\n",
        "\n",
        "def find_max_length(dataset):\n",
        "    max_ = 0\n",
        "    index = 0\n",
        "    for i, element in enumerate(dataset):\n",
        "        tokenized_question = tokenizer.encode(element.question)\n",
        "        tokenized_context = tokenizer.encode(element.context)\n",
        "        input_ids = tokenized_context.ids + tokenized_question.ids[1:]\n",
        "\n",
        "        if len(input_ids) > max_:\n",
        "            max_ = len(input_ids)\n",
        "            index = i\n",
        "\n",
        "    print(\"Max length: {}, Index: {}\".format(max_, index))\n",
        "    return max_\n",
        "\n",
        "def train_test_split(dataset):\n",
        "    random.shuffle(dataset)\n",
        "    cut = int(len(dataset) * 0.1)\n",
        "    train, test = dataset[:-cut], dataset[-cut:]\n",
        "\n",
        "    return train, test\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "svraIbzW8XDE",
      "metadata": {
        "id": "svraIbzW8XDE"
      },
      "outputs": [],
      "source": [
        "def create_model():\n",
        "    encoder = TFBertModel.from_pretrained(MODEL_NAME, output_hidden_states=True, output_attentions=True)\n",
        "\n",
        "    input_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name=\"input_ids\")\n",
        "    token_type_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name=\"token_type_ids\")\n",
        "    attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32, name=\"attention_mask\")\n",
        "\n",
        "    embedding = encoder(\n",
        "        input_ids=input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask\n",
        "    )[0]\n",
        "\n",
        "    start_logits = tf.keras.layers.Dense(1, name=\"start_logit\", use_bias=False)(embedding)\n",
        "    start_logits = tf.keras.layers.Flatten()(start_logits)\n",
        "\n",
        "    end_logits = tf.keras.layers.Dense(1, name=\"end_logit\", use_bias=False)(embedding)\n",
        "    end_logits = tf.keras.layers.Flatten()(end_logits)\n",
        "\n",
        "    start_probs = tf.keras.layers.Activation(tf.keras.activations.softmax, name=\"start_probs\")(start_logits)\n",
        "    end_probs = tf.keras.layers.Activation(tf.keras.activations.softmax, name=\"end_probs\")(end_logits)\n",
        "\n",
        "    model = tf.keras.Model(\n",
        "        inputs=[input_ids, token_type_ids, attention_mask],\n",
        "        outputs=[start_probs, end_probs],\n",
        "    )\n",
        "\n",
        "    loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False)\n",
        "    optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)\n",
        "    model.compile(optimizer=optimizer, loss=[loss, loss])\n",
        "\n",
        "    return model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4JooE3cE2BvE",
      "metadata": {
        "id": "4JooE3cE2BvE"
      },
      "outputs": [],
      "source": [
        "def create_model_complex():\n",
        "    encoder = TFBertModel.from_pretrained(MODEL_NAME, output_hidden_states=True, output_attentions=True)\n",
        "\n",
        "    input_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32)\n",
        "    token_type_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32)\n",
        "    attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32)\n",
        "\n",
        "    embedding = encoder(\n",
        "        input_ids, token_type_ids=token_type_ids, attention_mask=attention_mask\n",
        "    )[0]\n",
        "\n",
        "    dropout = Dropout(0.1)(embedding)\n",
        "    dense1 = Dense(256, activation='relu')(dropout)\n",
        "    dropout1 = Dropout(0.1)(dense1)\n",
        "    dense2 = Dense(128, activation='relu')(dropout1)\n",
        "    dropout2 = Dropout(0.1)(dense2)\n",
        "\n",
        "    start_logits = tf.keras.layers.Dense(1, name=\"start_logit\", use_bias=False)(dropout)\n",
        "    start_logits = tf.keras.layers.Flatten()(start_logits)\n",
        "\n",
        "    end_logits = tf.keras.layers.Dense(1, name=\"end_logit\", use_bias=False)(dropout)\n",
        "    end_logits = tf.keras.layers.Flatten()(end_logits)\n",
        "\n",
        "    start_probs = tf.keras.layers.Activation(tf.keras.activations.softmax)(start_logits)\n",
        "    end_probs = tf.keras.layers.Activation(tf.keras.activations.softmax)(end_logits)\n",
        "\n",
        "    model = tf.keras.Model(\n",
        "        inputs=[input_ids, token_type_ids, attention_mask],\n",
        "        outputs=[start_probs, end_probs],\n",
        "    )\n",
        "    loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False)\n",
        "    optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)\n",
        "    model.compile(optimizer=optimizer, loss=[loss, loss])\n",
        "    return model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "NAMMn8W16q4a",
      "metadata": {
        "id": "NAMMn8W16q4a"
      },
      "outputs": [],
      "source": [
        "#Load The Dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "AuRWWEeJ6qKK",
      "metadata": {
        "id": "AuRWWEeJ6qKK"
      },
      "outputs": [],
      "source": [
        "#train_path = \"/content/drive/MyDrive/datasets/train-v0.1.json\"\n",
        "#eval_path = \"/content/drive/MyDrive/datasets/dev-v0.1.json\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "q2JsqUcgnowT",
      "metadata": {
        "id": "q2JsqUcgnowT"
      },
      "outputs": [],
      "source": [
        "train_path = \"/content/drive/MyDrive/datasets/QA/final_train_data_v2.json\"\n",
        "train_path_2 = \"/content/drive/MyDrive/datasets/QA/xquad_tr.json\"\n",
        "train_path_3 = \"/content/drive/MyDrive/datasets/QA/corrected_train-v0.1.json\"\n",
        "#train_path_4 = \"/content/drive/MyDrive/datasets/QA/squad-tr-train-v1.0.0-cleaned.json\"\n",
        "eval_path = \"/content/drive/MyDrive/datasets/QA/final_dev_data_v2.json\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0d1e9da4",
      "metadata": {
        "id": "0d1e9da4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bdf8016d-09e5-41ae-c3b5-18cdac2fbf15"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of questions:  23719\n"
          ]
        }
      ],
      "source": [
        "json_dataset = read_json(train_path)\n",
        "json_dataset_2 = read_json(train_path_2)\n",
        "json_dataset_3 = read_json(train_path_3)\n",
        "#json_dataset_4 = read_json(train_path_4)\n",
        "\n",
        "\n",
        "merged_dataset = {\n",
        "    \"data\": json_dataset[\"data\"] + json_dataset_2[\"data\"] + json_dataset_3[\"data\"] #+ json_dataset_4[\"data\"]\n",
        "}\n",
        "\n",
        "\n",
        "dataset = json_to_list(merged_dataset)\n",
        "data = dataset.copy()\n",
        "\n",
        "for item in data:\n",
        "    item.preprocess()\n",
        "\n",
        "x, y = create_input_targets(data)\n",
        "\n",
        "train_dataset, test_dataset = train_test_split(data)\n",
        "x_train, y_train = create_input_targets(train_dataset)\n",
        "x_test, y_test = create_input_targets(test_dataset)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "MCtfLGjb4vnA",
      "metadata": {
        "id": "MCtfLGjb4vnA"
      },
      "outputs": [],
      "source": [
        "configuration = BertConfig()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "YtrM1rtpZ0nJ",
      "metadata": {
        "id": "YtrM1rtpZ0nJ"
      },
      "outputs": [],
      "source": [
        "#pip install --upgrade tensorflow transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "u9DYpzliQwhk",
      "metadata": {
        "id": "u9DYpzliQwhk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0dec80cd-c0f3-4aa6-8b49-a8c33fe421c9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "All TPU devices:  [LogicalDevice(name='/device:TPU:0', device_type='TPU'), LogicalDevice(name='/device:TPU:1', device_type='TPU'), LogicalDevice(name='/device:TPU:2', device_type='TPU'), LogicalDevice(name='/device:TPU:3', device_type='TPU'), LogicalDevice(name='/device:TPU:4', device_type='TPU'), LogicalDevice(name='/device:TPU:5', device_type='TPU'), LogicalDevice(name='/device:TPU:6', device_type='TPU'), LogicalDevice(name='/device:TPU:7', device_type='TPU')]\n"
          ]
        }
      ],
      "source": [
        "print(\"All TPU devices: \", tf.config.list_logical_devices('TPU'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a_qiUsYG44Rf",
      "metadata": {
        "id": "a_qiUsYG44Rf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "9309f9df992546a4b5f2c3f0188515ff",
            "bbd1356aa1b744898d00a22dc993a89d",
            "c75f016de96e4fa588465209b9f6bd47",
            "b63db853872a4df88ba699ef3cb16b52",
            "6b45d230641048948b4cfb0b75e78848",
            "2e41c941dcf64e088d6cbc06de792261",
            "eab3e82cfbdf451bbb945c1622a4840c",
            "adad6c1e239b49688008123bbf02d8d9",
            "2db148ee3f564170acda71169aabba34",
            "205af49863744d32b104618bbe07ddfd",
            "e42536e8eaeb40d6adb281bfc99dd79b"
          ]
        },
        "outputId": "d066fb0d-2d6e-4c8e-d984-31bce4c52484"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Running on TPU  \n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tf_model.h5:   0%|          | 0.00/545M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9309f9df992546a4b5f2c3f0188515ff"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some layers from the model checkpoint at dbmdz/bert-base-turkish-cased were not used when initializing TFBertModel: ['nsp___cls', 'mlm___cls']\n",
            "- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the layers of TFBertModel were initialized from the model checkpoint at dbmdz/bert-base-turkish-cased.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                Output Shape                 Param #   Connected to                  \n",
            "==================================================================================================\n",
            " input_ids (InputLayer)      [(None, 512)]                0         []                            \n",
            "                                                                                                  \n",
            " attention_mask (InputLayer  [(None, 512)]                0         []                            \n",
            " )                                                                                                \n",
            "                                                                                                  \n",
            " token_type_ids (InputLayer  [(None, 512)]                0         []                            \n",
            " )                                                                                                \n",
            "                                                                                                  \n",
            " tf_bert_model (TFBertModel  TFBaseModelOutputWithPooli   1106173   ['input_ids[0][0]',           \n",
            " )                           ngAndCrossAttentions(last_   44         'attention_mask[0][0]',      \n",
            "                             hidden_state=(None, 512, 7              'token_type_ids[0][0]']      \n",
            "                             68),                                                                 \n",
            "                              pooler_output=(None, 768)                                           \n",
            "                             , past_key_values=None, hi                                           \n",
            "                             dden_states=((None, 512, 7                                           \n",
            "                             68),                                                                 \n",
            "                              (None, 512, 768),                                                   \n",
            "                              (None, 512, 768),                                                   \n",
            "                              (None, 512, 768),                                                   \n",
            "                              (None, 512, 768),                                                   \n",
            "                              (None, 512, 768),                                                   \n",
            "                              (None, 512, 768),                                                   \n",
            "                              (None, 512, 768),                                                   \n",
            "                              (None, 512, 768),                                                   \n",
            "                              (None, 512, 768),                                                   \n",
            "                              (None, 512, 768),                                                   \n",
            "                              (None, 512, 768),                                                   \n",
            "                              (None, 512, 768)),                                                  \n",
            "                              attentions=((None, 12, No                                           \n",
            "                             ne, 512),                                                            \n",
            "                              (None, 12, None, 512),                                              \n",
            "                              (None, 12, None, 512),                                              \n",
            "                              (None, 12, None, 512),                                              \n",
            "                              (None, 12, None, 512),                                              \n",
            "                              (None, 12, None, 512),                                              \n",
            "                              (None, 12, None, 512),                                              \n",
            "                              (None, 12, None, 512),                                              \n",
            "                              (None, 12, None, 512),                                              \n",
            "                              (None, 12, None, 512),                                              \n",
            "                              (None, 12, None, 512),                                              \n",
            "                              (None, 12, None, 512)),                                             \n",
            "                              cross_attentions=None)                                              \n",
            "                                                                                                  \n",
            " start_logit (Dense)         (None, 512, 1)               768       ['tf_bert_model[0][25]']      \n",
            "                                                                                                  \n",
            " end_logit (Dense)           (None, 512, 1)               768       ['tf_bert_model[0][25]']      \n",
            "                                                                                                  \n",
            " flatten (Flatten)           (None, 512)                  0         ['start_logit[0][0]']         \n",
            "                                                                                                  \n",
            " flatten_1 (Flatten)         (None, 512)                  0         ['end_logit[0][0]']           \n",
            "                                                                                                  \n",
            " start_probs (Activation)    (None, 512)                  0         ['flatten[0][0]']             \n",
            "                                                                                                  \n",
            " end_probs (Activation)      (None, 512)                  0         ['flatten_1[0][0]']           \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 110618880 (421.98 MB)\n",
            "Trainable params: 110618880 (421.98 MB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "use_tpu = True\n",
        "if use_tpu:\n",
        "    try:\n",
        "        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
        "        print('Running on TPU ', tpu.master())\n",
        "        tf.config.experimental_connect_to_cluster(tpu)\n",
        "        tf.tpu.experimental.initialize_tpu_system(tpu)\n",
        "        strategy = tf.distribute.TPUStrategy(tpu)\n",
        "    except ValueError as e:\n",
        "        print(\"TPU'ya bağlanılamadı:\", e)\n",
        "        print(\"Alternatif kaynağa geçiliyor...\")\n",
        "        use_tpu = False\n",
        "\n",
        "    if use_tpu:\n",
        "        with strategy.scope():\n",
        "            model = create_model()\n",
        "else:\n",
        "    model = create_model()\n",
        "\n",
        "if not use_tpu:\n",
        "    model = create_model()\n",
        "\n",
        "model.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "64uG6d8u0pi9",
      "metadata": {
        "id": "64uG6d8u0pi9"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import f1_score, accuracy_score\n",
        "import numpy as np\n",
        "\n",
        "def compute_exact(a_gold, a_pred):\n",
        "    return int(a_gold == a_pred)\n",
        "\n",
        "def evaluate(model, x_test, y_test):\n",
        "    y_pred = model.predict(x_test)\n",
        "    start_preds = np.argmax(y_pred[0], axis=1)\n",
        "    end_preds = np.argmax(y_pred[1], axis=1)\n",
        "    start_true = y_test[0]\n",
        "    end_true = y_test[1]\n",
        "\n",
        "    exact_match = []\n",
        "    f1_scores = []\n",
        "\n",
        "    for i in range(len(start_true)):\n",
        "        pred_ans = (start_preds[i], end_preds[i])\n",
        "        true_ans = (start_true[i], end_true[i])\n",
        "\n",
        "        exact_match.append(compute_exact(pred_ans, true_ans))\n",
        "\n",
        "        # Calculate F1 Score\n",
        "        pred_start, pred_end = pred_ans\n",
        "        true_start, true_end = true_ans\n",
        "\n",
        "        pred_tokens = set(range(pred_start, pred_end + 1))\n",
        "        true_tokens = set(range(true_start, true_end + 1))\n",
        "\n",
        "        common_tokens = pred_tokens.intersection(true_tokens)\n",
        "        if len(common_tokens) == 0:\n",
        "            f1 = 0\n",
        "        else:\n",
        "            precision = len(common_tokens) / len(pred_tokens)\n",
        "            recall = len(common_tokens) / len(true_tokens)\n",
        "            f1 = 2 * (precision * recall) / (precision + recall)\n",
        "        f1_scores.append(f1)\n",
        "\n",
        "    exact_match_rate = np.mean(exact_match)\n",
        "    avg_f1_score = np.mean(f1_scores)\n",
        "\n",
        "    accuracy = accuracy_score(start_true, start_preds) * 0.5 + accuracy_score(end_true, end_preds) * 0.5\n",
        "\n",
        "    print(\"Exact Match: {:.4f}\".format(exact_match_rate))\n",
        "    print(\"F1 Score: {:.4f}\".format(avg_f1_score))\n",
        "    print(\"Accuracy: {:.4f}\".format(accuracy))\n",
        "\n",
        "    return avg_f1_score, accuracy, exact_match_rate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "tD5XlDsR8eHZ",
      "metadata": {
        "id": "tD5XlDsR8eHZ"
      },
      "outputs": [],
      "source": [
        "reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=2, min_lr=1e-7)\n",
        "early_stopping = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)\n",
        "checkpoint = ModelCheckpoint('best_model.h5', monitor='val_loss', save_best_only=True, mode='min')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "wqPUtkur8gRu",
      "metadata": {
        "id": "wqPUtkur8gRu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "953b470e-7ca5-471a-a64f-6fc382749adc"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/keras/src/engine/training.py:3103: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n",
            "  saving_api.save_model(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "588/588 - 184s - loss: 0.3928 - start_probs_loss: 0.1485 - end_probs_loss: 0.2443 - val_loss: 1.0743 - val_start_probs_loss: 0.4605 - val_end_probs_loss: 0.6137 - lr: 3.0000e-05 - 184s/epoch - 313ms/step\n",
            "Epoch 2/5\n",
            "588/588 - 113s - loss: 0.2472 - start_probs_loss: 0.0907 - end_probs_loss: 0.1566 - val_loss: 1.1292 - val_start_probs_loss: 0.4527 - val_end_probs_loss: 0.6764 - lr: 3.0000e-05 - 113s/epoch - 191ms/step\n",
            "Epoch 3/5\n",
            "588/588 - 111s - loss: 0.1999 - start_probs_loss: 0.0773 - end_probs_loss: 0.1226 - val_loss: 1.1154 - val_start_probs_loss: 0.4599 - val_end_probs_loss: 0.6555 - lr: 3.0000e-05 - 111s/epoch - 188ms/step\n",
            "Epoch 4/5\n",
            "588/588 - 117s - loss: 0.0990 - start_probs_loss: 0.0373 - end_probs_loss: 0.0618 - val_loss: 1.1775 - val_start_probs_loss: 0.5003 - val_end_probs_loss: 0.6772 - lr: 6.0000e-06 - 117s/epoch - 198ms/step\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.src.callbacks.History at 0x7a2784e0ebf0>"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ],
      "source": [
        "model.fit(\n",
        "    x_train,\n",
        "    y_train,\n",
        "    validation_data=(x_test, y_test),\n",
        "    epochs=5,\n",
        "    verbose=2,\n",
        "    batch_size=32,\n",
        "    callbacks=[reduce_lr, early_stopping, checkpoint],\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "w5mksYmf-oH7",
      "metadata": {
        "id": "w5mksYmf-oH7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "db60ec7a-1b3e-4211-c89f-cf9f843ced74"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "66/66 [==============================] - 10s 79ms/step\n",
            "Exact Match: 0.8509\n",
            "F1 Score: 0.9233\n",
            "Accuracy: 0.8978\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.9232514293660464, 0.8978084802286803, 0.8508813720819438)"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ],
      "source": [
        "evaluate(model, x_test, y_test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "iIUj1Kkf3rJQ",
      "metadata": {
        "id": "iIUj1Kkf3rJQ"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "z4pfuDoT3rMW",
      "metadata": {
        "id": "z4pfuDoT3rMW"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "Y2Hf213j3rPf",
      "metadata": {
        "id": "Y2Hf213j3rPf"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "id": "-aLF6akj4_Yz",
      "metadata": {
        "id": "-aLF6akj4_Yz"
      },
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "TPU",
    "colab": {
      "gpuType": "V28",
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.4"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "a6bc077ec0ce4c748184b2693d2157cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_45266c7574bb41809728467c53e898bb",
              "IPY_MODEL_1b03f0387fc642238127267eca3e64b6",
              "IPY_MODEL_ba494f962caf4095abce6b4b99db84a5"
            ],
            "layout": "IPY_MODEL_24cece3b178e4e76a49e37e9574617cc"
          }
        },
        "45266c7574bb41809728467c53e898bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ecbd74ed588246cd86802f2c56977213",
            "placeholder": "​",
            "style": "IPY_MODEL_eb4dd4c27f0b41ee8eaaa0c71213568a",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "1b03f0387fc642238127267eca3e64b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_47006209f84c40cfb58f7c7664df5fae",
            "max": 60,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e3d676bb6b3d48cf99983ec1345c1971",
            "value": 60
          }
        },
        "ba494f962caf4095abce6b4b99db84a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5e3ffbe0059b44199816bec3395ba892",
            "placeholder": "​",
            "style": "IPY_MODEL_37e83967b3b44f77a39cb56c0732ab39",
            "value": " 60.0/60.0 [00:00&lt;00:00, 5.20kB/s]"
          }
        },
        "24cece3b178e4e76a49e37e9574617cc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ecbd74ed588246cd86802f2c56977213": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eb4dd4c27f0b41ee8eaaa0c71213568a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "47006209f84c40cfb58f7c7664df5fae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e3d676bb6b3d48cf99983ec1345c1971": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5e3ffbe0059b44199816bec3395ba892": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "37e83967b3b44f77a39cb56c0732ab39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d5f1ba81fc994807b38bb7fb6d18e9e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d3b50cae04de4034a6de0b9dd45fd223",
              "IPY_MODEL_5ab7e310c3cd438ca514b1edb8f00652",
              "IPY_MODEL_ae219644199f4b6c9ee0d9f46b3cadac"
            ],
            "layout": "IPY_MODEL_af0ee1b64ab54726ba7ce8891242e9eb"
          }
        },
        "d3b50cae04de4034a6de0b9dd45fd223": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_18f02905fd594a73a015f0d743c60842",
            "placeholder": "​",
            "style": "IPY_MODEL_6458181a7e7a4268877d9b5db1b94c1e",
            "value": "vocab.txt: 100%"
          }
        },
        "5ab7e310c3cd438ca514b1edb8f00652": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a20aaa404557415ca1cc0eff69322fc5",
            "max": 251003,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_aad9e0642bfd40ac93edb67357ddbdbf",
            "value": 251003
          }
        },
        "ae219644199f4b6c9ee0d9f46b3cadac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cd3afd7162b7442f815f73d8524001e6",
            "placeholder": "​",
            "style": "IPY_MODEL_5a227dc6760345e0bc3701998d112087",
            "value": " 251k/251k [00:00&lt;00:00, 5.35MB/s]"
          }
        },
        "af0ee1b64ab54726ba7ce8891242e9eb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "18f02905fd594a73a015f0d743c60842": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6458181a7e7a4268877d9b5db1b94c1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a20aaa404557415ca1cc0eff69322fc5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "aad9e0642bfd40ac93edb67357ddbdbf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cd3afd7162b7442f815f73d8524001e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5a227dc6760345e0bc3701998d112087": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e612c186ee3444e48c72c41c18985b8d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_bb34cb37a8a744da863024ef3a3fe352",
              "IPY_MODEL_72cbf0b40224431885098cba5bad919f",
              "IPY_MODEL_abbd87e4902b497593a95b21bdc6f4dd"
            ],
            "layout": "IPY_MODEL_b6e9a060731242368522020c231b0523"
          }
        },
        "bb34cb37a8a744da863024ef3a3fe352": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_acd8c068f0f24be38e7b9594c0a35ca5",
            "placeholder": "​",
            "style": "IPY_MODEL_11dcb8c436064460b5758c11a4f02669",
            "value": "config.json: 100%"
          }
        },
        "72cbf0b40224431885098cba5bad919f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_38befdb42fe849cda31c43dc43ed91db",
            "max": 385,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e1f79734e2f44ca987311b9905053689",
            "value": 385
          }
        },
        "abbd87e4902b497593a95b21bdc6f4dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a103326c134742a8816e3fe3190dd083",
            "placeholder": "​",
            "style": "IPY_MODEL_9c1dd1a15bd24307a528b57120821f20",
            "value": " 385/385 [00:00&lt;00:00, 34.8kB/s]"
          }
        },
        "b6e9a060731242368522020c231b0523": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "acd8c068f0f24be38e7b9594c0a35ca5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "11dcb8c436064460b5758c11a4f02669": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "38befdb42fe849cda31c43dc43ed91db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e1f79734e2f44ca987311b9905053689": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a103326c134742a8816e3fe3190dd083": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9c1dd1a15bd24307a528b57120821f20": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9309f9df992546a4b5f2c3f0188515ff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_bbd1356aa1b744898d00a22dc993a89d",
              "IPY_MODEL_c75f016de96e4fa588465209b9f6bd47",
              "IPY_MODEL_b63db853872a4df88ba699ef3cb16b52"
            ],
            "layout": "IPY_MODEL_6b45d230641048948b4cfb0b75e78848"
          }
        },
        "bbd1356aa1b744898d00a22dc993a89d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2e41c941dcf64e088d6cbc06de792261",
            "placeholder": "​",
            "style": "IPY_MODEL_eab3e82cfbdf451bbb945c1622a4840c",
            "value": "tf_model.h5: 100%"
          }
        },
        "c75f016de96e4fa588465209b9f6bd47": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_adad6c1e239b49688008123bbf02d8d9",
            "max": 545150592,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2db148ee3f564170acda71169aabba34",
            "value": 545150592
          }
        },
        "b63db853872a4df88ba699ef3cb16b52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_205af49863744d32b104618bbe07ddfd",
            "placeholder": "​",
            "style": "IPY_MODEL_e42536e8eaeb40d6adb281bfc99dd79b",
            "value": " 545M/545M [00:03&lt;00:00, 198MB/s]"
          }
        },
        "6b45d230641048948b4cfb0b75e78848": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2e41c941dcf64e088d6cbc06de792261": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eab3e82cfbdf451bbb945c1622a4840c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "adad6c1e239b49688008123bbf02d8d9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2db148ee3f564170acda71169aabba34": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "205af49863744d32b104618bbe07ddfd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e42536e8eaeb40d6adb281bfc99dd79b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}