Fine-tuningOverAttentions16.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0f3a70c2-8f5f-4248-9bab-10f48ea1c363",
   "metadata": {},
   "source": [
    "### installation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f50ac50e-1727-4db3-b691-6ca4647abf9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "! pip install datasets transformers rouge-score nltk\n",
    "\n",
    "!git clone https://github.com/xmu-xiaoma666/External-Attention-pytorch.git ext_attns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6e2d1e19-bd47-4bae-bf5d-322afcea5012",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39745406-4976-4762-b166-2fe29fe8d92e",
   "metadata": {},
   "source": [
    "### About dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "53ea756c-8f5c-4258-b80a-47c2109c7c9c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['attention_mask', 'document', 'id', 'input_ids', 'labels', 'summary'],\n",
       "        num_rows: 204045\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['attention_mask', 'document', 'id', 'input_ids', 'labels', 'summary'],\n",
       "        num_rows: 11332\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['attention_mask', 'document', 'id', 'input_ids', 'labels', 'summary'],\n",
       "        num_rows: 11334\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "828b258b-c091-4f2f-9f91-fda068d293f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train: 204045\n",
      "val:   11332\n",
      "test:  11334\n"
     ]
    }
   ],
   "source": [
    "print(f'train: {len(tokenized_datasets[\"train\"])}')\n",
    "print(f'val:   {len(tokenized_datasets[\"validation\"])}')\n",
    "print(f'test:  {len(tokenized_datasets[\"test\"])}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "50ab7ce0-b46d-4ea3-b8f6-205b8651beed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "odict_keys(['attention_mask', 'document', 'id', 'input_ids', 'labels', 'summary'])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tr_max=20000\n",
    "tokenized_datasets['train'][:tr_max"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "59906a6b-a093-4cd3-8dda-553244dc7955",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['attention_mask', 'document', 'id', 'input_ids', 'labels', 'summary'],\n",
       "    num_rows: 204045\n",
       "})"
      ]
     },
     "execution_count": 210,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_datasets['train']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "id": "73094c79-f85d-4f76-9c8f-6ac6317b37ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.01960351883163028"
      ]
     },
     "execution_count": 217,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "4000/204045"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ce481b0-db3d-44c2-a62c-6927c523a9f5",
   "metadata": {},
   "source": [
    "### FT on small dataset "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "596aa51d-6b0b-4c8d-8316-8069c6329766",
   "metadata": {},
   "source": [
    "|i|attention | used|\n",
    "|---|--------| ----|\n",
    "|0  |UFOAttention| ✔|\n",
    "|1  | AFT_FULL   |x|\n",
    "|2  | MUSEAttention|✔ |\n",
    "|3  | EMSA|     x|\n",
    "|4  |SimplifiedScaledDotProductAttention|✔ |\n",
    "|5  |ScaledDotProductAttention|✔ |\n",
    "|6  |ExternalAttention|✔ |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "31a02343-6b97-41c9-89db-01f41a0151c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default\n",
      "Reusing dataset xsum (/root/.cache/huggingface/datasets/xsum/default/1.2.0/4957825a982999fbf80bca0b342793b01b2611e021ef589fb7c6250b3577b499)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c83f479b33d4dc492cf0f6bf53a2f9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at /root/.cache/huggingface/datasets/xsum/default/1.2.0/4957825a982999fbf80bca0b342793b01b2611e021ef589fb7c6250b3577b499/cache-3d31b94c15be5d86.arrow\n",
      "Loading cached processed dataset at /root/.cache/huggingface/datasets/xsum/default/1.2.0/4957825a982999fbf80bca0b342793b01b2611e021ef589fb7c6250b3577b499/cache-2e154596361058e4.arrow\n",
      "Loading cached processed dataset at /root/.cache/huggingface/datasets/xsum/default/1.2.0/4957825a982999fbf80bca0b342793b01b2611e021ef589fb7c6250b3577b499/cache-f7a9e643055dbd7b.arrow\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "ext_path = os.path.abspath('./ext_attns/')\n",
    "import sys\n",
    "sys.path.append(ext_path)\n",
    "# list\n",
    "from model.attention.UFOAttention import *\n",
    "from model.attention.AFT import AFT_FULL\n",
    "from model.attention.MUSEAttention import MUSEAttention\n",
    "from model.attention.EMSA import EMSA\n",
    "from model.attention.SimplifiedSelfAttention import SimplifiedScaledDotProductAttention\n",
    "from model.attention.SelfAttention import ScaledDotProductAttention\n",
    "from model.attention.ExternalAttention import ExternalAttention\n",
    "d_model = 768 \n",
    "n_head  = 12  \n",
    "ufo = UFOAttention(d_model=d_model, d_k=d_model//n_head, d_v=d_model//n_head, h=n_head)\n",
    "aft_full = AFT_FULL(d_model=d_model, n=n_head)\n",
    "ma = MUSEAttention(d_model=d_model, d_k=d_model//n_head, d_v=d_model//n_head, h=n_head)\n",
    "emsa = EMSA(d_model=d_model, d_k=d_model//n_head, d_v=d_model//n_head, h=n_head,H=n_head,W=n_head,ratio=2,apply_transform=True)\n",
    "ssa = SimplifiedScaledDotProductAttention(d_model=d_model, h=n_head)\n",
    "sa = ScaledDotProductAttention(d_model=d_model, d_k=d_model//n_head, d_v=d_model//n_head, h=n_head)\n",
    "ea = ExternalAttention(d_model=d_model,S=8) #v\n",
    "\n",
    "attn_li = [ufo,aft_full,ma,emsa,ssa,sa,ea]\n",
    "\n",
    "import transformers\n",
    "\n",
    "# print(transformers.__version__)\n",
    "\n",
    "model_checkpoint = \"t5-small\"\n",
    "model_checkpoint = 'facebook/bart-base'\n",
    "\n",
    "from datasets import load_dataset, load_metric\n",
    "\n",
    "raw_datasets = load_dataset(\"xsum\")\n",
    "metric = load_metric(\"rouge\")\n",
    "\n",
    "# raw_datasets[\"train\"][0]\n",
    "\n",
    "import datasets\n",
    "\n",
    "from transformers import AutoTokenizer\n",
    "    \n",
    "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)\n",
    "\n",
    "# ?\n",
    "# with tokenizer.as_target_tokenizer():\n",
    "#     print(tokenizer([\"Hello, this one sentence!\", \"This is another sentence.\"]))\n",
    "\n",
    "if model_checkpoint in [\"t5-small\", \"t5-base\", \"t5-larg\", \"t5-3b\", \"t5-11b\"]:\n",
    "    prefix = \"summarize: \"\n",
    "else:\n",
    "    prefix = \"\"\n",
    "\n",
    "max_input_length = 1024\n",
    "max_target_length = 128\n",
    "\n",
    "def preprocess_function(examples):\n",
    "    inputs = [prefix + doc for doc in examples[\"document\"]]\n",
    "    model_inputs = tokenizer(inputs, max_length=max_input_length, truncation=True)\n",
    "\n",
    "    # Setup the tokenizer for targets\n",
    "    with tokenizer.as_target_tokenizer():\n",
    "        labels = tokenizer(examples[\"summary\"], max_length=max_target_length, truncation=True)\n",
    "\n",
    "    model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
    "    return model_inputs\n",
    "\n",
    "tokenized_datasets = raw_datasets.map(preprocess_function, batched=True)\n",
    "\n",
    "from transformers import AutoModelForSeq2SeqLM, DataCollatorForSeq2Seq, Seq2SeqTrainingArguments, Seq2SeqTrainer\n",
    "\n",
    "model = AutoModelForSeq2SeqLM.from_pretrained(model_checkpoint)\n",
    "\n",
    "batch_size = 3 #16\n",
    "\n",
    "data_collator = DataCollatorForSeq2Seq(tokenizer, model=model)\n",
    "\n",
    "import nltk\n",
    "import numpy as np\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)\n",
    "    # Replace -100 in the labels as we can't decode them.\n",
    "    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n",
    "    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n",
    "    \n",
    "    # Rouge expects a newline after each sentence\n",
    "    decoded_preds = [\"\\n\".join(nltk.sent_tokenize(pred.strip())) for pred in decoded_preds]\n",
    "    decoded_labels = [\"\\n\".join(nltk.sent_tokenize(label.strip())) for label in decoded_labels]\n",
    "    \n",
    "    result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)\n",
    "    # Extract a few results\n",
    "    result = {key: value.mid.fmeasure * 100 for key, value in result.items()}\n",
    "    \n",
    "    # Add mean generated length\n",
    "    prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in predictions]\n",
    "    result[\"gen_len\"] = np.mean(prediction_lens)\n",
    "    \n",
    "    return {k: round(v, 4) for k, v in result.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "69f2fe42-e6c3-4d3a-98ac-ba6cd363e9cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached split indices for dataset at /root/.cache/huggingface/datasets/xsum/default/1.2.0/4957825a982999fbf80bca0b342793b01b2611e021ef589fb7c6250b3577b499/cache-914b7a1a143d71db.arrow and /root/.cache/huggingface/datasets/xsum/default/1.2.0/4957825a982999fbf80bca0b342793b01b2611e021ef589fb7c6250b3577b499/cache-37ec39fca056d44f.arrow\n"
     ]
    }
   ],
   "source": [
    "trvaltest = tokenized_datasets['train'].train_test_split(test_size=0.004,train_size=0.005) # 800,1080\n",
    "# trvaltest = tokenized_datasets['train'].train_test_split(test_size=0.004,train_size=0.01) # 800,2080\n",
    "# trvaltest = tokenized_datasets['train'].train_test_split(test_size=0.004,train_size=0.02) # 800,4080\n",
    "tr = trvaltest['train']\n",
    "valtest = trvaltest['test'].train_test_split(test_size=0.5,train_size=0.5) # 208, 208\n",
    "# valtest = trvaltest['test'].train_test_split(test_size=0.5,train_size=0.5) # 408, 408\n",
    "# valtest = trvaltest['test'].train_test_split(test_size=0.05,train_size=0.05)\n",
    "val = valtest['train']\n",
    "te = valtest['test']\n",
    "\n",
    "del trvaltest\n",
    "del valtest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "51b1d1b7-80ce-4576-a982-9130d9e2bea7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "UFOAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 16\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 64\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 240\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='240' max='240' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [240/240 09:03, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.497597</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>9.946655</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.924000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>9.249656</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>No log</td>\n",
       "      <td>8.560993</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>No log</td>\n",
       "      <td>8.102303</td>\n",
       "      <td>0.035000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.035000</td>\n",
       "      <td>0.035000</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.837111</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>9.007400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.673697</td>\n",
       "      <td>0.035000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.035000</td>\n",
       "      <td>0.035000</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.506831</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.995100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.395956</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.980400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.315826</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.274271</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.257858</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.235324</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.226910</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.225652</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-16\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-16/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-16/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-16/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-16/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-80] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-32\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-32/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-32/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-32/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-32/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-96] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-48\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-48/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-48/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-48/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-48/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-112] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-64\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-64/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-64/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-64/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-64/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-16] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-80\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-80/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-80/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-80/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-80/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-32] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-96\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-96/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-96/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-96/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-96/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-48] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-112\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-112/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-112/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-112/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-112/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-64] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-128\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-128/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-128/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-128/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-128/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-80] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-144\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-144/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-144/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-144/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-144/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-96] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-160\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-160/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-160/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-160/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-160/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-112] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-176\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-176/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-176/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-176/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-176/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-128] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-192\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-192/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-192/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-192/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-192/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-144] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-208\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-208/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-208/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-208/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-208/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-160] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-224\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-224/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-224/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-224/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-224/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-176] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/UFOAttention/checkpoint-240\n",
      "Configuration saved in ./output/UFOAttention/checkpoint-240/config.json\n",
      "Model weights saved in ./output/UFOAttention/checkpoint-240/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/UFOAttention/checkpoint-240/tokenizer_config.json\n",
      "Special tokens file saved in ./output/UFOAttention/checkpoint-240/special_tokens_map.json\n",
      "Deleting older checkpoint [output/UFOAttention/checkpoint-192] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/UFOAttention/checkpoint-240 (score: 7.225651741027832).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 64\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='7' max='7' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [7/7 00:10]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AFT_FULL(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (sigmoid): Sigmoid()\n",
      ")\n",
      "MUSEAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      "  (conv1): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Identity()\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (conv3): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Conv1d(768, 768, kernel_size=(3,), stride=(1,), padding=(1,), groups=768)\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (conv5): Depth_Pointwise_Conv1d(\n",
      "    (depth_conv): Conv1d(768, 768, kernel_size=(5,), stride=(1,), padding=(2,), groups=768)\n",
      "    (pointwise_conv): Conv1d(768, 768, kernel_size=(1,), stride=(1,))\n",
      "  )\n",
      "  (softmax): Softmax(dim=-1)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 16\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 64\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 240\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='240' max='240' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [240/240 10:14, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.560415</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>9.881172</td>\n",
       "      <td>7.428700</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.430900</td>\n",
       "      <td>7.432300</td>\n",
       "      <td>9.887300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>8.841069</td>\n",
       "      <td>7.834900</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.807200</td>\n",
       "      <td>7.328100</td>\n",
       "      <td>10.980400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>No log</td>\n",
       "      <td>8.266951</td>\n",
       "      <td>3.036300</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.043800</td>\n",
       "      <td>3.043700</td>\n",
       "      <td>10.527000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.804331</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.313700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.505759</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.371160</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.943600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.233803</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.135758</td>\n",
       "      <td>0.541500</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.521800</td>\n",
       "      <td>0.523800</td>\n",
       "      <td>6.926500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.133358</td>\n",
       "      <td>2.664600</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.641400</td>\n",
       "      <td>2.637200</td>\n",
       "      <td>8.264700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.035651</td>\n",
       "      <td>6.636400</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.325300</td>\n",
       "      <td>6.391900</td>\n",
       "      <td>14.161800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.021664</td>\n",
       "      <td>7.117800</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.869600</td>\n",
       "      <td>6.940000</td>\n",
       "      <td>16.002500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.011223</td>\n",
       "      <td>6.558100</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.305100</td>\n",
       "      <td>6.318100</td>\n",
       "      <td>13.779400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.998502</td>\n",
       "      <td>7.475300</td>\n",
       "      <td>0.016300</td>\n",
       "      <td>7.198300</td>\n",
       "      <td>7.220500</td>\n",
       "      <td>16.590700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.000281</td>\n",
       "      <td>7.372300</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.122300</td>\n",
       "      <td>7.142200</td>\n",
       "      <td>16.723000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-16\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-16/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-16/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-16/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-16/special_tokens_map.json\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-32\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-32/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-32/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-32/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-32/special_tokens_map.json\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-48\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-48/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-48/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-48/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-48/special_tokens_map.json\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-64\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-64/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-64/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-64/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-64/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-16] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-80\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-80/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-80/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-80/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-80/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-32] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-96\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-96/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-96/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-96/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-96/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-48] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-112\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-112/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-112/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-112/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-112/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-64] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-128\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-128/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-128/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-128/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-128/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-80] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-144\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-144/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-144/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-144/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-144/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-96] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-160\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-160/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-160/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-160/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-160/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-112] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-176\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-176/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-176/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-176/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-176/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-128] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-192\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-192/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-192/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-192/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-192/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-144] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-208\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-208/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-208/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-208/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-208/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-160] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-224\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-224/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-224/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-224/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-224/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-176] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/MUSEAttention/checkpoint-240\n",
      "Configuration saved in ./output/MUSEAttention/checkpoint-240/config.json\n",
      "Model weights saved in ./output/MUSEAttention/checkpoint-240/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/MUSEAttention/checkpoint-240/tokenizer_config.json\n",
      "Special tokens file saved in ./output/MUSEAttention/checkpoint-240/special_tokens_map.json\n",
      "Deleting older checkpoint [output/MUSEAttention/checkpoint-192] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/MUSEAttention/checkpoint-224 (score: 6.998502254486084).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 64\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='7' max='7' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [7/7 00:17]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EMSA(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      "  (sr): Sequential()\n",
      "  (sr_conv): Conv2d(768, 768, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=768)\n",
      "  (sr_ln): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (transform): Sequential(\n",
      "    (conv): Conv2d(12, 12, kernel_size=(1, 1), stride=(1, 1))\n",
      "    (softmax): Softmax(dim=-1)\n",
      "    (in): InstanceNorm2d(12, eps=1e-05, momentum=0.1, affine=False, track_running_stats=False)\n",
      "  )\n",
      ")\n",
      "SimplifiedScaledDotProductAttention(\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 16\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 64\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 240\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='240' max='240' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [240/240 10:40, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.271140</td>\n",
       "      <td>0.027200</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.027200</td>\n",
       "      <td>0.027200</td>\n",
       "      <td>5.024500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>9.536392</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.992600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>8.593362</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.031900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>No log</td>\n",
       "      <td>8.007288</td>\n",
       "      <td>15.054700</td>\n",
       "      <td>0.804700</td>\n",
       "      <td>13.385900</td>\n",
       "      <td>13.378800</td>\n",
       "      <td>14.681400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.590259</td>\n",
       "      <td>14.031300</td>\n",
       "      <td>0.329800</td>\n",
       "      <td>12.936400</td>\n",
       "      <td>12.952100</td>\n",
       "      <td>17.573500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.360931</td>\n",
       "      <td>15.359800</td>\n",
       "      <td>0.666300</td>\n",
       "      <td>13.790700</td>\n",
       "      <td>13.768100</td>\n",
       "      <td>15.928900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.235737</td>\n",
       "      <td>14.590900</td>\n",
       "      <td>0.445000</td>\n",
       "      <td>13.490100</td>\n",
       "      <td>13.519300</td>\n",
       "      <td>16.196100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.118075</td>\n",
       "      <td>13.946500</td>\n",
       "      <td>0.662800</td>\n",
       "      <td>12.994700</td>\n",
       "      <td>13.005800</td>\n",
       "      <td>17.451000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.034989</td>\n",
       "      <td>12.157000</td>\n",
       "      <td>0.548900</td>\n",
       "      <td>10.950700</td>\n",
       "      <td>11.067900</td>\n",
       "      <td>17.232800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.019287</td>\n",
       "      <td>9.261100</td>\n",
       "      <td>0.483500</td>\n",
       "      <td>7.944700</td>\n",
       "      <td>8.099900</td>\n",
       "      <td>14.600500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.954148</td>\n",
       "      <td>8.193100</td>\n",
       "      <td>0.281800</td>\n",
       "      <td>6.847800</td>\n",
       "      <td>7.149700</td>\n",
       "      <td>16.357800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.953715</td>\n",
       "      <td>7.796700</td>\n",
       "      <td>0.214800</td>\n",
       "      <td>6.706600</td>\n",
       "      <td>6.959900</td>\n",
       "      <td>15.777000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.940246</td>\n",
       "      <td>6.331100</td>\n",
       "      <td>0.149600</td>\n",
       "      <td>5.754500</td>\n",
       "      <td>5.788300</td>\n",
       "      <td>15.235300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.927143</td>\n",
       "      <td>6.727400</td>\n",
       "      <td>0.190700</td>\n",
       "      <td>5.952300</td>\n",
       "      <td>6.064500</td>\n",
       "      <td>15.583300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>No log</td>\n",
       "      <td>6.928213</td>\n",
       "      <td>6.984700</td>\n",
       "      <td>0.239100</td>\n",
       "      <td>6.172900</td>\n",
       "      <td>6.222100</td>\n",
       "      <td>15.948500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-16\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-16/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-16/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-16/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-16/special_tokens_map.json\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-32\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-32/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-32/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-32/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-32/special_tokens_map.json\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-48\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-48/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-48/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-48/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-48/special_tokens_map.json\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-64\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-64/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-64/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-64/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-64/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-16] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-80\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-80/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-80/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-80/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-80/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-32] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-96\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-96/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-96/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-96/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-96/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-48] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-112\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-112/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-112/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-112/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-112/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-64] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-128\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-128/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-128/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-128/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-128/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-80] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-144\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-144/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-144/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-144/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-144/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-96] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-160\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-160/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-160/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-160/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-160/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-112] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-176\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-176/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-176/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-176/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-176/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-128] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-192\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-192/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-192/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-192/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-192/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-144] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-208\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-208/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-208/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-208/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-208/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-160] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-224\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-224/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-224/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-224/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-224/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-176] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "/opt/conda/lib/python3.8/site-packages/transformers/trainer.py:1355: FutureWarning: Non-finite norm encountered in torch.nn.utils.clip_grad_norm_; continuing anyway. Note that the default behavior will change in a future release to error out if a non-finite total norm is encountered. At that point, setting error_if_nonfinite=false will be required to retain the old behavior.\n",
      "  nn.utils.clip_grad_norm_(\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/SimplifiedScaledDotProductAttention/checkpoint-240\n",
      "Configuration saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-240/config.json\n",
      "Model weights saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-240/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-240/tokenizer_config.json\n",
      "Special tokens file saved in ./output/SimplifiedScaledDotProductAttention/checkpoint-240/special_tokens_map.json\n",
      "Deleting older checkpoint [output/SimplifiedScaledDotProductAttention/checkpoint-192] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/SimplifiedScaledDotProductAttention/checkpoint-224 (score: 6.927143096923828).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 64\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='7' max='7' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [7/7 00:17]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ScaledDotProductAttention(\n",
      "  (fc_q): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_k): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_v): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (fc_o): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (dropout): Dropout(p=0.1, inplace=False)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 16\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 64\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 240\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='240' max='240' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [240/240 09:30, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.702778</td>\n",
       "      <td>7.388900</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.375700</td>\n",
       "      <td>7.389300</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.142927</td>\n",
       "      <td>7.636400</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.622500</td>\n",
       "      <td>7.634200</td>\n",
       "      <td>4.002500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>9.102008</td>\n",
       "      <td>7.825100</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.799500</td>\n",
       "      <td>7.817400</td>\n",
       "      <td>5.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>No log</td>\n",
       "      <td>8.458250</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>No log</td>\n",
       "      <td>8.026911</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.821119</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.803900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.709074</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.992600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.665245</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.772100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.644638</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.181400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.638343</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>11.323500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.634840</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.666700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.636078</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.634418</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>10.328400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.636116</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.061300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>No log</td>\n",
       "      <td>7.636063</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>11.227900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-16\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-16/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-16/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-16/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-16/special_tokens_map.json\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-32\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-32/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-32/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-32/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-32/special_tokens_map.json\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-48\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-48/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-48/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-48/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-48/special_tokens_map.json\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-64\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-64/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-64/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-64/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-64/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-16] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-80\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-80/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-80/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-80/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-80/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-32] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-96\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-96/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-96/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-96/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-96/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-48] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-112\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-112/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-112/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-112/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-112/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-64] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-128\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-128/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-128/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-128/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-128/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-80] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-144\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-144/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-144/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-144/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-144/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-96] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-160\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-160/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-160/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-160/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-160/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-112] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-176\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-176/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-176/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-176/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-176/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-128] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-192\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-192/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-192/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-192/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-192/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-144] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-208\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-208/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-208/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-208/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-208/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-160] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-224\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-224/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-224/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-224/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-224/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-176] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ScaledDotProductAttention/checkpoint-240\n",
      "Configuration saved in ./output/ScaledDotProductAttention/checkpoint-240/config.json\n",
      "Model weights saved in ./output/ScaledDotProductAttention/checkpoint-240/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ScaledDotProductAttention/checkpoint-240/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ScaledDotProductAttention/checkpoint-240/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ScaledDotProductAttention/checkpoint-192] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/ScaledDotProductAttention/checkpoint-208 (score: 7.634418487548828).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 64\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='7' max='7' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [7/7 00:15]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ExternalAttention(\n",
      "  (mk): Linear(in_features=768, out_features=8, bias=False)\n",
      "  (mv): Linear(in_features=8, out_features=768, bias=False)\n",
      "  (softmax): Softmax(dim=1)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 16\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 64\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 240\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='240' max='240' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [240/240 09:14, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.824655</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.824228</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.823728</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.823125</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.822416</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.821635</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.820818</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.819997</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.819201</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.818477</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.817848</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.817344</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.816971</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.816741</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.816659</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-16\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-16/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-16/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-16/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-16/special_tokens_map.json\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-32\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-32/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-32/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-32/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-32/special_tokens_map.json\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-48\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-48/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-48/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-48/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-48/special_tokens_map.json\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-64\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-64/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-64/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-64/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-64/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-16] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-80\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-80/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-80/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-80/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-80/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-32] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-96\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-96/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-96/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-96/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-96/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-48] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-112\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-112/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-112/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-112/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-112/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-64] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-128\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-128/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-128/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-128/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-128/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-80] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-144\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-144/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-144/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-144/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-144/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-96] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-160\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-160/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-160/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-160/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-160/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-112] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-176\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-176/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-176/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-176/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-176/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-128] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-192\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-192/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-192/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-192/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-192/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-144] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-208\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-208/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-208/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-208/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-208/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-160] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-224\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-224/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-224/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-224/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-224/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-176] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/ExternalAttention/checkpoint-240\n",
      "Configuration saved in ./output/ExternalAttention/checkpoint-240/config.json\n",
      "Model weights saved in ./output/ExternalAttention/checkpoint-240/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/ExternalAttention/checkpoint-240/tokenizer_config.json\n",
      "Special tokens file saved in ./output/ExternalAttention/checkpoint-240/special_tokens_map.json\n",
      "Deleting older checkpoint [output/ExternalAttention/checkpoint-192] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/ExternalAttention/checkpoint-240 (score: 10.816658973693848).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 64\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='7' max='7' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [7/7 00:11]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from torch import nn\n",
    "batch_size = 16\n",
    "evals = {}\n",
    "for i,attn in enumerate(attn_li[:]):\n",
    "    not_allowed = ('AFT_FULL','EMSA')\n",
    "    print(attn)\n",
    "    if attn.__str__().startswith(not_allowed):\n",
    "        continue\n",
    "    \n",
    "    \n",
    "    import os\n",
    "    model_name = attn.__str__().split('(')[0]\n",
    "    # model_name = model_checkpoint.split(\"/\")[-1] + model_name\n",
    "    saved_path = os.path.join('./output',model_name)\n",
    "    \n",
    "    args = Seq2SeqTrainingArguments(\n",
    "        saved_path,\n",
    "        evaluation_strategy = \"epoch\",\n",
    "        learning_rate=2e-5,\n",
    "        per_device_train_batch_size=batch_size,\n",
    "        per_device_eval_batch_size=batch_size,\n",
    "        weight_decay=0.01,\n",
    "        save_total_limit=3,\n",
    "        num_train_epochs=15,\n",
    "        predict_with_generate=True,\n",
    "        fp16=True,\n",
    "        save_strategy='epoch',\n",
    "        load_best_model_at_end=True,\n",
    "    #     push_to_hub=True,\n",
    "    )\n",
    "\n",
    "    class tel2lin1(nn.Module):\n",
    "        def __init__(self):\n",
    "            super(tel2lin1,self).__init__()\n",
    "            # self.tel = nn.TransformerEncoderLayer(d_model=768,nhead=3,batch_first=True)\n",
    "            self.tel = attn\n",
    "            self.lin = nn.Linear(in_features=768,out_features=50265, bias=False)\n",
    "            \n",
    "        def forward(self,x):\n",
    "            if self.tel.__str__().startswith('ExternalAttention'):\n",
    "                x = self.tel(x)\n",
    "            else:\n",
    "                x = self.tel(x,x,x)\n",
    "            x = self.lin(x)\n",
    "            return x\n",
    "    model.lm_head = tel2lin1()\n",
    "\n",
    "    trainer = Seq2SeqTrainer(\n",
    "        model,\n",
    "        args,\n",
    "        train_dataset=tr,\n",
    "        eval_dataset=val,\n",
    "        data_collator=data_collator,\n",
    "        tokenizer=tokenizer,\n",
    "        compute_metrics=compute_metrics\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    \n",
    "    evals[f'{model_name}'] = trainer.evaluate(eval_dataset=te)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "65ff5e70-5744-4ddd-9546-ce1d824ec62e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(\"attns.pickle\",\"wb\") as fw:\n",
    "    pickle.dump(evals, fw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "3412372c-ec9f-4bac-b8d6-5beaa50b9bbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('attns.pickle','rb') as fw:\n",
    "    data = pickle.load(fw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "de975d56-0826-4bc2-9f24-9eaf160f868f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'UFOAttention': {'eval_loss': 7.220878601074219,\n",
       "  'eval_rouge1': 0.0,\n",
       "  'eval_rouge2': 0.0,\n",
       "  'eval_rougeL': 0.0,\n",
       "  'eval_rougeLsum': 0.0,\n",
       "  'eval_gen_len': 5.0,\n",
       "  'eval_runtime': 12.9911,\n",
       "  'eval_samples_per_second': 31.483,\n",
       "  'eval_steps_per_second': 0.539,\n",
       "  'epoch': 15.0},\n",
       " 'MUSEAttention': {'eval_loss': 6.988565921783447,\n",
       "  'eval_rouge1': 8.2008,\n",
       "  'eval_rouge2': 0.0585,\n",
       "  'eval_rougeL': 7.7756,\n",
       "  'eval_rougeLsum': 7.7983,\n",
       "  'eval_gen_len': 16.4719,\n",
       "  'eval_runtime': 22.3288,\n",
       "  'eval_samples_per_second': 18.317,\n",
       "  'eval_steps_per_second': 0.313,\n",
       "  'epoch': 15.0},\n",
       " 'SimplifiedScaledDotProductAttention': {'eval_loss': 6.916492938995361,\n",
       "  'eval_rouge1': 7.1489,\n",
       "  'eval_rouge2': 0.1581,\n",
       "  'eval_rougeL': 6.4239,\n",
       "  'eval_rougeLsum': 6.5806,\n",
       "  'eval_gen_len': 15.8093,\n",
       "  'eval_runtime': 21.9655,\n",
       "  'eval_samples_per_second': 18.62,\n",
       "  'eval_steps_per_second': 0.319,\n",
       "  'epoch': 15.0},\n",
       " 'ScaledDotProductAttention': {'eval_loss': 7.626367092132568,\n",
       "  'eval_rouge1': 0.0,\n",
       "  'eval_rouge2': 0.0,\n",
       "  'eval_rougeL': 0.0,\n",
       "  'eval_rougeLsum': 0.0,\n",
       "  'eval_gen_len': 10.2812,\n",
       "  'eval_runtime': 19.9632,\n",
       "  'eval_samples_per_second': 20.488,\n",
       "  'eval_steps_per_second': 0.351,\n",
       "  'epoch': 15.0},\n",
       " 'ExternalAttention': {'eval_loss': 10.816548347473145,\n",
       "  'eval_rouge1': 8.07,\n",
       "  'eval_rouge2': 0.0,\n",
       "  'eval_rougeL': 8.096,\n",
       "  'eval_rougeLsum': 7.8017,\n",
       "  'eval_gen_len': 8.0,\n",
       "  'eval_runtime': 15.1313,\n",
       "  'eval_samples_per_second': 27.03,\n",
       "  'eval_steps_per_second': 0.463,\n",
       "  'epoch': 15.0},\n",
       " 'basic': {'eval_loss': 10.795363426208496,\n",
       "  'eval_rouge1': 8.07,\n",
       "  'eval_rouge2': 0.0,\n",
       "  'eval_rougeL': 8.096,\n",
       "  'eval_rougeLsum': 7.8017,\n",
       "  'eval_gen_len': 8.0,\n",
       "  'eval_runtime': 14.9132,\n",
       "  'eval_samples_per_second': 27.425,\n",
       "  'eval_steps_per_second': 0.469,\n",
       "  'epoch': 15.0}}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1f8e1593-3a0b-4cf2-97f7-49d21c7a645c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'UFOAttention': {'eval_loss': 7.220878601074219,\n",
       "  'eval_rouge1': 0.0,\n",
       "  'eval_rouge2': 0.0,\n",
       "  'eval_rougeL': 0.0,\n",
       "  'eval_rougeLsum': 0.0,\n",
       "  'eval_gen_len': 5.0,\n",
       "  'eval_runtime': 12.9911,\n",
       "  'eval_samples_per_second': 31.483,\n",
       "  'eval_steps_per_second': 0.539,\n",
       "  'epoch': 15.0},\n",
       " 'MUSEAttention': {'eval_loss': 6.988565921783447,\n",
       "  'eval_rouge1': 8.2008,\n",
       "  'eval_rouge2': 0.0585,\n",
       "  'eval_rougeL': 7.7756,\n",
       "  'eval_rougeLsum': 7.7983,\n",
       "  'eval_gen_len': 16.4719,\n",
       "  'eval_runtime': 22.3288,\n",
       "  'eval_samples_per_second': 18.317,\n",
       "  'eval_steps_per_second': 0.313,\n",
       "  'epoch': 15.0},\n",
       " 'SimplifiedScaledDotProductAttention': {'eval_loss': 6.916492938995361,\n",
       "  'eval_rouge1': 7.1489,\n",
       "  'eval_rouge2': 0.1581,\n",
       "  'eval_rougeL': 6.4239,\n",
       "  'eval_rougeLsum': 6.5806,\n",
       "  'eval_gen_len': 15.8093,\n",
       "  'eval_runtime': 21.9655,\n",
       "  'eval_samples_per_second': 18.62,\n",
       "  'eval_steps_per_second': 0.319,\n",
       "  'epoch': 15.0},\n",
       " 'ScaledDotProductAttention': {'eval_loss': 7.626367092132568,\n",
       "  'eval_rouge1': 0.0,\n",
       "  'eval_rouge2': 0.0,\n",
       "  'eval_rougeL': 0.0,\n",
       "  'eval_rougeLsum': 0.0,\n",
       "  'eval_gen_len': 10.2812,\n",
       "  'eval_runtime': 19.9632,\n",
       "  'eval_samples_per_second': 20.488,\n",
       "  'eval_steps_per_second': 0.351,\n",
       "  'epoch': 15.0},\n",
       " 'ExternalAttention': {'eval_loss': 10.816548347473145,\n",
       "  'eval_rouge1': 8.07,\n",
       "  'eval_rouge2': 0.0,\n",
       "  'eval_rougeL': 8.096,\n",
       "  'eval_rougeLsum': 7.8017,\n",
       "  'eval_gen_len': 8.0,\n",
       "  'eval_runtime': 15.1313,\n",
       "  'eval_samples_per_second': 27.03,\n",
       "  'eval_steps_per_second': 0.463,\n",
       "  'epoch': 15.0},\n",
       " 'basic': {'eval_loss': 10.795363426208496,\n",
       "  'eval_rouge1': 8.07,\n",
       "  'eval_rouge2': 0.0,\n",
       "  'eval_rougeL': 8.096,\n",
       "  'eval_rougeLsum': 7.8017,\n",
       "  'eval_gen_len': 8.0,\n",
       "  'eval_runtime': 14.9132,\n",
       "  'eval_samples_per_second': 27.425,\n",
       "  'eval_steps_per_second': 0.469,\n",
       "  'epoch': 15.0}}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "701c6582-8ed1-4b3d-be42-0f4c12d5f24e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n",
      "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
      "Using amp fp16 backend\n",
      "The following columns in the training set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running training *****\n",
      "  Num examples = 1020\n",
      "  Num Epochs = 15\n",
      "  Instantaneous batch size per device = 16\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 64\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 240\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='240' max='240' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [240/240 09:16, Epoch 15/15]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "      <th>Rougelsum</th>\n",
       "      <th>Gen Len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.815116</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.813450</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.811693</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.809827</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.807894</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.805979</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.804138</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.802417</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.800825</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.799348</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.798061</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.796983</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.796213</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.795749</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>No log</td>\n",
       "      <td>10.795593</td>\n",
       "      <td>7.376000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7.352500</td>\n",
       "      <td>7.225000</td>\n",
       "      <td>8.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/basic/checkpoint-16\n",
      "Configuration saved in ./output/basic/checkpoint-16/config.json\n",
      "Model weights saved in ./output/basic/checkpoint-16/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/basic/checkpoint-16/tokenizer_config.json\n",
      "Special tokens file saved in ./output/basic/checkpoint-16/special_tokens_map.json\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/basic/checkpoint-32\n",
      "Configuration saved in ./output/basic/checkpoint-32/config.json\n",
      "Model weights saved in ./output/basic/checkpoint-32/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/basic/checkpoint-32/tokenizer_config.json\n",
      "Special tokens file saved in ./output/basic/checkpoint-32/special_tokens_map.json\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/basic/checkpoint-48\n",
      "Configuration saved in ./output/basic/checkpoint-48/config.json\n",
      "Model weights saved in ./output/basic/checkpoint-48/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/basic/checkpoint-48/tokenizer_config.json\n",
      "Special tokens file saved in ./output/basic/checkpoint-48/special_tokens_map.json\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/basic/checkpoint-64\n",
      "Configuration saved in ./output/basic/checkpoint-64/config.json\n",
      "Model weights saved in ./output/basic/checkpoint-64/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/basic/checkpoint-64/tokenizer_config.json\n",
      "Special tokens file saved in ./output/basic/checkpoint-64/special_tokens_map.json\n",
      "Deleting older checkpoint [output/basic/checkpoint-16] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/basic/checkpoint-80\n",
      "Configuration saved in ./output/basic/checkpoint-80/config.json\n",
      "Model weights saved in ./output/basic/checkpoint-80/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/basic/checkpoint-80/tokenizer_config.json\n",
      "Special tokens file saved in ./output/basic/checkpoint-80/special_tokens_map.json\n",
      "Deleting older checkpoint [output/basic/checkpoint-32] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/basic/checkpoint-96\n",
      "Configuration saved in ./output/basic/checkpoint-96/config.json\n",
      "Model weights saved in ./output/basic/checkpoint-96/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/basic/checkpoint-96/tokenizer_config.json\n",
      "Special tokens file saved in ./output/basic/checkpoint-96/special_tokens_map.json\n",
      "Deleting older checkpoint [output/basic/checkpoint-48] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/basic/checkpoint-112\n",
      "Configuration saved in ./output/basic/checkpoint-112/config.json\n",
      "Model weights saved in ./output/basic/checkpoint-112/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/basic/checkpoint-112/tokenizer_config.json\n",
      "Special tokens file saved in ./output/basic/checkpoint-112/special_tokens_map.json\n",
      "Deleting older checkpoint [output/basic/checkpoint-64] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/basic/checkpoint-128\n",
      "Configuration saved in ./output/basic/checkpoint-128/config.json\n",
      "Model weights saved in ./output/basic/checkpoint-128/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/basic/checkpoint-128/tokenizer_config.json\n",
      "Special tokens file saved in ./output/basic/checkpoint-128/special_tokens_map.json\n",
      "Deleting older checkpoint [output/basic/checkpoint-80] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/basic/checkpoint-144\n",
      "Configuration saved in ./output/basic/checkpoint-144/config.json\n",
      "Model weights saved in ./output/basic/checkpoint-144/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/basic/checkpoint-144/tokenizer_config.json\n",
      "Special tokens file saved in ./output/basic/checkpoint-144/special_tokens_map.json\n",
      "Deleting older checkpoint [output/basic/checkpoint-96] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/basic/checkpoint-160\n",
      "Configuration saved in ./output/basic/checkpoint-160/config.json\n",
      "Model weights saved in ./output/basic/checkpoint-160/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/basic/checkpoint-160/tokenizer_config.json\n",
      "Special tokens file saved in ./output/basic/checkpoint-160/special_tokens_map.json\n",
      "Deleting older checkpoint [output/basic/checkpoint-112] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/basic/checkpoint-176\n",
      "Configuration saved in ./output/basic/checkpoint-176/config.json\n",
      "Model weights saved in ./output/basic/checkpoint-176/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/basic/checkpoint-176/tokenizer_config.json\n",
      "Special tokens file saved in ./output/basic/checkpoint-176/special_tokens_map.json\n",
      "Deleting older checkpoint [output/basic/checkpoint-128] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/basic/checkpoint-192\n",
      "Configuration saved in ./output/basic/checkpoint-192/config.json\n",
      "Model weights saved in ./output/basic/checkpoint-192/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/basic/checkpoint-192/tokenizer_config.json\n",
      "Special tokens file saved in ./output/basic/checkpoint-192/special_tokens_map.json\n",
      "Deleting older checkpoint [output/basic/checkpoint-144] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/basic/checkpoint-208\n",
      "Configuration saved in ./output/basic/checkpoint-208/config.json\n",
      "Model weights saved in ./output/basic/checkpoint-208/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/basic/checkpoint-208/tokenizer_config.json\n",
      "Special tokens file saved in ./output/basic/checkpoint-208/special_tokens_map.json\n",
      "Deleting older checkpoint [output/basic/checkpoint-160] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/basic/checkpoint-224\n",
      "Configuration saved in ./output/basic/checkpoint-224/config.json\n",
      "Model weights saved in ./output/basic/checkpoint-224/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/basic/checkpoint-224/tokenizer_config.json\n",
      "Special tokens file saved in ./output/basic/checkpoint-224/special_tokens_map.json\n",
      "Deleting older checkpoint [output/basic/checkpoint-176] due to args.save_total_limit\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 408\n",
      "  Batch size = 64\n",
      "Saving model checkpoint to ./output/basic/checkpoint-240\n",
      "Configuration saved in ./output/basic/checkpoint-240/config.json\n",
      "Model weights saved in ./output/basic/checkpoint-240/pytorch_model.bin\n",
      "tokenizer config file saved in ./output/basic/checkpoint-240/tokenizer_config.json\n",
      "Special tokens file saved in ./output/basic/checkpoint-240/special_tokens_map.json\n",
      "Deleting older checkpoint [output/basic/checkpoint-192] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./output/basic/checkpoint-240 (score: 10.79559326171875).\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `BartForConditionalGeneration.forward` and have been ignored: document, summary, id.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 409\n",
      "  Batch size = 64\n",
      "/opt/conda/lib/python3.8/site-packages/torch/nn/parallel/_functions.py:68: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='7' max='7' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [7/7 00:11]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from torch import nn\n",
    "batch_size = 16\n",
    "evalbase = {}\n",
    "for i,attn in enumerate(['basic']):\n",
    "    \n",
    "    import os\n",
    "    model_name = attn\n",
    "    # model_name = model_checkpoint.split(\"/\")[-1] + model_name\n",
    "    saved_path = os.path.join('./output',model_name)\n",
    "    \n",
    "    args = Seq2SeqTrainingArguments(\n",
    "        saved_path,\n",
    "        evaluation_strategy = \"epoch\",\n",
    "        learning_rate=2e-5,\n",
    "        per_device_train_batch_size=batch_size,\n",
    "        per_device_eval_batch_size=batch_size,\n",
    "        weight_decay=0.01,\n",
    "        save_total_limit=3,\n",
    "        num_train_epochs=15,\n",
    "        predict_with_generate=True,\n",
    "        fp16=True,\n",
    "        save_strategy='epoch',\n",
    "        load_best_model_at_end=True,\n",
    "    #     push_to_hub=True,\n",
    "    )\n",
    "\n",
    "\n",
    "    trainer = Seq2SeqTrainer(\n",
    "        model,\n",
    "        args,\n",
    "        train_dataset=tr,\n",
    "        eval_dataset=val,\n",
    "        data_collator=data_collator,\n",
    "        tokenizer=tokenizer,\n",
    "        compute_metrics=compute_metrics\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    \n",
    "    evals[f'{model_name}'] = trainer.evaluate(eval_dataset=te)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1e237ff-8fcf-4aa4-b88e-715066b18820",
   "metadata": {},
   "source": [
    "### evals "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e48b9478-0b57-412f-a243-25ba7c43bbae",
   "metadata": {},
   "outputs": [],
   "source": [
    "evals.update(evalbase)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "790f4b99-e761-4657-8b93-c35642dfc3be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eval_loss</th>\n",
       "      <th>eval_rouge1</th>\n",
       "      <th>eval_rouge2</th>\n",
       "      <th>eval_rougeL</th>\n",
       "      <th>eval_rougeLsum</th>\n",
       "      <th>eval_gen_len</th>\n",
       "      <th>eval_runtime</th>\n",
       "      <th>eval_samples_per_second</th>\n",
       "      <th>eval_steps_per_second</th>\n",
       "      <th>epoch</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>UFOAttention</th>\n",
       "      <td>7.220879</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>5.0000</td>\n",
       "      <td>12.9911</td>\n",
       "      <td>31.483</td>\n",
       "      <td>0.539</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MUSEAttention</th>\n",
       "      <td>6.988566</td>\n",
       "      <td>8.2008</td>\n",
       "      <td>0.0585</td>\n",
       "      <td>7.7756</td>\n",
       "      <td>7.7983</td>\n",
       "      <td>16.4719</td>\n",
       "      <td>22.3288</td>\n",
       "      <td>18.317</td>\n",
       "      <td>0.313</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SimplifiedScaledDotProductAttention</th>\n",
       "      <td>6.916493</td>\n",
       "      <td>7.1489</td>\n",
       "      <td>0.1581</td>\n",
       "      <td>6.4239</td>\n",
       "      <td>6.5806</td>\n",
       "      <td>15.8093</td>\n",
       "      <td>21.9655</td>\n",
       "      <td>18.620</td>\n",
       "      <td>0.319</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ScaledDotProductAttention</th>\n",
       "      <td>7.626367</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>10.2812</td>\n",
       "      <td>19.9632</td>\n",
       "      <td>20.488</td>\n",
       "      <td>0.351</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ExternalAttention</th>\n",
       "      <td>10.816548</td>\n",
       "      <td>8.0700</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>8.0960</td>\n",
       "      <td>7.8017</td>\n",
       "      <td>8.0000</td>\n",
       "      <td>15.1313</td>\n",
       "      <td>27.030</td>\n",
       "      <td>0.463</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>basic</th>\n",
       "      <td>10.795363</td>\n",
       "      <td>8.0700</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>8.0960</td>\n",
       "      <td>7.8017</td>\n",
       "      <td>8.0000</td>\n",
       "      <td>14.9132</td>\n",
       "      <td>27.425</td>\n",
       "      <td>0.469</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     eval_loss  eval_rouge1  eval_rouge2  \\\n",
       "UFOAttention                          7.220879       0.0000       0.0000   \n",
       "MUSEAttention                         6.988566       8.2008       0.0585   \n",
       "SimplifiedScaledDotProductAttention   6.916493       7.1489       0.1581   \n",
       "ScaledDotProductAttention             7.626367       0.0000       0.0000   \n",
       "ExternalAttention                    10.816548       8.0700       0.0000   \n",
       "basic                                10.795363       8.0700       0.0000   \n",
       "\n",
       "                                     eval_rougeL  eval_rougeLsum  \\\n",
       "UFOAttention                              0.0000          0.0000   \n",
       "MUSEAttention                             7.7756          7.7983   \n",
       "SimplifiedScaledDotProductAttention       6.4239          6.5806   \n",
       "ScaledDotProductAttention                 0.0000          0.0000   \n",
       "ExternalAttention                         8.0960          7.8017   \n",
       "basic                                     8.0960          7.8017   \n",
       "\n",
       "                                     eval_gen_len  eval_runtime  \\\n",
       "UFOAttention                               5.0000       12.9911   \n",
       "MUSEAttention                             16.4719       22.3288   \n",
       "SimplifiedScaledDotProductAttention       15.8093       21.9655   \n",
       "ScaledDotProductAttention                 10.2812       19.9632   \n",
       "ExternalAttention                          8.0000       15.1313   \n",
       "basic                                      8.0000       14.9132   \n",
       "\n",
       "                                     eval_samples_per_second  \\\n",
       "UFOAttention                                          31.483   \n",
       "MUSEAttention                                         18.317   \n",
       "SimplifiedScaledDotProductAttention                   18.620   \n",
       "ScaledDotProductAttention                             20.488   \n",
       "ExternalAttention                                     27.030   \n",
       "basic                                                 27.425   \n",
       "\n",
       "                                     eval_steps_per_second  epoch  \n",
       "UFOAttention                                         0.539   15.0  \n",
       "MUSEAttention                                        0.313   15.0  \n",
       "SimplifiedScaledDotProductAttention                  0.319   15.0  \n",
       "ScaledDotProductAttention                            0.351   15.0  \n",
       "ExternalAttention                                    0.463   15.0  \n",
       "basic                                                0.469   15.0  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "pd.DataFrame(evals).T"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch1.9",
   "language": "python",
   "name": "torch1.9"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}