sc1-main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-14T17:40:48.683292Z",
     "iopub.status.busy": "2024-11-14T17:40:48.682747Z",
     "iopub.status.idle": "2024-11-14T17:41:00.477379Z",
     "shell.execute_reply": "2024-11-14T17:41:00.476237Z",
     "shell.execute_reply.started": "2024-11-14T17:40:48.683246Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: w3lib in /opt/conda/lib/python3.10/site-packages (2.2.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install w3lib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-14T17:41:00.481697Z",
     "iopub.status.busy": "2024-11-14T17:41:00.481125Z",
     "iopub.status.idle": "2024-11-14T17:41:00.489574Z",
     "shell.execute_reply": "2024-11-14T17:41:00.488340Z",
     "shell.execute_reply.started": "2024-11-14T17:41:00.481638Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import argparse\n",
    "import time\n",
    "import json\n",
    "import torch\n",
    "import random\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "from utils import initNetParams, train_step, val_step\n",
    "from dataset import EurDataset, collate_data\n",
    "from models.transformer import Model\n",
    "from torch.utils.data import DataLoader\n",
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "from models.autoencoders import AE\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-14T17:41:00.491321Z",
     "iopub.status.busy": "2024-11-14T17:41:00.490972Z",
     "iopub.status.idle": "2024-11-14T17:41:00.512700Z",
     "shell.execute_reply": "2024-11-14T17:41:00.511878Z",
     "shell.execute_reply.started": "2024-11-14T17:41:00.491284Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "_StoreAction(option_strings=['--ae_cpath'], dest='ae_cpath', nargs=None, const=None, default='/kaggle/input/datasc/checks/checks', type=<class 'str'>, choices=None, required=False, help=None, metavar=None)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parser = argparse.ArgumentParser()\n",
    "#parser.add_argument('--data-dir', default='data/train_data.pkl', type=str)\n",
    "parser.add_argument('--vocab-file', default='data/vocab.json', type=str)\n",
    "parser.add_argument('--checkpoint-path', default='checkpoints/deepsc-Rayleigh', type=str)\n",
    "parser.add_argument('--channel', default='Rayleigh', type=str, help = 'Please choose AWGN, Rayleigh, and Rician')\n",
    "parser.add_argument('--MAX-LENGTH', default=30, type=int)\n",
    "parser.add_argument('--MIN-LENGTH', default=4, type=int)\n",
    "parser.add_argument('--d-model', default=128, type=int)\n",
    "parser.add_argument('--dff', default=512, type=int)\n",
    "parser.add_argument('--num-layers_deepsc', default=4, type=int)\n",
    "parser.add_argument('--num-layers_ae', default=6, type=int)\n",
    "parser.add_argument('--num-heads', default=8, type=int)\n",
    "parser.add_argument('--batch-size', default=128, type=int)\n",
    "parser.add_argument('--epochs', default=8, type=int)\n",
    "parser.add_argument('--ae_cpath', default=\"data/checks/\", type=str)\n",
    "# parser.add_argument('--ae_cpath', default=\"/kaggle/working/checks/ae\", type = str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-14T17:42:19.535356Z",
     "iopub.status.busy": "2024-11-14T17:42:19.534457Z",
     "iopub.status.idle": "2024-11-14T17:42:19.683140Z",
     "shell.execute_reply": "2024-11-14T17:42:19.682160Z",
     "shell.execute_reply.started": "2024-11-14T17:42:19.535312Z"
    },
    "trusted": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25899\n"
     ]
    }
   ],
   "source": [
    "# # !pip install torchsummary\n",
    "# from torchsummary import summary\n",
    "# from autoencodersc.autoencodersc import AE\n",
    "# vocab = json.load(open('/kaggle/input/datasc/vocab.json', 'rb'))\n",
    "# token_to_idx = vocab['token_to_idx']\n",
    "# num_vocab = len(token_to_idx)\n",
    "# model1 = Model(4, num_vocab, num_vocab,\n",
    "#                     num_vocab, num_vocab, 128, 8,\n",
    "#                     512, 0.1).to(device)\n",
    "# # mod = model1.encoder\n",
    "# # ae = AE().to(device)\n",
    "# # summary(mod, input_size=(26, num_vocab))\n",
    "# print(model1)\n",
    "\n",
    "train_eur= EurDataset('test')\n",
    "print(len(train_eur.data))\n",
    "  \n",
    "# train_iterator = DataLoader(train_eur, batch_size=args.batch_size, num_workers=0,\n",
    "#                             pin_memory=True, collate_fn=collate_data)\n",
    "# pbar = tqdm(train_iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-14T10:06:37.725722Z",
     "iopub.status.busy": "2024-11-14T10:06:37.725121Z",
     "iopub.status.idle": "2024-11-14T10:06:37.734710Z",
     "shell.execute_reply": "2024-11-14T10:06:37.733739Z",
     "shell.execute_reply.started": "2024-11-14T10:06:37.725686Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "def setup_seed(seed): \n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed_all(seed)\n",
    "    np.random.seed(seed)\n",
    "    random.seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "\n",
    "def validate(epoch, args, net, type, ae=None):\n",
    "    test_eur = EurDataset('test')\n",
    "    test_iterator = DataLoader(test_eur, batch_size=args.batch_size, num_workers=0,\n",
    "                                pin_memory=True, collate_fn=collate_data)\n",
    "    net.eval()\n",
    "    pbar = tqdm(test_iterator)\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for sents in pbar:\n",
    "            sents = sents.to(device)\n",
    "            if type != 3: #else\n",
    "                loss = val_step(net, sents, sents, pad_idx,\n",
    "                                 criterion, type)\n",
    "            if type == 3:\n",
    "                loss = val_step(net, sents, sents, pad_idx,\n",
    "                                 criterion, type, ae, args.ae_cpath)\n",
    "\n",
    "            total += loss\n",
    "            pbar.set_description(\n",
    "                'Epoch: {}; Type: VAL; Loss: {:.5f}'.format(\n",
    "                    epoch + 1, loss\n",
    "                )\n",
    "            )\n",
    "\n",
    "    return total/len(test_iterator)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-14T10:06:38.017813Z",
     "iopub.status.busy": "2024-11-14T10:06:38.017101Z",
     "iopub.status.idle": "2024-11-14T10:06:38.025853Z",
     "shell.execute_reply": "2024-11-14T10:06:38.024766Z",
     "shell.execute_reply.started": "2024-11-14T10:06:38.017776Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "def train(epoch, args, net, optimizer, type, ae=None):\n",
    "    print(\"Train Model Type\", type)\n",
    "    train_eur= EurDataset('train')\n",
    "  \n",
    "    train_iterator = DataLoader(train_eur, batch_size=args.batch_size, num_workers=0,\n",
    "                                pin_memory=True, collate_fn=collate_data)\n",
    "    pbar = tqdm(train_iterator)\n",
    "\n",
    "    # noise_std = np.random.uniform(SNR_to_noise(5), SNR_to_noise(10), size=(1))\n",
    "    # enc_op_list = []\n",
    "    \n",
    "    for sents in pbar:\n",
    "        sents = sents.to(device)\n",
    "\n",
    "        if type != 3:\n",
    "            loss, enc_op = train_step(net, sents, sents, pad_idx,\n",
    "                                optimizer, criterion, type)\n",
    "        if type == 3:\n",
    "            loss, enc_op = train_step(net, sents, sents, pad_idx,\n",
    "                            optimizer, criterion, type, ae, args.ae_cpath)\n",
    "        pbar.set_description(\n",
    "            'Epoch: {};  Type: Train; Loss: {:.5f}'.format(\n",
    "                epoch + 1, loss\n",
    "            )\n",
    "        )\n",
    "        # \"\"\"Extract encoded tensors\"\"\"\n",
    "        # enc_op_list.append(enc_op.cpu().detach().numpy())\n",
    "    # print(len(enc_op_list))\n",
    "    \n",
    "    # print(len(enc_op_list[0]), len(enc_op_list[0][0]), len(enc_op_list[0][0][0]))\n",
    "    # try:\n",
    "    #     print(len(enc_op_list[0][0][0][0]))\n",
    "    # except:\n",
    "    #     print(\"Nope\")\n",
    "    # data = []\n",
    "    # for batch in enc_op_list:\n",
    "    #     for line in batch:\n",
    "    #         data.append(line)\n",
    "    # op1 = pd.DataFrame({\"enc\": data, \"dec\": data})\n",
    "    # op1.to_csv(\"enc_op.csv\", encoding='utf-8', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-14T10:06:38.804043Z",
     "iopub.status.busy": "2024-11-14T10:06:38.803436Z",
     "iopub.status.idle": "2024-11-14T10:51:20.199934Z",
     "shell.execute_reply": "2024-11-14T10:51:20.198949Z",
     "shell.execute_reply.started": "2024-11-14T10:06:38.804004Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    # setup_seed(10)\n",
    "    args, unknown = parser.parse_known_args()\n",
    "    \n",
    "    # args.vocab_file = './' + args.vocab_file\n",
    "    args.vocab_file = args.vocab_file\n",
    "\n",
    "    \"\"\" preparing the dataset \"\"\"\n",
    "    vocab = json.load(open(args.vocab_file, 'rb'))\n",
    "    token_to_idx = vocab['token_to_idx']\n",
    "    num_vocab = len(token_to_idx)\n",
    "    pad_idx = token_to_idx[\"<PAD>\"]\n",
    "    start_idx = token_to_idx[\"<START>\"]\n",
    "    end_idx = token_to_idx[\"<END>\"]\n",
    "\n",
    "\n",
    "    \"\"\" define optimizer and loss function \"\"\"\n",
    "    model1 = Model(args.num_layers_deepsc, num_vocab, num_vocab,\n",
    "                        num_vocab, num_vocab, args.d_model, args.num_heads,\n",
    "                        args.dff, 0.1).to(device)\n",
    "    # for types 3, 4, enc blocks = 6\n",
    "    model2 = Model(args.num_layers_ae, num_vocab, num_vocab,\n",
    "                        num_vocab, num_vocab, args.d_model, args.num_heads,\n",
    "                        args.dff, 0.1).to(device)\n",
    "    \n",
    "\n",
    "    \"\"\"Define type and model\"\"\"\n",
    "\n",
    "    type = 0 #type can be 0, 1, 3 is automaticlly model2\n",
    "    deepsc = model1\n",
    "    \n",
    "    criterion = nn.CrossEntropyLoss(reduction = 'none')\n",
    "    optimizer1 = torch.optim.Adam(deepsc.parameters(),\n",
    "                                 lr=1e-4, betas=(0.9, 0.98), eps=1e-8, weight_decay = 5e-4)\n",
    "    optimizer2 = torch.optim.Adam(deepsc.parameters(),\n",
    "                                 lr=2e-4, betas=(0.9, 0.98), eps=1e-8, weight_decay = 5e-4)\n",
    "    #opt = NoamOpt(args.d_model, 1, 4000, optimizer)\n",
    "    initNetParams(deepsc)\n",
    "    # for param_tensor in deepsc.state_dict():\n",
    "    #     print(param_tensor, \"\\t\", deepsc.state_dict()[param_tensor].size())\n",
    "    pths = ['checkpoints/deepsc-1', 'checkpoints/deepsc-2']\n",
    "    optimizers = {\"1\":optimizer1,\"2\":optimizer2}\n",
    "    # args.checkpoint_path ='checkpoints/deepsc-Rayleigh1'\n",
    "    # args.checkpoint_path ='checkpoints/deepsc-Rayleigh2'\n",
    "    for p in pths:\n",
    "        print(p, p[-1])\n",
    "        args.checkpoint_path = p\n",
    "        op = optimizers[p[-1]]\n",
    "\n",
    "        for epoch in range(args.epochs):\n",
    "            start = time.time()\n",
    "            record_acc = 10\n",
    "            \n",
    "            \"\"\"Check if type 3\"\"\"\n",
    "            if type == 3:\n",
    "                ae = AE().to(device)\n",
    "                train(epoch, args, deepsc, op, type, ae) \n",
    "                avg_acc = validate(epoch, args, deepsc, type, ae)\n",
    "            else:\n",
    "                train(epoch, args, deepsc, op, type) \n",
    "                avg_acc = validate(epoch, args, deepsc, type)\n",
    "            # print(avg_acc)\n",
    "            if avg_acc < record_acc:\n",
    "                if not os.path.exists(args.checkpoint_path):\n",
    "                    os.makedirs(args.checkpoint_path)\n",
    "                with open(args.checkpoint_path + '/checkpoint_{}.pkl'.format(str(epoch + 1).zfill(2)), 'wb') as f:\n",
    "                    torch.save(deepsc.state_dict(), f)\n",
    "                record_acc = avg_acc\n",
    "        record_loss = []\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-14T17:20:53.539406Z",
     "iopub.status.busy": "2024-11-14T17:20:53.538944Z",
     "iopub.status.idle": "2024-11-14T17:20:54.779825Z",
     "shell.execute_reply": "2024-11-14T17:20:54.778378Z",
     "shell.execute_reply.started": "2024-11-14T17:20:53.539362Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import torch\n",
    "import argparse\n",
    "import numpy as np\n",
    "from dataset import EurDataset, collate_data\n",
    "from models.transformer import Model\n",
    "from torch.utils.data import DataLoader\n",
    "from utils import BleuScore, greedy_decode, SeqtoText\n",
    "from tqdm import tqdm\n",
    "from sklearn.preprocessing import normalize\n",
    "# from bert4keras.backend import keras\n",
    "# from bert4keras.models import build_bert_model\n",
    "# from bert4keras.tokenizers import Tokenizer\n",
    "from w3lib.html import remove_tags\n",
    "from models.autoencoders import AE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-14T17:21:06.808642Z",
     "iopub.status.busy": "2024-11-14T17:21:06.807917Z",
     "iopub.status.idle": "2024-11-14T17:21:06.852719Z",
     "shell.execute_reply": "2024-11-14T17:21:06.851643Z",
     "shell.execute_reply.started": "2024-11-14T17:21:06.808595Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "parser = argparse.ArgumentParser()\n",
    "# parser.add_argument('--data-dir', default='europarl/train_data.pkl', type=str)\n",
    "# parser.add_argument('--vocab-file', default='europarl/vocab.json', type=str)\n",
    "parser.add_argument('--data-dir', default='data/train_data.pkl', type=str)\n",
    "parser.add_argument('--vocab-file', default='data/vocab.json', type=str)\n",
    "parser.add_argument('--checkpoint-path', default='checkpoints/deepsc', type=str)\n",
    "# parser.add_argument('--channel', default='Rayleigh', type=str)\n",
    "parser.add_argument('--MAX-LENGTH', default=30, type=int)\n",
    "parser.add_argument('--MIN-LENGTH', default=4, type=int)\n",
    "parser.add_argument('--d-model', default=128, type = int)\n",
    "parser.add_argument('--dff', default=512, type=int)\n",
    "parser.add_argument('--num-layers_deepsc', default=4, type=int)\n",
    "parser.add_argument('--num-layers_ae', default=6, type=int)\n",
    "parser.add_argument('--num-heads', default=8, type=int)\n",
    "parser.add_argument('--batch-size', default=64, type=int)\n",
    "parser.add_argument('--epochs', default=2, type = int)\n",
    "parser.add_argument('--ae_cpath', default=\"data/checks\", type=str)\n",
    "# parser.add_argument('--ae_cpath', default=\"/kaggle/working/checks/ae\", type = str)\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-11-14T10:51:20.223677Z",
     "iopub.status.busy": "2024-11-14T10:51:20.223357Z",
     "iopub.status.idle": "2024-11-14T11:00:10.275139Z",
     "shell.execute_reply": "2024-11-14T11:00:10.274178Z",
     "shell.execute_reply.started": "2024-11-14T10:51:20.223638Z"
    },
    "trusted": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "import numpy as np\n",
    "def performance(args, net, lr, type, ae=None):\n",
    "    # similarity = Similarity(args.bert_config_path, args.bert_checkpoint_path, args.bert_dict_path)\n",
    "    bleu_score_1gram = BleuScore(1, 0, 0, 0)\n",
    "    bleu_score_2gram = BleuScore(0, 1, 0, 0)\n",
    "    bleu_score_3gram = BleuScore(0, 0, 1, 0)\n",
    "    bleu_score_4gram = BleuScore(0, 0, 0, 1)\n",
    "    bleu_score_avg = BleuScore(0.25, 0.25, 0.25, 0.25)    \n",
    "    \n",
    "    # gram_dict = {\"one\":bleu_score_1gram, \"two\":bleu_score_2gram,\n",
    "    #             \"three\":bleu_score_3gram, \"four\":bleu_score_4gram, \"avg\":bleu_score_avg}\n",
    "\n",
    "    # bleu_score_gram = gram_dict[gram]    \n",
    "    \n",
    "    test_eur = EurDataset('test')\n",
    "    test_iterator = DataLoader(test_eur, batch_size=args.batch_size, num_workers=0,\n",
    "                               pin_memory=True, collate_fn=collate_data)\n",
    "\n",
    "    StoT = SeqtoText(token_to_idx, end_idx)\n",
    "    scoreavg = []\n",
    "    score1 = []\n",
    "    score2 = []\n",
    "    score3 = []\n",
    "    score4 = []\n",
    "    \n",
    "    net.eval()\n",
    "    \n",
    "    print(\"Test Model Type\", type)\n",
    "    with torch.no_grad():\n",
    "        for epoch in (range(args.epochs)):\n",
    "            print(\"In Epoch {}\".format(epoch))\n",
    "            \n",
    "            Tx_word = []\n",
    "            Rx_word = []\n",
    "            for i in range(1):\n",
    "                word = []\n",
    "                target_word = []\n",
    "                print(\"Encoding-Decoding\")\n",
    "                for sents in tqdm(test_iterator):\n",
    "    \n",
    "                    sents = sents.to(device)\n",
    "                    # src = batch.src.transpose(0, 1)[:1]\n",
    "                    target = sents\n",
    "                    if type != 3:\n",
    "                        out = greedy_decode(net, sents, args.MAX_LENGTH, pad_idx,\n",
    "                                                start_idx, type)\n",
    "                    if type == 3:\n",
    "                        out = greedy_decode(net, sents, args.MAX_LENGTH, pad_idx,\n",
    "                                            start_idx, type, ae, args.ae_cpath)\n",
    "                    # print(\"out####################\", len(out), len(out[0]),len(out[1]))\n",
    "    \n",
    "                    sentences = out.cpu().numpy().tolist()\n",
    "                    result_string = list(map(StoT.sequence_to_text, sentences))\n",
    "                    word = word + result_string\n",
    "                    # print(\"word###################\", (result_string))\n",
    "    \n",
    "                    target_sent = target.cpu().numpy().tolist()\n",
    "                    result_string = list(map(StoT.sequence_to_text, target_sent))\n",
    "                    target_word = target_word + result_string\n",
    "                    # print(\"w********************\", (result_string))\n",
    "\n",
    "                Tx_word.append(word)\n",
    "                Rx_word.append(target_word)\n",
    "                \n",
    "            # op_to_csv = {\"sent1\" : Tx_word, \"sent2\" : Rx_word}\n",
    "            # op = pd.DataFrame(op_to_csv)\n",
    "            # print(op.head())\n",
    "            # op.to_csv(\"op.csv\", sep = \"\\t\", encoding='utf-8')\n",
    "            # print(len(Tx_word))    \n",
    "            bleu_score = []\n",
    "            bleu1 = []\n",
    "            bleu2 = []\n",
    "            bleu3 = []\n",
    "            bleu4 = []\n",
    "\n",
    "            # op_to_csv = {\"sent1\" : Tx_word, \"sent2\" : Rx_word}\n",
    "            # op = pd.DataFrame(op_to_csv)\n",
    "            # print(op.head(25))\n",
    "            # op.to_csv(\"op.csv\", encoding='utf-8')\n",
    "            s1 = []\n",
    "            s2 = []\n",
    "            print(\"Scoring\")\n",
    "            for sent1, sent2 in tqdm(zip(Tx_word, Rx_word)):#cos no snr, len of txword is 1\n",
    "                #therefore there is only 1 list to iter over, ie., 1 sent1, 2\n",
    "                \n",
    "                # print(sent1[1], sent2[1])\n",
    "                # 1-gram\n",
    "                s1.append(sent1)\n",
    "                s2.append(sent2)\n",
    "                a = time.time()\n",
    "                bleu1.append(bleu_score_1gram.compute_blue_score(sent1, sent2)) # 7*num_sent\n",
    "                bleu2.append(bleu_score_2gram.compute_blue_score(sent1, sent2))\n",
    "                bleu3.append(bleu_score_3gram.compute_blue_score(sent1, sent2))\n",
    "                bleu4.append(bleu_score_4gram.compute_blue_score(sent1, sent2))\n",
    "                bleu_score.append(bleu_score_avg.compute_blue_score(sent1, sent2))\n",
    "                b = time.time() - a\n",
    "                # print(b)\n",
    "                # sim_score.append(similarity.compute_similarity(sent1, sent2)) # 7*num_sent\n",
    "            # scores = [bleu_score, bleu1, bleu2, bleu3, bleu4]\n",
    "                dict = {\"sent1\":sent1, \"sent2\":sent2}\n",
    "                # print(len(dict[\"sent1\"]))\n",
    "                op1 = pd.DataFrame(dict)\n",
    "                # print(op1.head())\n",
    "                op1.to_csv(\"opnew{}.csv\".format(lr), encoding='utf-8')\n",
    "\n",
    "            \n",
    "            # print(len(s1[0]), len(s2[0]), len(bleu_score[0]))\n",
    "\n",
    "   \n",
    "            scoreavg.append(np.mean(np.array(bleu_score, float), axis=1))\n",
    "            score1.append(np.mean(np.array(bleu1), axis=1))\n",
    "            score2.append(np.mean(np.array(bleu2), axis=1))\n",
    "            score3.append(np.mean(np.array(bleu3), axis=1))\n",
    "            score4.append(np.mean(np.array(bleu4), axis=1))\n",
    "       #avg over epochs     \n",
    "    scoref_avg = np.mean(np.array(scoreavg), axis=0)\n",
    "    scoref_1 = np.mean(np.array(score1), axis=0)\n",
    "    scoref_2 = np.mean(np.array(score2), axis=0)\n",
    "    scoref_3 = np.mean(np.array(score3), axis=0)\n",
    "    scoref_4 = np.mean(np.array(score4), axis=0)\n",
    "            \n",
    "            \n",
    "            # sim_score = np.array(sim_score)\n",
    "            # sim_score = np.mean(sim_score, axis=1)\n",
    "            # score2.append(sim_score)\n",
    "    \n",
    "    # score2 = np.mean(np.array(score2), axis=0)\n",
    "\n",
    "    return {\"avg\": scoref_avg, \"1\":scoref_1,\"2\":scoref_2, \"3\":scoref_3, \"4\":scoref_4}#, score2\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    args, unk = parser.parse_known_args()\n",
    "\n",
    "    # args.vocab_file = '/import/antennas/Datasets/hx301/' + args.vocab_file\n",
    "    args.vocab_file = '' + args.vocab_file\n",
    "    vocab = json.load(open(args.vocab_file, 'rb'))\n",
    "    token_to_idx = vocab['token_to_idx']\n",
    "    idx_to_token = dict(zip(token_to_idx.values(), token_to_idx.keys()))\n",
    "    num_vocab = len(token_to_idx)\n",
    "    pad_idx = token_to_idx[\"<PAD>\"]\n",
    "    start_idx = token_to_idx[\"<START>\"]\n",
    "    end_idx = token_to_idx[\"<END>\"]\n",
    "\n",
    "\n",
    "    \n",
    "    \n",
    "    \"\"\" define optimizer and loss function \"\"\"\n",
    "    model1 = Model(args.num_layers_deepsc, num_vocab, num_vocab,\n",
    "                        num_vocab, num_vocab, args.d_model, args.num_heads,\n",
    "                        args.dff, 0.1).to(device)\n",
    "\n",
    "    model2 = Model(args.num_layers_ae, num_vocab, num_vocab,\n",
    "                        num_vocab, num_vocab, args.d_model, args.num_heads,\n",
    "                        args.dff, 0.1).to(device)\n",
    "    \n",
    "    \"\"\"Define type and model \"\"\"\n",
    "    type = 0\n",
    "    deepsc = model1\n",
    "    \n",
    "    \n",
    "    pths = ['checkpoints/deepsc-1','checkpoints/deepsc-2']\n",
    "    bleus_df = pd.DataFrame()\n",
    "    for p in pths:\n",
    "        #lr\n",
    "        print(p)\n",
    "        \n",
    "        model_paths = []\n",
    "        args.checkpoint_path = p\n",
    "        for fn in os.listdir(args.checkpoint_path):\n",
    "            \n",
    "            if not fn.endswith('.pkl'): continue\n",
    "            idx = int(os.path.splitext(fn)[0].split('_')[-1])  # read the idx of image\n",
    "            model_paths.append((os.path.join(args.checkpoint_path, fn), idx))\n",
    "\n",
    "        model_paths.sort(key=lambda x: x[1])  # sort the image by the idx\n",
    "    \n",
    "        model_path, _ = model_paths[-1]\n",
    "        checkpoint = torch.load(model_path)\n",
    "        deepsc.load_state_dict(checkpoint)\n",
    "        print('model load!')\n",
    "\n",
    "        if type == 3:\n",
    "            ae = AE().to(device)\n",
    "            bleus_all = performance(args, deepsc, p[-1], type, ae)\n",
    "        else:\n",
    "            bleus_all = performance(args, deepsc, p[-1], type)\n",
    "        # bleu1 = performance(args, SNR, deepsc, \"one\", ch)\n",
    "        # bleu2 = performance(args, SNR, deepsc, \"two\", ch)\n",
    "        # bleu3 = performance(args, SNR, deepsc, \"three\", ch)\n",
    "        # bleu4 = performance(args, SNR, deepsc, \"four\", ch)\n",
    "        # bleu_avg = performance(args, SNR, deepsc, \"avg\", ch)\n",
    "    #     bleu_dict = {\"SNR\":SNR, \"1gram\": bleu1, \"2gram\":bleu2,\"3gram\":bleu3,\n",
    "    #                  \"4gram\":bleu4,\"avg\":bleu_avg, \"Channel\":ch, \"LR\":p[-1]}\n",
    "    #     bleus = pd.DataFrame(bleu_dict)\n",
    "    #     bleus_df.append(bleus, ignore_index=True)\n",
    "    #     print(bleus.head())\n",
    "        \n",
    "    # bleus_df.to_csv(\"bleus.csv\", encoding='utf-8')\n",
    "        print(p[-1], bleus_all)\n",
    "    #similarity.compute_similarity(sent1, real)\n",
    "\n",
    "\n",
    "# len of belu score op = 50 = the no of row items\n",
    "# but each is a list - but if bleu returns 1 score per sentence, then it should be a float,\n",
    "# makes no sense to return a score for each word.\n"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "datasetId": 5998475,
     "sourceId": 9903436,
     "sourceType": "datasetVersion"
    },
    {
     "sourceId": 206756172,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 207263103,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 207275494,
     "sourceType": "kernelVersion"
    },
    {
     "sourceId": 207100083,
     "sourceType": "kernelVersion"
    }
   ],
   "dockerImageVersionId": 30787,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}