fine_tunning_covid.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "lIefVYJR_wfJ",
    "outputId": "568a0b16-7e96-4400-c3ec-94ef348b1327"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyTorch Version:  1.5.0\n",
      "Torchvision Version:  0.6.0\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function \n",
    "from __future__ import division\n",
    "import itertools\n",
    "from PIL import Image\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "import torchvision\n",
    "import torchxrayvision as xrv\n",
    "from torchvision import datasets, models, transforms\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "import matplotlib.pyplot as plt\n",
    "from skimage.exposure import equalize_adapthist\n",
    "from sklearn.metrics import classification_report, confusion_matrix, precision_score, recall_score, f1_score, balanced_accuracy_score\n",
    "import datetime\n",
    "import time\n",
    "import os\n",
    "import copy\n",
    "import sys\n",
    "import dataset\n",
    "\n",
    "print(\"PyTorch Version: \",torch.__version__)\n",
    "print(\"Torchvision Version: \",torchvision.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Q1zxbdpcAVDF"
   },
   "outputs": [],
   "source": [
    "# Top level data directory. Here we assume the format of the directory conforms \n",
    "#   to the ImageFolder structure\n",
    "data_dir = \"/home/users/ester/datasets/covidx\"\n",
    "\n",
    "path_train = '/home/users/ester/datasets/covidx/train'\n",
    "path_val = '/home/users/ester/datasets/covidx/test'\n",
    "\n",
    "list_img_train, lbl_train = dataset.get_dataset_info(path_train)\n",
    "list_img_val, lbl_val = dataset.get_dataset_info(path_val)\n",
    "\n",
    "# creating tensorboar file\n",
    "current_time = datetime.datetime.now().strftime(\"%d%m%Y-%H%M%S\")\n",
    "\n",
    "writer = SummaryWriter('runs/fine_tunning_covid/' + current_time)\n",
    "\n",
    "# selecting gpu device\n",
    "device = torch.device(\"cuda:2\") # if torch.cuda.is_available() else \"cpu\") \n",
    "\n",
    "model_name= \"\"\n",
    "\n",
    "# Number of classes in the dataset\n",
    "num_classes = 3\n",
    "\n",
    "class_labels = ['covid', 'normal', 'pneumonia']\n",
    "\n",
    "# classes weights\n",
    "weights = [13906/476, 13906/7966, 13906/5464]\n",
    "\n",
    "# Batch size for training (change depending on how much memory you have)\n",
    "batch_size = 30\n",
    "\n",
    "# Number of epochs to train for \n",
    "num_epochs = 100\n",
    "\n",
    "# Flag for feature extracting. When False, we finetune the whole model, \n",
    "#   when True we only update the reshaped layer params\n",
    "feature_extract = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "l2aCsfdFAX3K"
   },
   "outputs": [],
   "source": [
    "def train_model(model, dataloaders, criterion, optimizer, num_epochs=25, is_inception=False):\n",
    "    since = time.time()\n",
    "\n",
    "    val_acc_history = []\n",
    "    \n",
    "    best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    best_acc = 0.0\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n",
    "        print('-' * 10)\n",
    "\n",
    "        # Each epoch has a training and validation phase\n",
    "        for phase in ['train', 'test']:\n",
    "            if phase == 'train':\n",
    "                model.train()  # Set model to training mode\n",
    "            else:\n",
    "                model.eval()   # Set model to evaluate mode\n",
    "\n",
    "            running_loss = 0.0\n",
    "            running_corrects = 0\n",
    "            labels_list = []\n",
    "            pred_list = []\n",
    "            \n",
    "            \n",
    "\n",
    "            # Iterate over data.\n",
    "            for inputs, labels in dataloaders[phase]:\n",
    "                inputs = inputs.to(device)\n",
    "                labels = labels.to(device)\n",
    "            \n",
    "                # zero the parameter gradients\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # forward\n",
    "                # track history if only in train\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    # Get model outputs and calculate loss\n",
    "                    # Special case for inception because in training it has an auxiliary output. In train\n",
    "                    #   mode we calculate the loss by summing the final output and the auxiliary output\n",
    "                    #   but in testing we only consider the final output.\n",
    "                    if is_inception and phase == 'train':\n",
    "                        # From https://discuss.pytorch.org/t/how-to-optimize-inception-model-with-auxiliary-classifiers/7958\n",
    "                        outputs, aux_outputs = model(inputs)\n",
    "                        loss1 = criterion(outputs, labels)\n",
    "                        loss2 = criterion(aux_outputs, labels)\n",
    "                        loss = loss1 + 0.4*loss2\n",
    "                    else:\n",
    "                        outputs = model(inputs)\n",
    "                        loss = criterion(outputs, labels)\n",
    "\n",
    "                    _, preds = torch.max(outputs, 1)\n",
    "                    \n",
    "                   \n",
    "\n",
    "                    # backward + optimize only if in training phase\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "\n",
    "                # statistics\n",
    "                running_loss += loss.item() * inputs.size(0)\n",
    "                running_corrects += torch.sum(preds == labels.data)\n",
    "                \n",
    "                labels_list.extend(labels.cpu().numpy())\n",
    "                pred_list.extend(preds.cpu().numpy())\n",
    "\n",
    "\n",
    "            epoch_loss = running_loss / len(dataloaders[phase].dataset)\n",
    "            epoch_acc = running_corrects.double() / len(dataloaders[phase].dataset)\n",
    "            \n",
    "            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))\n",
    "            \n",
    "            writer.add_scalar('Loss/{}'.format(phase) , epoch_loss, epoch)\n",
    "            writer.add_scalar('Accuracy/{}'.format(phase) , epoch_acc, epoch)\n",
    "            \n",
    "            \n",
    "            labels = labels_list\n",
    "            preds = pred_list\n",
    "\n",
    "            balanced_accuracy = balanced_accuracy_score(y_true=labels, y_pred=preds)\n",
    "            precision = precision_score(labels, preds, average=None)\n",
    "            recall = recall_score(labels, preds, average=None)\n",
    "            f1 = f1_score(labels, preds, average=None)\n",
    "\n",
    "            print('{} Balanced Accuracy: {} Precision: {} Recall: {} F1-Score: {}'.format(phase, balanced_accuracy, precision, recall, f1))\n",
    "\n",
    "            writer.add_scalar('Balanced_Accuracy/{}'.format(phase) , balanced_accuracy, epoch)\n",
    "            writer.add_scalar('Precision covid/{}'.format(phase) , precision[0], epoch)\n",
    "            writer.add_scalar('Recall covid/{}'.format(phase) , recall[0], epoch)\n",
    "            writer.add_scalar('F1 Score covid/{}'.format(phase) , f1[0], epoch)\n",
    "\n",
    "            writer.add_scalar('Precision normal/{}'.format(phase) , precision[1], epoch)\n",
    "            writer.add_scalar('Recall normal/{}'.format(phase) , recall[1], epoch)\n",
    "            writer.add_scalar('F1 Score normal/{}'.format(phase) , f1[1], epoch)\n",
    "\n",
    "            writer.add_scalar('Precision pneumonia/{}'.format(phase) , precision[2], epoch)\n",
    "            writer.add_scalar('Recall pneumonia/{}'.format(phase) , recall[2], epoch)\n",
    "            writer.add_scalar('F1 Score pneumonia/{}'.format(phase) , f1[2], epoch)\n",
    "\n",
    "\n",
    "#             print(classification_report(labels, preds, target_names=class_labels))\n",
    "\n",
    "            cm = confusion_matrix(labels, preds)\n",
    "\n",
    "            cm_fig = plot_confusion_matrix(cm, classes=class_labels, normalize=True, title='{} Confusion Matrix'.format(phase))\n",
    "\n",
    "            writer.add_figure('Confusion Matrix/'+ phase, cm_fig, epoch)\n",
    "\n",
    "            # deep copy the model\n",
    "            if phase == 'test' and epoch_acc > best_acc:\n",
    "                best_acc = epoch_acc\n",
    "                best_model_wts = copy.deepcopy(model.state_dict())\n",
    "            if phase == 'test':\n",
    "                val_acc_history.append(epoch_acc)\n",
    "\n",
    "        print()\n",
    "\n",
    "    time_elapsed = time.time() - since\n",
    "    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n",
    "    print('Best val Acc: {:4f}'.format(best_acc))\n",
    "    \n",
    "    writer.close()\n",
    "\n",
    "    # load best model weights\n",
    "    model.load_state_dict(best_model_wts)\n",
    "    return model, val_acc_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BhkW97JtAcKd"
   },
   "outputs": [],
   "source": [
    "def set_parameter_requires_grad(model, feature_extracting):\n",
    "    if feature_extracting:\n",
    "        for param in model.parameters():\n",
    "            param.requires_grad = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues,\n",
    "                          save=False):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    \n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "    \n",
    "    fig = plt.figure(figsize=(12, 12))\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "    \n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, '%.2f' % (cm[i, j] * 100.0),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "    \n",
    "#     plt.show()\n",
    "    \n",
    "    return fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# class EqualizeAdapthistTransform(object):\n",
    "#     def __init__(self):\n",
    "#         '''\n",
    "#         initialize your transformation here, if necessary\n",
    "#         '''\n",
    "#         pass\n",
    "    \n",
    "    \n",
    "#         def __call__(self, pic):\n",
    "#             arr = np.array(pic) # transform to np array\n",
    "#             pic_dtype = arr.dtype\n",
    "#             arr = equalize_adapthist(arr) # apply your transformation\n",
    "#             arr = arr.astype(pic_dtype)\n",
    "#             pic = Image.from_array(arr)\n",
    "#             return pic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3ELERlI6F-c8"
   },
   "source": [
    "# Initialize and Reshape the Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "RH8e7QF1ArQa",
    "outputId": "1e8299bd-3219-4260-b69f-607badcb9c8c"
   },
   "outputs": [],
   "source": [
    "model_ft = xrv.models.DenseNet(weights=\"all\")\n",
    "set_parameter_requires_grad(model_ft, feature_extract)\n",
    "model_ft.classifier = nn.Linear(1024, num_classes)\n",
    "model_ft.pathologies = ['normal', \"pneumonia\", 'COVID-19']\n",
    "model_ft.op_threshs = None\n",
    "input_size = 224\n",
    "# Print the model we just instantiated\n",
    "# print(model_ft)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "diPb9wuPHpIS"
   },
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "qBe5x_QPGNCq",
    "outputId": "22e98de2-a227-45fe-93b5-55db463c742c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initializing Datasets and Dataloaders...\n"
     ]
    }
   ],
   "source": [
    "# Data augmentation and normalization for training\n",
    "# Just normalization for validation\n",
    "\n",
    "std = 0.24671278988052675\n",
    "mean = 0.4912771402827791\n",
    "\n",
    "data_transforms = {\n",
    "    'train': transforms.Compose([\n",
    "        transforms.RandomResizedCrop(input_size),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.Grayscale(num_output_channels=1),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([mean], [std]),\n",
    "          \n",
    "    ]),\n",
    "    'test': transforms.Compose([\n",
    "        transforms.Resize(input_size),\n",
    "        transforms.CenterCrop(input_size),\n",
    "        transforms.Grayscale(num_output_channels=1),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([mean], [std]),\n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "    ]),\n",
    "}\n",
    "print(\"Initializing Datasets and Dataloaders...\")\n",
    "\n",
    "# Create training and validation datasets\n",
    "image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x), data_transforms[x]) for x in ['train', 'test']}\n",
    "# Create training and validation dataloaders\n",
    "dataloaders_dict = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=batch_size, shuffle=True, num_workers=4) for x in ['train', 'test']}\n",
    "\n",
    "\n",
    "# tmp_dataset_train = dataset.COVID19_Dataset(list_img_train, lbl_train, transform = data_transforms['train'])\n",
    "# tmp_dataset_val = dataset.COVID19_Dataset(list_img_val, lbl_val, transform = data_transforms['test'])\n",
    "\n",
    "# Create a sampler by samples weights \n",
    "# sampler = torch.utils.data.sampler.WeightedRandomSampler(\n",
    "#     weights=tmp_dataset_train.samples_weights,\n",
    "#     num_samples=tmp_dataset_train.len)\n",
    "\n",
    "\n",
    "# dataloaders_dict = {}\n",
    "\n",
    "# dataloaders_dict['train'] = torch.utils.data.DataLoader(tmp_dataset_train, \n",
    "#                                                     batch_size=batch_size, \n",
    "# #                                                     sampler=sampler,\n",
    "#                                                     num_workers=4)\n",
    "\n",
    "# dataloaders_dict['test'] = torch.utils.data.DataLoader(tmp_dataset_val, \n",
    "#                                                     batch_size=batch_size, \n",
    "#                                                     num_workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 67
    },
    "colab_type": "code",
    "id": "-9JI91I27JOr",
    "outputId": "73c122c5-cd04-4366-fb16-ffb79df03bb7",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Params to learn:\n",
      "\t features.conv0.weight\n",
      "\t features.norm0.weight\n",
      "\t features.norm0.bias\n",
      "\t features.denseblock1.denselayer1.norm1.weight\n",
      "\t features.denseblock1.denselayer1.norm1.bias\n",
      "\t features.denseblock1.denselayer1.conv1.weight\n",
      "\t features.denseblock1.denselayer1.norm2.weight\n",
      "\t features.denseblock1.denselayer1.norm2.bias\n",
      "\t features.denseblock1.denselayer1.conv2.weight\n",
      "\t features.denseblock1.denselayer2.norm1.weight\n",
      "\t features.denseblock1.denselayer2.norm1.bias\n",
      "\t features.denseblock1.denselayer2.conv1.weight\n",
      "\t features.denseblock1.denselayer2.norm2.weight\n",
      "\t features.denseblock1.denselayer2.norm2.bias\n",
      "\t features.denseblock1.denselayer2.conv2.weight\n",
      "\t features.denseblock1.denselayer3.norm1.weight\n",
      "\t features.denseblock1.denselayer3.norm1.bias\n",
      "\t features.denseblock1.denselayer3.conv1.weight\n",
      "\t features.denseblock1.denselayer3.norm2.weight\n",
      "\t features.denseblock1.denselayer3.norm2.bias\n",
      "\t features.denseblock1.denselayer3.conv2.weight\n",
      "\t features.denseblock1.denselayer4.norm1.weight\n",
      "\t features.denseblock1.denselayer4.norm1.bias\n",
      "\t features.denseblock1.denselayer4.conv1.weight\n",
      "\t features.denseblock1.denselayer4.norm2.weight\n",
      "\t features.denseblock1.denselayer4.norm2.bias\n",
      "\t features.denseblock1.denselayer4.conv2.weight\n",
      "\t features.denseblock1.denselayer5.norm1.weight\n",
      "\t features.denseblock1.denselayer5.norm1.bias\n",
      "\t features.denseblock1.denselayer5.conv1.weight\n",
      "\t features.denseblock1.denselayer5.norm2.weight\n",
      "\t features.denseblock1.denselayer5.norm2.bias\n",
      "\t features.denseblock1.denselayer5.conv2.weight\n",
      "\t features.denseblock1.denselayer6.norm1.weight\n",
      "\t features.denseblock1.denselayer6.norm1.bias\n",
      "\t features.denseblock1.denselayer6.conv1.weight\n",
      "\t features.denseblock1.denselayer6.norm2.weight\n",
      "\t features.denseblock1.denselayer6.norm2.bias\n",
      "\t features.denseblock1.denselayer6.conv2.weight\n",
      "\t features.transition1.norm.weight\n",
      "\t features.transition1.norm.bias\n",
      "\t features.transition1.conv.weight\n",
      "\t features.denseblock2.denselayer1.norm1.weight\n",
      "\t features.denseblock2.denselayer1.norm1.bias\n",
      "\t features.denseblock2.denselayer1.conv1.weight\n",
      "\t features.denseblock2.denselayer1.norm2.weight\n",
      "\t features.denseblock2.denselayer1.norm2.bias\n",
      "\t features.denseblock2.denselayer1.conv2.weight\n",
      "\t features.denseblock2.denselayer2.norm1.weight\n",
      "\t features.denseblock2.denselayer2.norm1.bias\n",
      "\t features.denseblock2.denselayer2.conv1.weight\n",
      "\t features.denseblock2.denselayer2.norm2.weight\n",
      "\t features.denseblock2.denselayer2.norm2.bias\n",
      "\t features.denseblock2.denselayer2.conv2.weight\n",
      "\t features.denseblock2.denselayer3.norm1.weight\n",
      "\t features.denseblock2.denselayer3.norm1.bias\n",
      "\t features.denseblock2.denselayer3.conv1.weight\n",
      "\t features.denseblock2.denselayer3.norm2.weight\n",
      "\t features.denseblock2.denselayer3.norm2.bias\n",
      "\t features.denseblock2.denselayer3.conv2.weight\n",
      "\t features.denseblock2.denselayer4.norm1.weight\n",
      "\t features.denseblock2.denselayer4.norm1.bias\n",
      "\t features.denseblock2.denselayer4.conv1.weight\n",
      "\t features.denseblock2.denselayer4.norm2.weight\n",
      "\t features.denseblock2.denselayer4.norm2.bias\n",
      "\t features.denseblock2.denselayer4.conv2.weight\n",
      "\t features.denseblock2.denselayer5.norm1.weight\n",
      "\t features.denseblock2.denselayer5.norm1.bias\n",
      "\t features.denseblock2.denselayer5.conv1.weight\n",
      "\t features.denseblock2.denselayer5.norm2.weight\n",
      "\t features.denseblock2.denselayer5.norm2.bias\n",
      "\t features.denseblock2.denselayer5.conv2.weight\n",
      "\t features.denseblock2.denselayer6.norm1.weight\n",
      "\t features.denseblock2.denselayer6.norm1.bias\n",
      "\t features.denseblock2.denselayer6.conv1.weight\n",
      "\t features.denseblock2.denselayer6.norm2.weight\n",
      "\t features.denseblock2.denselayer6.norm2.bias\n",
      "\t features.denseblock2.denselayer6.conv2.weight\n",
      "\t features.denseblock2.denselayer7.norm1.weight\n",
      "\t features.denseblock2.denselayer7.norm1.bias\n",
      "\t features.denseblock2.denselayer7.conv1.weight\n",
      "\t features.denseblock2.denselayer7.norm2.weight\n",
      "\t features.denseblock2.denselayer7.norm2.bias\n",
      "\t features.denseblock2.denselayer7.conv2.weight\n",
      "\t features.denseblock2.denselayer8.norm1.weight\n",
      "\t features.denseblock2.denselayer8.norm1.bias\n",
      "\t features.denseblock2.denselayer8.conv1.weight\n",
      "\t features.denseblock2.denselayer8.norm2.weight\n",
      "\t features.denseblock2.denselayer8.norm2.bias\n",
      "\t features.denseblock2.denselayer8.conv2.weight\n",
      "\t features.denseblock2.denselayer9.norm1.weight\n",
      "\t features.denseblock2.denselayer9.norm1.bias\n",
      "\t features.denseblock2.denselayer9.conv1.weight\n",
      "\t features.denseblock2.denselayer9.norm2.weight\n",
      "\t features.denseblock2.denselayer9.norm2.bias\n",
      "\t features.denseblock2.denselayer9.conv2.weight\n",
      "\t features.denseblock2.denselayer10.norm1.weight\n",
      "\t features.denseblock2.denselayer10.norm1.bias\n",
      "\t features.denseblock2.denselayer10.conv1.weight\n",
      "\t features.denseblock2.denselayer10.norm2.weight\n",
      "\t features.denseblock2.denselayer10.norm2.bias\n",
      "\t features.denseblock2.denselayer10.conv2.weight\n",
      "\t features.denseblock2.denselayer11.norm1.weight\n",
      "\t features.denseblock2.denselayer11.norm1.bias\n",
      "\t features.denseblock2.denselayer11.conv1.weight\n",
      "\t features.denseblock2.denselayer11.norm2.weight\n",
      "\t features.denseblock2.denselayer11.norm2.bias\n",
      "\t features.denseblock2.denselayer11.conv2.weight\n",
      "\t features.denseblock2.denselayer12.norm1.weight\n",
      "\t features.denseblock2.denselayer12.norm1.bias\n",
      "\t features.denseblock2.denselayer12.conv1.weight\n",
      "\t features.denseblock2.denselayer12.norm2.weight\n",
      "\t features.denseblock2.denselayer12.norm2.bias\n",
      "\t features.denseblock2.denselayer12.conv2.weight\n",
      "\t features.transition2.norm.weight\n",
      "\t features.transition2.norm.bias\n",
      "\t features.transition2.conv.weight\n",
      "\t features.denseblock3.denselayer1.norm1.weight\n",
      "\t features.denseblock3.denselayer1.norm1.bias\n",
      "\t features.denseblock3.denselayer1.conv1.weight\n",
      "\t features.denseblock3.denselayer1.norm2.weight\n",
      "\t features.denseblock3.denselayer1.norm2.bias\n",
      "\t features.denseblock3.denselayer1.conv2.weight\n",
      "\t features.denseblock3.denselayer2.norm1.weight\n",
      "\t features.denseblock3.denselayer2.norm1.bias\n",
      "\t features.denseblock3.denselayer2.conv1.weight\n",
      "\t features.denseblock3.denselayer2.norm2.weight\n",
      "\t features.denseblock3.denselayer2.norm2.bias\n",
      "\t features.denseblock3.denselayer2.conv2.weight\n",
      "\t features.denseblock3.denselayer3.norm1.weight\n",
      "\t features.denseblock3.denselayer3.norm1.bias\n",
      "\t features.denseblock3.denselayer3.conv1.weight\n",
      "\t features.denseblock3.denselayer3.norm2.weight\n",
      "\t features.denseblock3.denselayer3.norm2.bias\n",
      "\t features.denseblock3.denselayer3.conv2.weight\n",
      "\t features.denseblock3.denselayer4.norm1.weight\n",
      "\t features.denseblock3.denselayer4.norm1.bias\n",
      "\t features.denseblock3.denselayer4.conv1.weight\n",
      "\t features.denseblock3.denselayer4.norm2.weight\n",
      "\t features.denseblock3.denselayer4.norm2.bias\n",
      "\t features.denseblock3.denselayer4.conv2.weight\n",
      "\t features.denseblock3.denselayer5.norm1.weight\n",
      "\t features.denseblock3.denselayer5.norm1.bias\n",
      "\t features.denseblock3.denselayer5.conv1.weight\n",
      "\t features.denseblock3.denselayer5.norm2.weight\n",
      "\t features.denseblock3.denselayer5.norm2.bias\n",
      "\t features.denseblock3.denselayer5.conv2.weight\n",
      "\t features.denseblock3.denselayer6.norm1.weight\n",
      "\t features.denseblock3.denselayer6.norm1.bias\n",
      "\t features.denseblock3.denselayer6.conv1.weight\n",
      "\t features.denseblock3.denselayer6.norm2.weight\n",
      "\t features.denseblock3.denselayer6.norm2.bias\n",
      "\t features.denseblock3.denselayer6.conv2.weight\n",
      "\t features.denseblock3.denselayer7.norm1.weight\n",
      "\t features.denseblock3.denselayer7.norm1.bias\n",
      "\t features.denseblock3.denselayer7.conv1.weight\n",
      "\t features.denseblock3.denselayer7.norm2.weight\n",
      "\t features.denseblock3.denselayer7.norm2.bias\n",
      "\t features.denseblock3.denselayer7.conv2.weight\n",
      "\t features.denseblock3.denselayer8.norm1.weight\n",
      "\t features.denseblock3.denselayer8.norm1.bias\n",
      "\t features.denseblock3.denselayer8.conv1.weight\n",
      "\t features.denseblock3.denselayer8.norm2.weight\n",
      "\t features.denseblock3.denselayer8.norm2.bias\n",
      "\t features.denseblock3.denselayer8.conv2.weight\n",
      "\t features.denseblock3.denselayer9.norm1.weight\n",
      "\t features.denseblock3.denselayer9.norm1.bias\n",
      "\t features.denseblock3.denselayer9.conv1.weight\n",
      "\t features.denseblock3.denselayer9.norm2.weight\n",
      "\t features.denseblock3.denselayer9.norm2.bias\n",
      "\t features.denseblock3.denselayer9.conv2.weight\n",
      "\t features.denseblock3.denselayer10.norm1.weight\n",
      "\t features.denseblock3.denselayer10.norm1.bias\n",
      "\t features.denseblock3.denselayer10.conv1.weight\n",
      "\t features.denseblock3.denselayer10.norm2.weight\n",
      "\t features.denseblock3.denselayer10.norm2.bias\n",
      "\t features.denseblock3.denselayer10.conv2.weight\n",
      "\t features.denseblock3.denselayer11.norm1.weight\n",
      "\t features.denseblock3.denselayer11.norm1.bias\n",
      "\t features.denseblock3.denselayer11.conv1.weight\n",
      "\t features.denseblock3.denselayer11.norm2.weight\n",
      "\t features.denseblock3.denselayer11.norm2.bias\n",
      "\t features.denseblock3.denselayer11.conv2.weight\n",
      "\t features.denseblock3.denselayer12.norm1.weight\n",
      "\t features.denseblock3.denselayer12.norm1.bias\n",
      "\t features.denseblock3.denselayer12.conv1.weight\n",
      "\t features.denseblock3.denselayer12.norm2.weight\n",
      "\t features.denseblock3.denselayer12.norm2.bias\n",
      "\t features.denseblock3.denselayer12.conv2.weight\n",
      "\t features.denseblock3.denselayer13.norm1.weight\n",
      "\t features.denseblock3.denselayer13.norm1.bias\n",
      "\t features.denseblock3.denselayer13.conv1.weight\n",
      "\t features.denseblock3.denselayer13.norm2.weight\n",
      "\t features.denseblock3.denselayer13.norm2.bias\n",
      "\t features.denseblock3.denselayer13.conv2.weight\n",
      "\t features.denseblock3.denselayer14.norm1.weight\n",
      "\t features.denseblock3.denselayer14.norm1.bias\n",
      "\t features.denseblock3.denselayer14.conv1.weight\n",
      "\t features.denseblock3.denselayer14.norm2.weight\n",
      "\t features.denseblock3.denselayer14.norm2.bias\n",
      "\t features.denseblock3.denselayer14.conv2.weight\n",
      "\t features.denseblock3.denselayer15.norm1.weight\n",
      "\t features.denseblock3.denselayer15.norm1.bias\n",
      "\t features.denseblock3.denselayer15.conv1.weight\n",
      "\t features.denseblock3.denselayer15.norm2.weight\n",
      "\t features.denseblock3.denselayer15.norm2.bias\n",
      "\t features.denseblock3.denselayer15.conv2.weight\n",
      "\t features.denseblock3.denselayer16.norm1.weight\n",
      "\t features.denseblock3.denselayer16.norm1.bias\n",
      "\t features.denseblock3.denselayer16.conv1.weight\n",
      "\t features.denseblock3.denselayer16.norm2.weight\n",
      "\t features.denseblock3.denselayer16.norm2.bias\n",
      "\t features.denseblock3.denselayer16.conv2.weight\n",
      "\t features.denseblock3.denselayer17.norm1.weight\n",
      "\t features.denseblock3.denselayer17.norm1.bias\n",
      "\t features.denseblock3.denselayer17.conv1.weight\n",
      "\t features.denseblock3.denselayer17.norm2.weight\n",
      "\t features.denseblock3.denselayer17.norm2.bias\n",
      "\t features.denseblock3.denselayer17.conv2.weight\n",
      "\t features.denseblock3.denselayer18.norm1.weight\n",
      "\t features.denseblock3.denselayer18.norm1.bias\n",
      "\t features.denseblock3.denselayer18.conv1.weight\n",
      "\t features.denseblock3.denselayer18.norm2.weight\n",
      "\t features.denseblock3.denselayer18.norm2.bias\n",
      "\t features.denseblock3.denselayer18.conv2.weight\n",
      "\t features.denseblock3.denselayer19.norm1.weight\n",
      "\t features.denseblock3.denselayer19.norm1.bias\n",
      "\t features.denseblock3.denselayer19.conv1.weight\n",
      "\t features.denseblock3.denselayer19.norm2.weight\n",
      "\t features.denseblock3.denselayer19.norm2.bias\n",
      "\t features.denseblock3.denselayer19.conv2.weight\n",
      "\t features.denseblock3.denselayer20.norm1.weight\n",
      "\t features.denseblock3.denselayer20.norm1.bias\n",
      "\t features.denseblock3.denselayer20.conv1.weight\n",
      "\t features.denseblock3.denselayer20.norm2.weight\n",
      "\t features.denseblock3.denselayer20.norm2.bias\n",
      "\t features.denseblock3.denselayer20.conv2.weight\n",
      "\t features.denseblock3.denselayer21.norm1.weight\n",
      "\t features.denseblock3.denselayer21.norm1.bias\n",
      "\t features.denseblock3.denselayer21.conv1.weight\n",
      "\t features.denseblock3.denselayer21.norm2.weight\n",
      "\t features.denseblock3.denselayer21.norm2.bias\n",
      "\t features.denseblock3.denselayer21.conv2.weight\n",
      "\t features.denseblock3.denselayer22.norm1.weight\n",
      "\t features.denseblock3.denselayer22.norm1.bias\n",
      "\t features.denseblock3.denselayer22.conv1.weight\n",
      "\t features.denseblock3.denselayer22.norm2.weight\n",
      "\t features.denseblock3.denselayer22.norm2.bias\n",
      "\t features.denseblock3.denselayer22.conv2.weight\n",
      "\t features.denseblock3.denselayer23.norm1.weight\n",
      "\t features.denseblock3.denselayer23.norm1.bias\n",
      "\t features.denseblock3.denselayer23.conv1.weight\n",
      "\t features.denseblock3.denselayer23.norm2.weight\n",
      "\t features.denseblock3.denselayer23.norm2.bias\n",
      "\t features.denseblock3.denselayer23.conv2.weight\n",
      "\t features.denseblock3.denselayer24.norm1.weight\n",
      "\t features.denseblock3.denselayer24.norm1.bias\n",
      "\t features.denseblock3.denselayer24.conv1.weight\n",
      "\t features.denseblock3.denselayer24.norm2.weight\n",
      "\t features.denseblock3.denselayer24.norm2.bias\n",
      "\t features.denseblock3.denselayer24.conv2.weight\n",
      "\t features.transition3.norm.weight\n",
      "\t features.transition3.norm.bias\n",
      "\t features.transition3.conv.weight\n",
      "\t features.denseblock4.denselayer1.norm1.weight\n",
      "\t features.denseblock4.denselayer1.norm1.bias\n",
      "\t features.denseblock4.denselayer1.conv1.weight\n",
      "\t features.denseblock4.denselayer1.norm2.weight\n",
      "\t features.denseblock4.denselayer1.norm2.bias\n",
      "\t features.denseblock4.denselayer1.conv2.weight\n",
      "\t features.denseblock4.denselayer2.norm1.weight\n",
      "\t features.denseblock4.denselayer2.norm1.bias\n",
      "\t features.denseblock4.denselayer2.conv1.weight\n",
      "\t features.denseblock4.denselayer2.norm2.weight\n",
      "\t features.denseblock4.denselayer2.norm2.bias\n",
      "\t features.denseblock4.denselayer2.conv2.weight\n",
      "\t features.denseblock4.denselayer3.norm1.weight\n",
      "\t features.denseblock4.denselayer3.norm1.bias\n",
      "\t features.denseblock4.denselayer3.conv1.weight\n",
      "\t features.denseblock4.denselayer3.norm2.weight\n",
      "\t features.denseblock4.denselayer3.norm2.bias\n",
      "\t features.denseblock4.denselayer3.conv2.weight\n",
      "\t features.denseblock4.denselayer4.norm1.weight\n",
      "\t features.denseblock4.denselayer4.norm1.bias\n",
      "\t features.denseblock4.denselayer4.conv1.weight\n",
      "\t features.denseblock4.denselayer4.norm2.weight\n",
      "\t features.denseblock4.denselayer4.norm2.bias\n",
      "\t features.denseblock4.denselayer4.conv2.weight\n",
      "\t features.denseblock4.denselayer5.norm1.weight\n",
      "\t features.denseblock4.denselayer5.norm1.bias\n",
      "\t features.denseblock4.denselayer5.conv1.weight\n",
      "\t features.denseblock4.denselayer5.norm2.weight\n",
      "\t features.denseblock4.denselayer5.norm2.bias\n",
      "\t features.denseblock4.denselayer5.conv2.weight\n",
      "\t features.denseblock4.denselayer6.norm1.weight\n",
      "\t features.denseblock4.denselayer6.norm1.bias\n",
      "\t features.denseblock4.denselayer6.conv1.weight\n",
      "\t features.denseblock4.denselayer6.norm2.weight\n",
      "\t features.denseblock4.denselayer6.norm2.bias\n",
      "\t features.denseblock4.denselayer6.conv2.weight\n",
      "\t features.denseblock4.denselayer7.norm1.weight\n",
      "\t features.denseblock4.denselayer7.norm1.bias\n",
      "\t features.denseblock4.denselayer7.conv1.weight\n",
      "\t features.denseblock4.denselayer7.norm2.weight\n",
      "\t features.denseblock4.denselayer7.norm2.bias\n",
      "\t features.denseblock4.denselayer7.conv2.weight\n",
      "\t features.denseblock4.denselayer8.norm1.weight\n",
      "\t features.denseblock4.denselayer8.norm1.bias\n",
      "\t features.denseblock4.denselayer8.conv1.weight\n",
      "\t features.denseblock4.denselayer8.norm2.weight\n",
      "\t features.denseblock4.denselayer8.norm2.bias\n",
      "\t features.denseblock4.denselayer8.conv2.weight\n",
      "\t features.denseblock4.denselayer9.norm1.weight\n",
      "\t features.denseblock4.denselayer9.norm1.bias\n",
      "\t features.denseblock4.denselayer9.conv1.weight\n",
      "\t features.denseblock4.denselayer9.norm2.weight\n",
      "\t features.denseblock4.denselayer9.norm2.bias\n",
      "\t features.denseblock4.denselayer9.conv2.weight\n",
      "\t features.denseblock4.denselayer10.norm1.weight\n",
      "\t features.denseblock4.denselayer10.norm1.bias\n",
      "\t features.denseblock4.denselayer10.conv1.weight\n",
      "\t features.denseblock4.denselayer10.norm2.weight\n",
      "\t features.denseblock4.denselayer10.norm2.bias\n",
      "\t features.denseblock4.denselayer10.conv2.weight\n",
      "\t features.denseblock4.denselayer11.norm1.weight\n",
      "\t features.denseblock4.denselayer11.norm1.bias\n",
      "\t features.denseblock4.denselayer11.conv1.weight\n",
      "\t features.denseblock4.denselayer11.norm2.weight\n",
      "\t features.denseblock4.denselayer11.norm2.bias\n",
      "\t features.denseblock4.denselayer11.conv2.weight\n",
      "\t features.denseblock4.denselayer12.norm1.weight\n",
      "\t features.denseblock4.denselayer12.norm1.bias\n",
      "\t features.denseblock4.denselayer12.conv1.weight\n",
      "\t features.denseblock4.denselayer12.norm2.weight\n",
      "\t features.denseblock4.denselayer12.norm2.bias\n",
      "\t features.denseblock4.denselayer12.conv2.weight\n",
      "\t features.denseblock4.denselayer13.norm1.weight\n",
      "\t features.denseblock4.denselayer13.norm1.bias\n",
      "\t features.denseblock4.denselayer13.conv1.weight\n",
      "\t features.denseblock4.denselayer13.norm2.weight\n",
      "\t features.denseblock4.denselayer13.norm2.bias\n",
      "\t features.denseblock4.denselayer13.conv2.weight\n",
      "\t features.denseblock4.denselayer14.norm1.weight\n",
      "\t features.denseblock4.denselayer14.norm1.bias\n",
      "\t features.denseblock4.denselayer14.conv1.weight\n",
      "\t features.denseblock4.denselayer14.norm2.weight\n",
      "\t features.denseblock4.denselayer14.norm2.bias\n",
      "\t features.denseblock4.denselayer14.conv2.weight\n",
      "\t features.denseblock4.denselayer15.norm1.weight\n",
      "\t features.denseblock4.denselayer15.norm1.bias\n",
      "\t features.denseblock4.denselayer15.conv1.weight\n",
      "\t features.denseblock4.denselayer15.norm2.weight\n",
      "\t features.denseblock4.denselayer15.norm2.bias\n",
      "\t features.denseblock4.denselayer15.conv2.weight\n",
      "\t features.denseblock4.denselayer16.norm1.weight\n",
      "\t features.denseblock4.denselayer16.norm1.bias\n",
      "\t features.denseblock4.denselayer16.conv1.weight\n",
      "\t features.denseblock4.denselayer16.norm2.weight\n",
      "\t features.denseblock4.denselayer16.norm2.bias\n",
      "\t features.denseblock4.denselayer16.conv2.weight\n",
      "\t features.norm5.weight\n",
      "\t features.norm5.bias\n",
      "\t classifier.weight\n",
      "\t classifier.bias\n"
     ]
    }
   ],
   "source": [
    "# Send the model to GPU\n",
    "model_ft = model_ft.to(device)\n",
    "\n",
    "# Gather the parameters to be optimized/updated in this run. If we are\n",
    "#  finetuning we will be updating all parameters. However, if we are \n",
    "#  doing feature extract method, we will only update the parameters\n",
    "#  that we have just initialized, i.e. the parameters with requires_grad\n",
    "#  is True.\n",
    "params_to_update = model_ft.parameters()\n",
    "print(\"Params to learn:\")\n",
    "if feature_extract:\n",
    "    params_to_update = []\n",
    "    for name,param in model_ft.named_parameters():\n",
    "        if param.requires_grad == True:\n",
    "            params_to_update.append(param)\n",
    "            print(\"\\t\",name)\n",
    "else:\n",
    "    for name,param in model_ft.named_parameters():\n",
    "        if param.requires_grad == True:\n",
    "            print(\"\\t\",name)\n",
    "\n",
    "# Observe that all parameters are being optimized\n",
    "optimizer_ft = optim.SGD(params_to_update, lr=0.001, momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %tensorboard --logdir=runs --host 0.0.0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "HQI_fRAk7JOv",
    "outputId": "7a492623-d1a4-49b9-ce45-cae667c020e5",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0/99\n",
      "----------\n",
      "train Loss: 0.8986 Acc: 0.7249\n",
      "train Balanced Accuracy: 0.605052015195282 Precision: [0.14877589 0.85013263 0.70062829] Recall: [0.33193277 0.72809882 0.75512445] F1-Score: [0.20546164 0.78439771 0.72685634]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.7795 Acc: 0.8700\n",
      "test Balanced Accuracy: 0.6839048025692911 Precision: [0.44444444 0.86904762 0.918     ] Recall: [0.28915663 0.98983051 0.77272727] F1-Score: [0.35036496 0.92551506 0.83912249]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 1/99\n",
      "----------\n",
      "train Loss: 0.7601 Acc: 0.7420\n",
      "train Balanced Accuracy: 0.6781640447165378 Precision: [0.16688482 0.88381397 0.74706745] Recall: [0.53571429 0.7528042  0.74597365] F1-Score: [0.25449102 0.81306548 0.74652015]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.8575 Acc: 0.8739\n",
      "test Balanced Accuracy: 0.635817389285499 Precision: [0.28125    0.88372093 0.8909427 ] Recall: [0.10843373 0.98757062 0.81144781] F1-Score: [0.15652174 0.93276414 0.84933921]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 2/99\n",
      "----------\n",
      "train Loss: 0.7098 Acc: 0.7655\n",
      "train Balanced Accuracy: 0.6912456628552183 Precision: [0.17801418 0.88630159 0.77964834] Recall: [0.52731092 0.78361494 0.76281113] F1-Score: [0.26617179 0.83180106 0.77113784]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5900 Acc: 0.8496\n",
      "test Balanced Accuracy: 0.7602032394830708 Precision: [0.26737968 0.92508143 0.93612335] Recall: [0.60240964 0.96271186 0.71548822] F1-Score: [0.37037037 0.94352159 0.8110687 ]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 3/99\n",
      "----------\n",
      "train Loss: 0.6626 Acc: 0.7743\n",
      "train Balanced Accuracy: 0.7141403160599967 Precision: [0.18134377 0.89618881 0.80227843] Recall: [0.58403361 0.79795542 0.76043192] F1-Score: [0.2767546  0.84422412 0.78079489]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.6415 Acc: 0.9052\n",
      "test Balanced Accuracy: 0.7575703716538241 Precision: [0.55555556 0.93406593 0.89813243] Recall: [0.42168675 0.96045198 0.89057239] F1-Score: [0.47945205 0.94707521 0.89433643]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 4/99\n",
      "----------\n",
      "train Loss: 0.6348 Acc: 0.7906\n",
      "train Balanced Accuracy: 0.7318632360717473 Precision: [0.21433906 0.90110935 0.80446194] Recall: [0.60294118 0.80732642 0.78532211] F1-Score: [0.31625344 0.85164383 0.79477681]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5261 Acc: 0.8566\n",
      "test Balanced Accuracy: 0.801211511571596 Precision: [0.29       0.93771626 0.94343434] Recall: [0.69879518 0.91864407 0.78619529] F1-Score: [0.40989399 0.92808219 0.85766758]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 5/99\n",
      "----------\n",
      "train Loss: 0.6257 Acc: 0.7986\n",
      "train Balanced Accuracy: 0.7373699475268625 Precision: [0.22984836 0.89759876 0.81251185] Recall: [0.60504202 0.8226608  0.78440703] F1-Score: [0.33314054 0.85849756 0.79821212]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4825 Acc: 0.9046\n",
      "test Balanced Accuracy: 0.8196173611870297 Precision: [0.51485149 0.95287356 0.9001692 ] Recall: [0.62650602 0.93672316 0.8956229 ] F1-Score: [0.56521739 0.94472934 0.8978903 ]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 6/99\n",
      "----------\n",
      "train Loss: 0.5949 Acc: 0.8065\n",
      "train Balanced Accuracy: 0.7582063720103402 Precision: [0.23867997 0.90251866 0.82983994] Recall: [0.65336134 0.82422263 0.79703514] F1-Score: [0.34963463 0.86159555 0.8131068 ]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.6855 Acc: 0.9085\n",
      "test Balanced Accuracy: 0.7183638852619861 Precision: [0.63888889 0.93428258 0.8858075 ] Recall: [0.27710843 0.96384181 0.91414141] F1-Score: [0.38655462 0.94883204 0.89975145]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 7/99\n",
      "----------\n",
      "train Loss: 0.5868 Acc: 0.8074\n",
      "train Balanced Accuracy: 0.7529426099985191 Precision: [0.23024316 0.90481333 0.83201543] Recall: [0.63655462 0.83274173 0.78953148] F1-Score: [0.33816964 0.86728281 0.81021692]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5190 Acc: 0.9014\n",
      "test Balanced Accuracy: 0.791520129482147 Precision: [0.51136364 0.94563986 0.89340102] Recall: [0.54216867 0.94350282 0.88888889] F1-Score: [0.52631579 0.94457014 0.89113924]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 8/99\n",
      "----------\n",
      "train Loss: 0.5813 Acc: 0.8151\n",
      "train Balanced Accuracy: 0.760034478242153 Precision: [0.25102881 0.90539917 0.83368141] Recall: [0.6407563  0.83572341 0.80362372] F1-Score: [0.36073329 0.86916716 0.81837667]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5306 Acc: 0.9078\n",
      "test Balanced Accuracy: 0.7756643420110848 Precision: [0.54166667 0.9480226  0.89256198] Recall: [0.46987952 0.9480226  0.90909091] F1-Score: [0.50322581 0.9480226  0.90075063]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 9/99\n",
      "----------\n",
      "train Loss: 0.5514 Acc: 0.8178\n",
      "train Balanced Accuracy: 0.7755361808483077 Precision: [0.26715686 0.90365297 0.83888996] Recall: [0.68697479 0.84296465 0.79666911] F1-Score: [0.38470588 0.87225446 0.81723458]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5006 Acc: 0.9110\n",
      "test Balanced Accuracy: 0.8024699379610549 Precision: [0.56790123 0.94276094 0.91016949] Recall: [0.55421687 0.94915254 0.9040404 ] F1-Score: [0.56097561 0.94594595 0.90709459]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 10/99\n",
      "----------\n",
      "train Loss: 0.5467 Acc: 0.8257\n",
      "train Balanced Accuracy: 0.7748388731317958 Precision: [0.26683502 0.90978244 0.84699349] Recall: [0.66596639 0.84907    0.80948023] F1-Score: [0.38100962 0.87837838 0.82781209]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5562 Acc: 0.9123\n",
      "test Balanced Accuracy: 0.7823093320608807 Precision: [0.55555556 0.94825647 0.90183028] Recall: [0.48192771 0.95254237 0.91245791] F1-Score: [0.51612903 0.95039459 0.90711297]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 11/99\n",
      "----------\n",
      "train Loss: 0.5432 Acc: 0.8222\n",
      "train Balanced Accuracy: 0.779508611184171 Precision: [0.27062706 0.90967446 0.84066565] Recall: [0.68907563 0.84509442 0.80435578] F1-Score: [0.38862559 0.87619608 0.82210999]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4067 Acc: 0.8835\n",
      "test Balanced Accuracy: 0.8468015530706975 Precision: [0.38323353 0.95838288 0.92057762] Recall: [0.77108434 0.91073446 0.85858586] F1-Score: [0.512      0.93395133 0.88850174]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 12/99\n",
      "----------\n",
      "train Loss: 0.5303 Acc: 0.8273\n",
      "train Balanced Accuracy: 0.7801709204645634 Precision: [0.28546256 0.90499849 0.84633116] Recall: [0.68067227 0.85347153 0.80636896] F1-Score: [0.40223464 0.87848009 0.82586692]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5640 Acc: 0.9200\n",
      "test Balanced Accuracy: 0.7852194029785752 Precision: [0.69642857 0.94837262 0.89918699] Recall: [0.46987952 0.95480226 0.93097643] F1-Score: [0.56115108 0.95157658 0.91480562]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 13/99\n",
      "----------\n",
      "train Loss: 0.5390 Acc: 0.8252\n",
      "train Balanced Accuracy: 0.7816689877235451 Precision: [0.26995885 0.90864799 0.84961538] Recall: [0.68907563 0.84736618 0.80856515] F1-Score: [0.38793613 0.87693777 0.82858215]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5280 Acc: 0.9059\n",
      "test Balanced Accuracy: 0.7840274640764736 Precision: [0.51898734 0.95804196 0.8832    ] Recall: [0.4939759  0.92881356 0.92929293] F1-Score: [0.50617284 0.94320138 0.90566038]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 14/99\n",
      "----------\n",
      "train Loss: 0.5097 Acc: 0.8315\n",
      "train Balanced Accuracy: 0.7979360195812486 Precision: [0.29531915 0.91126228 0.85114577] Recall: [0.7289916  0.85588528 0.80893119] F1-Score: [0.4203513  0.88270611 0.82950174]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5467 Acc: 0.9168\n",
      "test Balanced Accuracy: 0.7700701338711005 Precision: [0.77777778 0.95221843 0.87774295] Recall: [0.42168675 0.94576271 0.94276094] F1-Score: [0.546875   0.94897959 0.90909091]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 15/99\n",
      "----------\n",
      "train Loss: 0.5030 Acc: 0.8322\n",
      "train Balanced Accuracy: 0.7920575864666644 Precision: [0.2907679  0.91190692 0.85151748] Recall: [0.70798319 0.85687917 0.8113104 ] F1-Score: [0.41223242 0.88353708 0.83092784]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4647 Acc: 0.9136\n",
      "test Balanced Accuracy: 0.8318511963736932 Precision: [0.63095238 0.95260116 0.89722675] Recall: [0.63855422 0.93107345 0.92592593] F1-Score: [0.63473054 0.94171429 0.91135046]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 16/99\n",
      "----------\n",
      "train Loss: 0.5183 Acc: 0.8360\n",
      "train Balanced Accuracy: 0.7899017507717149 Precision: [0.2970297  0.90914525 0.85714286] Recall: [0.69327731 0.86383643 0.81259151] F1-Score: [0.41587902 0.8859119  0.83427283]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4976 Acc: 0.9193\n",
      "test Balanced Accuracy: 0.8076018506486139 Precision: [0.66176471 0.95747126 0.89423077] Recall: [0.54216867 0.94124294 0.93939394] F1-Score: [0.59602649 0.94928775 0.91625616]\n",
      "Normalized confusion matrix\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 17/99\n",
      "----------\n",
      "train Loss: 0.5102 Acc: 0.8339\n",
      "train Balanced Accuracy: 0.7901212685435518 Precision: [0.28522337 0.91697025 0.85146244] Recall: [0.69747899 0.85773108 0.81515373] F1-Score: [0.40487805 0.88636197 0.83291258]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4687 Acc: 0.9059\n",
      "test Balanced Accuracy: 0.8296792030302343 Precision: [0.52941176 0.9600939  0.89309211] Recall: [0.65060241 0.92429379 0.91414141] F1-Score: [0.58378378 0.94185377 0.90349418]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 18/99\n",
      "----------\n",
      "train Loss: 0.5122 Acc: 0.8322\n",
      "train Balanced Accuracy: 0.795185574924823 Precision: [0.28983051 0.91169811 0.85419081] Recall: [0.71848739 0.8575891  0.80948023] F1-Score: [0.41304348 0.88381621 0.83123473]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4645 Acc: 0.9129\n",
      "test Balanced Accuracy: 0.8151226603612418 Precision: [0.70588235 0.95896835 0.87363495] Recall: [0.57831325 0.92429379 0.94276094] F1-Score: [0.63576159 0.94131185 0.90688259]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 19/99\n",
      "----------\n",
      "train Loss: 0.4958 Acc: 0.8359\n",
      "train Balanced Accuracy: 0.8017134244923049 Precision: [0.29803587 0.90995826 0.86206897] Recall: [0.73319328 0.86667613 0.80527086] F1-Score: [0.42380085 0.88778998 0.8327025 ]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5684 Acc: 0.9052\n",
      "test Balanced Accuracy: 0.7797759089774609 Precision: [0.53246753 0.94078212 0.9       ] Recall: [0.4939759  0.95141243 0.89393939] F1-Score: [0.5125     0.94606742 0.89695946]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 20/99\n",
      "----------\n",
      "train Loss: 0.4876 Acc: 0.8383\n",
      "train Balanced Accuracy: 0.8067224490188062 Precision: [0.29889924 0.91835807 0.85923077] Recall: [0.74159664 0.86085475 0.81771596] F1-Score: [0.42607121 0.88867717 0.83795949]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4742 Acc: 0.9129\n",
      "test Balanced Accuracy: 0.8196846737517897 Precision: [0.75384615 0.96208531 0.86523737] Recall: [0.59036145 0.91751412 0.95117845] F1-Score: [0.66216216 0.93927126 0.90617482]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 21/99\n",
      "----------\n",
      "train Loss: 0.4915 Acc: 0.8409\n",
      "train Balanced Accuracy: 0.8021166660203911 Precision: [0.32511848 0.90928112 0.85755203] Recall: [0.72058824 0.86383643 0.82192533] F1-Score: [0.44807315 0.88597641 0.83936081]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.3754 Acc: 0.8816\n",
      "test Balanced Accuracy: 0.8584350527717216 Precision: [0.38068182 0.96107056 0.92198582] Recall: [0.80722892 0.89265537 0.87542088] F1-Score: [0.51737452 0.92560047 0.89810017]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 22/99\n",
      "----------\n",
      "train Loss: 0.4762 Acc: 0.8386\n",
      "train Balanced Accuracy: 0.8079342809787987 Precision: [0.3042735  0.91190476 0.86451993] Recall: [0.74789916 0.87008377 0.80581991] F1-Score: [0.43256379 0.89050352 0.83413849]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.3917 Acc: 0.9033\n",
      "test Balanced Accuracy: 0.870000414830583 Precision: [0.48529412 0.96437055 0.91267123] Recall: [0.79518072 0.91751412 0.8973064 ] F1-Score: [0.60273973 0.940359   0.9049236 ]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 23/99\n",
      "----------\n",
      "train Loss: 0.4733 Acc: 0.8432\n",
      "train Balanced Accuracy: 0.812182716320592 Precision: [0.30636833 0.91890667 0.86670514] Recall: [0.74789916 0.86397842 0.82467057] F1-Score: [0.43467643 0.89059641 0.84516553]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4379 Acc: 0.9155\n",
      "test Balanced Accuracy: 0.8273629815730422 Precision: [0.66233766 0.96018735 0.8858954 ] Recall: [0.61445783 0.92655367 0.94107744] F1-Score: [0.6375     0.94307073 0.91265306]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 24/99\n",
      "----------\n",
      "train Loss: 0.4678 Acc: 0.8411\n",
      "train Balanced Accuracy: 0.808289497894544 Precision: [0.31546023 0.91138296 0.86378609] Recall: [0.74159664 0.86738606 0.8158858 ] F1-Score: [0.44263323 0.88884039 0.83915294]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4078 Acc: 0.9213\n",
      "test Balanced Accuracy: 0.8560442076472299 Precision: [0.69047619 0.9614486  0.89710611] Recall: [0.69879518 0.9299435  0.93939394] F1-Score: [0.69461078 0.94543366 0.91776316]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 25/99\n",
      "----------\n",
      "train Loss: 0.4877 Acc: 0.8356\n",
      "train Balanced Accuracy: 0.8002255875263825 Precision: [0.2860676  0.91327142 0.86454653] Recall: [0.7289916  0.86568224 0.80600293] F1-Score: [0.41089402 0.88884029 0.83424891]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4993 Acc: 0.9123\n",
      "test Balanced Accuracy: 0.8070981408943426 Precision: [0.65714286 0.96023392 0.87598116] Recall: [0.55421687 0.92768362 0.93939394] F1-Score: [0.60130719 0.94367816 0.90658002]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 26/99\n",
      "----------\n",
      "train Loss: 0.4612 Acc: 0.8486\n",
      "train Balanced Accuracy: 0.8175913243837623 Precision: [0.34125475 0.91507747 0.86529987] Recall: [0.75420168 0.87207156 0.82650073] F1-Score: [0.46989529 0.89305707 0.8454554 ]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4623 Acc: 0.9161\n",
      "test Balanced Accuracy: 0.837550762314223 Precision: [0.675      0.95591647 0.89193548] Recall: [0.65060241 0.93107345 0.93097643] F1-Score: [0.66257669 0.94333143 0.91103789]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 27/99\n",
      "----------\n",
      "train Loss: 0.4657 Acc: 0.8427\n",
      "train Balanced Accuracy: 0.8126165367789797 Precision: [0.32575068 0.91044332 0.86458737] Recall: [0.75210084 0.86894789 0.81680088] F1-Score: [0.45460317 0.88921177 0.84001506]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.6055 Acc: 0.9033\n",
      "test Balanced Accuracy: 0.7617914905642086 Precision: [0.68627451 0.95560748 0.8519084 ] Recall: [0.42168675 0.92429379 0.93939394] F1-Score: [0.52238806 0.93968983 0.89351481]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 28/99\n",
      "----------\n",
      "train Loss: 0.4630 Acc: 0.8505\n",
      "train Balanced Accuracy: 0.820148821147786 Precision: [0.33992467 0.91639393 0.86959031] Recall: [0.75840336 0.87462729 0.82741581] F1-Score: [0.46944083 0.89502361 0.84797899]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5170 Acc: 0.8982\n",
      "test Balanced Accuracy: 0.7842032514008555 Precision: [0.51807229 0.95692666 0.86774194] Recall: [0.51807229 0.92881356 0.90572391] F1-Score: [0.51807229 0.94266055 0.88632619]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 29/99\n",
      "----------\n",
      "train Loss: 0.4708 Acc: 0.8433\n",
      "train Balanced Accuracy: 0.8106123193145348 Precision: [0.32420091 0.91248334 0.8630964 ] Recall: [0.74579832 0.87491126 0.81112738] F1-Score: [0.45194144 0.89330241 0.83630531]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4183 Acc: 0.9142\n",
      "test Balanced Accuracy: 0.8289574665723327 Precision: [0.59770115 0.95702671 0.8990228 ] Recall: [0.62650602 0.93107345 0.92929293] F1-Score: [0.61176471 0.94387171 0.91390728]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 30/99\n",
      "----------\n",
      "train Loss: 0.4586 Acc: 0.8409\n",
      "train Balanced Accuracy: 0.8129980137130626 Precision: [0.3125     0.91860465 0.85909004] Recall: [0.75630252 0.86369445 0.81899707] F1-Score: [0.44226044 0.8903037  0.8385646 ]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4673 Acc: 0.9129\n",
      "test Balanced Accuracy: 0.8196846737517897 Precision: [0.75384615 0.96551724 0.86128049] Recall: [0.59036145 0.91751412 0.95117845] F1-Score: [0.66216216 0.94090382 0.904     ]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 31/99\n",
      "----------\n",
      "train Loss: 0.4466 Acc: 0.8513\n",
      "train Balanced Accuracy: 0.8267101234125406 Precision: [0.34709193 0.91654247 0.87055886] Recall: [0.77731092 0.87320744 0.82961201] F1-Score: [0.47989624 0.89435032 0.84959235]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4168 Acc: 0.9174\n",
      "test Balanced Accuracy: 0.850329423612386 Precision: [0.73076923 0.96117647 0.88170347] Recall: [0.68674699 0.92316384 0.94107744] F1-Score: [0.70807453 0.94178674 0.91042345]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 32/99\n",
      "----------\n",
      "train Loss: 0.4549 Acc: 0.8446\n",
      "train Balanced Accuracy: 0.8193926394587758 Precision: [0.33363719 0.91343998 0.86395998] Recall: [0.76890756 0.86752804 0.82174231] F1-Score: [0.46535283 0.88989222 0.84232248]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4208 Acc: 0.9117\n",
      "test Balanced Accuracy: 0.8513170871009003 Precision: [0.69047619 0.96863691 0.86748844] Recall: [0.69879518 0.90734463 0.94781145] F1-Score: [0.69461078 0.9369895  0.90587289]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 33/99\n",
      "----------\n",
      "train Loss: 0.4320 Acc: 0.8471\n",
      "train Balanced Accuracy: 0.8222540803346409 Precision: [0.35350624 0.91337997 0.86023358] Recall: [0.77310924 0.87136164 0.82229136] F1-Score: [0.48516809 0.89187618 0.84083466]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.3811 Acc: 0.8796\n",
      "test Balanced Accuracy: 0.8706221106534224 Precision: [0.38043478 0.96616541 0.91896552] Recall: [0.84337349 0.87118644 0.8973064 ] F1-Score: [0.52434457 0.91622103 0.90800681]\n",
      "Normalized confusion matrix\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 34/99\n",
      "----------\n",
      "train Loss: 0.4390 Acc: 0.8542\n",
      "train Balanced Accuracy: 0.8226257361204211 Precision: [0.3546169  0.91639538 0.87113501] Recall: [0.75840336 0.87931279 0.83016105] F1-Score: [0.4832664  0.8974712  0.85015462]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5047 Acc: 0.9155\n",
      "test Balanced Accuracy: 0.8189770323661824 Precision: [0.62025316 0.95722543 0.89482201] Recall: [0.59036145 0.93559322 0.93097643] F1-Score: [0.60493827 0.94628571 0.91254125]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 35/99\n",
      "----------\n",
      "train Loss: 0.4316 Acc: 0.8538\n",
      "train Balanced Accuracy: 0.8209737745312111 Precision: [0.33395349 0.91985752 0.87582205] Recall: [0.75420168 0.88002272 0.82869693] F1-Score: [0.46292714 0.89949931 0.85160805]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5224 Acc: 0.9193\n",
      "test Balanced Accuracy: 0.8077863700421729 Precision: [0.73770492 0.95852535 0.88309637] Recall: [0.54216867 0.94011299 0.94107744] F1-Score: [0.625      0.94922989 0.91116544]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 36/99\n",
      "----------\n",
      "train Loss: 0.4456 Acc: 0.8510\n",
      "train Balanced Accuracy: 0.8289348414405868 Precision: [0.34311927 0.91848313 0.87045236] Recall: [0.78571429 0.87349141 0.82759883] F1-Score: [0.47765006 0.89542246 0.84848485]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5232 Acc: 0.9168\n",
      "test Balanced Accuracy: 0.8095501563407085 Precision: [0.6969697  0.9583815  0.88272583] Recall: [0.55421687 0.93672316 0.93771044] F1-Score: [0.61744966 0.94742857 0.90938776]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 37/99\n",
      "----------\n",
      "train Loss: 0.4338 Acc: 0.8510\n",
      "train Balanced Accuracy: 0.8300008552573912 Precision: [0.35572375 0.91464138 0.86925454] Recall: [0.78991597 0.87633111 0.82375549] F1-Score: [0.49054142 0.8950765  0.84589363]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4640 Acc: 0.9232\n",
      "test Balanced Accuracy: 0.8353376526135358 Precision: [0.72222222 0.95852535 0.89710611] Recall: [0.62650602 0.94011299 0.93939394] F1-Score: [0.67096774 0.94922989 0.91776316]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 38/99\n",
      "----------\n",
      "train Loss: 0.4270 Acc: 0.8577\n",
      "train Balanced Accuracy: 0.8309466487186036 Precision: [0.37037037 0.92063492 0.86954034] Recall: [0.77731092 0.8811586  0.83437042] F1-Score: [0.50169492 0.90046431 0.85159242]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4774 Acc: 0.9181\n",
      "test Balanced Accuracy: 0.8241230401260351 Precision: [0.69444444 0.95737327 0.88906752] Recall: [0.60240964 0.93898305 0.93097643] F1-Score: [0.64516129 0.94808899 0.90953947]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 39/99\n",
      "----------\n",
      "train Loss: 0.4414 Acc: 0.8507\n",
      "train Balanced Accuracy: 0.829234355346685 Precision: [0.33753375 0.91788376 0.87369237] Recall: [0.78781513 0.8744853  0.82540264] F1-Score: [0.47258979 0.89565913 0.84886128]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4853 Acc: 0.9193\n",
      "test Balanced Accuracy: 0.8246918164010261 Precision: [0.69444444 0.95642202 0.89320388] Recall: [0.60240964 0.94237288 0.92929293] F1-Score: [0.64516129 0.94934548 0.91089109]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 40/99\n",
      "----------\n",
      "train Loss: 0.4305 Acc: 0.8586\n",
      "train Balanced Accuracy: 0.836587755309742 Precision: [0.36346154 0.91999408 0.87782281] Recall: [0.79411765 0.88328837 0.83235725] F1-Score: [0.49868074 0.90126766 0.85448567]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5405 Acc: 0.8662\n",
      "test Balanced Accuracy: 0.8188865947153334 Precision: [0.33152174 0.9567757  0.90613027] Recall: [0.73493976 0.92542373 0.7962963 ] F1-Score: [0.45692884 0.9408386  0.84767025]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 41/99\n",
      "----------\n",
      "train Loss: 0.4248 Acc: 0.8569\n",
      "train Balanced Accuracy: 0.8322544192645965 Precision: [0.37077535 0.91791155 0.87154908] Recall: [0.78361345 0.8811586  0.83199122] F1-Score: [0.50337382 0.89915966 0.85131086]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.3794 Acc: 0.8534\n",
      "test Balanced Accuracy: 0.8469781196348483 Precision: [0.30493274 0.96214099 0.92146597] Recall: [0.81927711 0.83276836 0.88888889] F1-Score: [0.44444444 0.89279225 0.90488432]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 42/99\n",
      "----------\n",
      "train Loss: 0.4244 Acc: 0.8543\n",
      "train Balanced Accuracy: 0.8351576641657066 Precision: [0.36494253 0.91143695 0.87790584] Recall: [0.80042017 0.88257845 0.82247438] F1-Score: [0.50131579 0.89677559 0.84928659]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4771 Acc: 0.9245\n",
      "test Balanced Accuracy: 0.8284430766494615 Precision: [0.79365079 0.95553022 0.89389068] Recall: [0.60240964 0.94689266 0.93602694] F1-Score: [0.68493151 0.95119183 0.91447368]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 43/99\n",
      "----------\n",
      "train Loss: 0.4214 Acc: 0.8580\n",
      "train Balanced Accuracy: 0.8370461794089391 Precision: [0.36903603 0.91909913 0.87536009] Recall: [0.79621849 0.88073264 0.83418741] F1-Score: [0.50432468 0.89950696 0.85427795]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5026 Acc: 0.9174\n",
      "test Balanced Accuracy: 0.8168365982333717 Precision: [0.73846154 0.95732411 0.88095238] Recall: [0.57831325 0.93785311 0.93434343] F1-Score: [0.64864865 0.94748858 0.90686275]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 44/99\n",
      "----------\n",
      "train Loss: 0.4184 Acc: 0.8553\n",
      "train Balanced Accuracy: 0.8348836374085229 Precision: [0.36724806 0.91667898 0.87244307] Recall: [0.79621849 0.88101661 0.82741581] F1-Score: [0.50265252 0.89849406 0.84933308]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4256 Acc: 0.9341\n",
      "test Balanced Accuracy: 0.859384189729299 Precision: [0.79166667 0.95490417 0.92039801] Recall: [0.68674699 0.95706215 0.93434343] F1-Score: [0.73548387 0.95598194 0.9273183 ]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 45/99\n",
      "----------\n",
      "train Loss: 0.4198 Acc: 0.8568\n",
      "train Balanced Accuracy: 0.8385314531240854 Precision: [0.37475538 0.91714665 0.87302505] Recall: [0.80462185 0.88172654 0.82924597] F1-Score: [0.51134846 0.89908788 0.85057255]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5602 Acc: 0.8585\n",
      "test Balanced Accuracy: 0.8014698295550517 Precision: [0.29842932 0.96047904 0.89925373] Recall: [0.68674699 0.90621469 0.81144781] F1-Score: [0.41605839 0.93255814 0.85309735]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 46/99\n",
      "----------\n",
      "train Loss: 0.4131 Acc: 0.8598\n",
      "train Balanced Accuracy: 0.8435498237804433 Precision: [0.36962751 0.92143175 0.87872381] Recall: [0.81302521 0.88087463 0.83674963] F1-Score: [0.50820749 0.90069686 0.85722321]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.3827 Acc: 0.8835\n",
      "test Balanced Accuracy: 0.8488822378482714 Precision: [0.39375    0.96158612 0.9092437 ] Recall: [0.75903614 0.87683616 0.91077441] F1-Score: [0.51851852 0.91725768 0.91000841]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 47/99\n",
      "----------\n",
      "train Loss: 0.4202 Acc: 0.8583\n",
      "train Balanced Accuracy: 0.8410374898719094 Precision: [0.37671233 0.91708543 0.87641963] Recall: [0.80882353 0.88101661 0.83327233] F1-Score: [0.51401869 0.89868926 0.85430153]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4868 Acc: 0.9225\n",
      "test Balanced Accuracy: 0.8192962227268946 Precision: [0.64       0.95676906 0.90789474] Recall: [0.57831325 0.95028249 0.92929293] F1-Score: [0.60759494 0.95351474 0.91846922]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 48/99\n",
      "----------\n",
      "train Loss: 0.4056 Acc: 0.8608\n",
      "train Balanced Accuracy: 0.8403431204858628 Precision: [0.37389598 0.92129835 0.87770749] Recall: [0.80042017 0.88257845 0.83803075] F1-Score: [0.509699   0.90152284 0.85741035]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4303 Acc: 0.9315\n",
      "test Balanced Accuracy: 0.8533155850016513 Precision: [0.75675676 0.94754464 0.92905405] Recall: [0.6746988  0.95932203 0.92592593] F1-Score: [0.7133758  0.95339697 0.92748735]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 49/99\n",
      "----------\n",
      "train Loss: 0.4145 Acc: 0.8595\n",
      "train Balanced Accuracy: 0.8367899614033639 Precision: [0.37252964 0.92139997 0.87394797] Recall: [0.79201681 0.88215249 0.83620059] F1-Score: [0.50672043 0.90134919 0.85465769]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.3971 Acc: 0.9270\n",
      "test Balanced Accuracy: 0.8659747934269749 Precision: [0.74074074 0.95752009 0.90819672] Recall: [0.72289157 0.94237288 0.93265993] F1-Score: [0.73170732 0.9498861  0.92026578]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 50/99\n",
      "----------\n",
      "train Loss: 0.4052 Acc: 0.8632\n",
      "train Balanced Accuracy: 0.8342757718948858 Precision: [0.38883035 0.91909814 0.87671494] Recall: [0.77521008 0.88556013 0.8420571 ] F1-Score: [0.51789474 0.9020175  0.85903659]\n",
      "Normalized confusion matrix\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test Loss: 0.5516 Acc: 0.9136\n",
      "test Balanced Accuracy: 0.7965131770590895 Precision: [0.54320988 0.95061728 0.90847458] Recall: [0.53012048 0.95706215 0.9023569 ] F1-Score: [0.53658537 0.95382883 0.90540541]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 51/99\n",
      "----------\n",
      "train Loss: 0.4059 Acc: 0.8594\n",
      "train Balanced Accuracy: 0.84279345422404 Precision: [0.36787072 0.92099823 0.87885249] Recall: [0.81302521 0.88556013 0.82979502] F1-Score: [0.5065445 0.9029316 0.8536195]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4484 Acc: 0.9334\n",
      "test Balanced Accuracy: 0.8444498762040115 Precision: [0.81538462 0.95398429 0.91584158] Recall: [0.63855422 0.96045198 0.93434343] F1-Score: [0.71621622 0.95720721 0.925     ]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 52/99\n",
      "----------\n",
      "train Loss: 0.3979 Acc: 0.8595\n",
      "train Balanced Accuracy: 0.8478080796154416 Precision: [0.37994214 0.91908714 0.87783763] Recall: [0.82773109 0.88059066 0.83510249] F1-Score: [0.52081956 0.89942716 0.85593697]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.3901 Acc: 0.9187\n",
      "test Balanced Accuracy: 0.8639797104299181 Precision: [0.66304348 0.96261682 0.89576547] Recall: [0.73493976 0.93107345 0.92592593] F1-Score: [0.69714286 0.94658242 0.91059603]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 53/99\n",
      "----------\n",
      "train Loss: 0.3897 Acc: 0.8599\n",
      "train Balanced Accuracy: 0.8504240184971618 Precision: [0.3747646  0.9198462  0.88117852] Recall: [0.83613445 0.88314639 0.83199122] F1-Score: [0.51755527 0.90112278 0.85587875]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4917 Acc: 0.9206\n",
      "test Balanced Accuracy: 0.820380511946777 Precision: [0.81355932 0.96270396 0.8744186 ] Recall: [0.57831325 0.93333333 0.94949495] F1-Score: [0.67605634 0.94779116 0.91041162]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 54/99\n",
      "----------\n",
      "train Loss: 0.4035 Acc: 0.8584\n",
      "train Balanced Accuracy: 0.8438470729418404 Precision: [0.37189293 0.91936442 0.87718624] Recall: [0.81722689 0.87902882 0.83528551] F1-Score: [0.51116951 0.89874428 0.85572326]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4774 Acc: 0.9213\n",
      "test Balanced Accuracy: 0.8331005927493574 Precision: [0.74285714 0.95756881 0.89032258] Recall: [0.62650602 0.94350282 0.92929293] F1-Score: [0.67973856 0.95048378 0.90939044]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 55/99\n",
      "----------\n",
      "train Loss: 0.3999 Acc: 0.8638\n",
      "train Balanced Accuracy: 0.8454673608601081 Precision: [0.3891129  0.92059517 0.88026139] Recall: [0.81092437 0.88726395 0.83821376] F1-Score: [0.52588556 0.9036223  0.85872316]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4006 Acc: 0.9277\n",
      "test Balanced Accuracy: 0.86635144126125 Precision: [0.6741573  0.96087457 0.91721854] Recall: [0.72289157 0.94350282 0.93265993] F1-Score: [0.69767442 0.95210946 0.92487479]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 56/99\n",
      "----------\n",
      "train Loss: 0.4026 Acc: 0.8611\n",
      "train Balanced Accuracy: 0.8380847511932256 Precision: [0.37363726 0.9193429  0.88000767] Recall: [0.79201681 0.88201051 0.84022694] F1-Score: [0.50774411 0.90028986 0.85965734]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4905 Acc: 0.9251\n",
      "test Balanced Accuracy: 0.8353604797550069 Precision: [0.66666667 0.95459705 0.91542289] Recall: [0.62650602 0.95028249 0.92929293] F1-Score: [0.64596273 0.95243488 0.92230576]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 57/99\n",
      "----------\n",
      "train Loss: 0.3964 Acc: 0.8655\n",
      "train Balanced Accuracy: 0.8442797340960931 Precision: [0.38276553 0.92465956 0.88123924] Recall: [0.80252101 0.88697998 0.84333821] F1-Score: [0.5183175  0.90542793 0.86187225]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.7573 Acc: 0.7708\n",
      "test Balanced Accuracy: 0.7624286336695295 Precision: [0.17827298 0.96423841 0.91964286] Recall: [0.77108434 0.82259887 0.69360269] F1-Score: [0.28959276 0.88780488 0.79078695]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 58/99\n",
      "----------\n",
      "train Loss: 0.3804 Acc: 0.8661\n",
      "train Balanced Accuracy: 0.8509416916485462 Precision: [0.40621762 0.91927198 0.88184438] Recall: [0.82352941 0.88925174 0.84004392] F1-Score: [0.54406662 0.9040127  0.86043678]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5696 Acc: 0.7574\n",
      "test Balanced Accuracy: 0.8204169299465739 Precision: [0.1902439  0.96835443 0.94807692] Recall: [0.93975904 0.69152542 0.82996633] F1-Score: [0.31643002 0.80685564 0.88509874]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 59/99\n",
      "----------\n",
      "train Loss: 0.3904 Acc: 0.8667\n",
      "train Balanced Accuracy: 0.8477035025504164 Precision: [0.3991727  0.92107201 0.88267943] Recall: [0.81092437 0.88811586 0.84407028] F1-Score: [0.53499653 0.90429377 0.86294321]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.3957 Acc: 0.9251\n",
      "test Balanced Accuracy: 0.8694068633146981 Precision: [0.72619048 0.96378505 0.89871383] Recall: [0.73493976 0.93220339 0.94107744] F1-Score: [0.73053892 0.94773119 0.91940789]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 60/99\n",
      "----------\n",
      "train Loss: 0.3762 Acc: 0.8659\n",
      "train Balanced Accuracy: 0.8521876726016209 Precision: [0.401222   0.91887125 0.88435636] Recall: [0.82773109 0.8876899  0.84114202] F1-Score: [0.54046639 0.90301148 0.86220805]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4716 Acc: 0.9091\n",
      "test Balanced Accuracy: 0.8580945020738091 Precision: [0.50819672 0.96060255 0.91681109] Recall: [0.74698795 0.93672316 0.89057239] F1-Score: [0.60487805 0.94851259 0.90350128]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 61/99\n",
      "----------\n",
      "train Loss: 0.3879 Acc: 0.8639\n",
      "train Balanced Accuracy: 0.8475145757190009 Precision: [0.37512054 0.92284954 0.88474903] Recall: [0.81722689 0.88655403 0.83876281] F1-Score: [0.51421018 0.90433775 0.86114243]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.4940 Acc: 0.9309\n",
      "test Balanced Accuracy: 0.8507756758930948 Precision: [0.79710145 0.95785877 0.90731707] Recall: [0.6626506  0.95028249 0.93939394] F1-Score: [0.72368421 0.95405559 0.92307692]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 62/99\n",
      "----------\n",
      "train Loss: 0.3828 Acc: 0.8637\n",
      "train Balanced Accuracy: 0.8445654716665915 Precision: [0.4004171  0.92049156 0.87533207] Recall: [0.80672269 0.88272043 0.84425329] F1-Score: [0.53519164 0.90121041 0.85951183]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5297 Acc: 0.7247\n",
      "test Balanced Accuracy: 0.7932225850270912 Precision: [0.16561845 0.97191888 0.96846847] Recall: [0.95180723 0.7039548  0.72390572] F1-Score: [0.28214286 0.81651376 0.82851638]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 63/99\n",
      "----------\n",
      "train Loss: 0.3860 Acc: 0.8634\n",
      "train Balanced Accuracy: 0.846085280154434 Precision: [0.38339921 0.9184719  0.88480249] Recall: [0.81512605 0.89095556 0.83217423] F1-Score: [0.52150538 0.9045045  0.85768179]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.3820 Acc: 0.9014\n",
      "test Balanced Accuracy: 0.873247965324747 Precision: [0.4527027  0.95828367 0.93391304] Recall: [0.80722892 0.90847458 0.9040404 ] F1-Score: [0.58008658 0.93271462 0.91873396]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 64/99\n",
      "----------\n",
      "train Loss: 0.3913 Acc: 0.8655\n",
      "train Balanced Accuracy: 0.8459103402360567 Precision: [0.38308458 0.92312224 0.88357749] Recall: [0.80882353 0.88996166 0.83894583] F1-Score: [0.51991897 0.9062387  0.86068344]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.5077 Acc: 0.9257\n",
      "test Balanced Accuracy: 0.8173555260819573 Precision: [0.734375   0.95280899 0.90625   ] Recall: [0.56626506 0.95819209 0.92760943] F1-Score: [0.63945578 0.95549296 0.91680532]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 65/99\n",
      "----------\n",
      "train Loss: 0.3874 Acc: 0.8669\n",
      "train Balanced Accuracy: 0.8483690486673492 Precision: [0.38392857 0.92319033 0.88713405] Recall: [0.81302521 0.88910975 0.84297218] F1-Score: [0.52156334 0.9058296  0.86448949]\n",
      "Normalized confusion matrix\n",
      "test Loss: 0.6981 Acc: 0.7919\n",
      "test Balanced Accuracy: 0.783746548139714 Precision: [0.19822485 0.9660804  0.93691589] Recall: [0.80722892 0.86892655 0.67508418] F1-Score: [0.31828979 0.91493159 0.78473581]\n",
      "Normalized confusion matrix\n",
      "\n",
      "Epoch 66/99\n",
      "----------\n",
      "train Loss: 0.3782 Acc: 0.8664\n",
      "train Balanced Accuracy: 0.8535412030152537 Precision: [0.40142276 0.92365846 0.88006853] Recall: [0.82983193 0.88470822 0.84608346] F1-Score: [0.54109589 0.90376387 0.86274144]\n",
      "Normalized confusion matrix\n"
     ]
    }
   ],
   "source": [
    "weights = torch.Tensor(weights).to(device)\n",
    "\n",
    "# Setup the loss fxn\n",
    "criterion = nn.CrossEntropyLoss(weight=weights)\n",
    "\n",
    "# Train and evaluate\n",
    "model_ft, hist = train_model(model_ft, dataloaders_dict, criterion, optimizer_ft, num_epochs=num_epochs, is_inception=(model_name==\"inception\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "adh0gzugl1nT"
   },
   "outputs": [],
   "source": [
    "torch.save(model_ft.state_dict(), \"covid/xrayvision_ft.pt\")"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "fine_tunning_covid.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}