adv4_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torchvision.datasets import STL10, CIFAR10, CIFAR100, ImageFolder, ImageNet\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "from datasets.imagenette import ImageNette\n",
    "from hash_classifier import use_imagenet_categories, balance_classes\n",
    "\n",
    "sns.set_style(\"ticks\", {'axes.grid': True})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "VAL_DATASET = 'imagenet_val'\n",
    "TRAIN_DATASET = 'imagenet_train'\n",
    "NUM_CLASSES = 1000\n",
    "BALANCE_CLASSES = True\n",
    "USE_NORMALIZED_BIT_VALUES = False\n",
    "SEEDS = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n",
    "MODEL_BASE_PATH_CATEGORY_MODELS = './pretrained_models/category_classifier_models'\n",
    "MODEL_BASE_PATH_CLASS_MODELS = './pretrained_models/class_classifier_models'\n",
    "TRAIN_HASH_DIR = f'dataset_hashes/{TRAIN_DATASET}/{TRAIN_DATASET}_original_with_targets.csv'\n",
    "VAL_HASH_DIR = f'dataset_hashes/{VAL_DATASET}/{VAL_DATASET}_original_with_targets.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the dataset and the calcualted hashes from the csv files\n",
    "def get_dataset(dataset_name: str, additional_transforms=None):\n",
    "    if dataset_name.lower() == 'stl10':\n",
    "        dataset = STL10(root='data', split='train', download=True)\n",
    "    elif dataset_name.lower() == 'imagenette':\n",
    "        dataset = ImageNette(root='data', train=True, download=True)\n",
    "    elif dataset_name.lower() == 'cifar10':\n",
    "        dataset = CIFAR10(root='data', train=True, download=True)\n",
    "    elif dataset_name.lower() == 'cifar100':\n",
    "        dataset = CIFAR100(root='data', train=True, download=True)\n",
    "    elif dataset_name.lower() == 'imagenet_test':\n",
    "        dataset = ImageFolder(root='data/ILSVRC2012_test')\n",
    "    elif dataset_name.lower() == 'imagenet_train':\n",
    "        dataset = ImageNet(root='data/ILSVRC2012', split='train')\n",
    "    elif dataset_name.lower() == 'imagenet_val':\n",
    "        dataset = ImageNet(root='data/ILSVRC2012', split='val')\n",
    "    else:\n",
    "        raise RuntimeError(f'Dataset with name {dataset_name} was not found.')\n",
    "    return dataset\n",
    "\n",
    "\n",
    "def get_prediction_vecs(model, dataloader):\n",
    "    # get the predictions for the validation set\n",
    "    predictions = []\n",
    "    with torch.no_grad():\n",
    "        for x, y in dataloader:\n",
    "            x, y = x.cuda(), y.cuda()\n",
    "            predictions.append(model(x).softmax(1).cpu())\n",
    "\n",
    "    predictions = torch.cat(predictions)\n",
    "\n",
    "    return predictions\n",
    "\n",
    "\n",
    "def topk_accuracy(output, target, topk=(1,)):\n",
    "    \"\"\"\n",
    "    Computes the accuracy over the k top predictions for the specified values of k.\n",
    "    Taken from the PyTorch ImageNet example (https://github.com/pytorch/examples/blob/master/imagenet/main.py)\n",
    "    and slightly modified.\n",
    "    \"\"\"\n",
    "    with torch.no_grad():\n",
    "        maxk = max(topk)\n",
    "        batch_size = target.size(0)\n",
    "\n",
    "        _, pred = output.topk(maxk, 1, True, True)\n",
    "        pred = pred.T\n",
    "        correct = pred == target.view(1, -1).expand_as(pred)\n",
    "\n",
    "        res = []\n",
    "        for k in topk:\n",
    "            correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True)\n",
    "            res.append(correct_k.mul_(100.0 / batch_size))\n",
    "        return res\n",
    "\n",
    "\n",
    "def get_acc_per_class(targets, predictions):\n",
    "    # get the accuracy per class\n",
    "    with torch.no_grad():\n",
    "        accuracy_per_class = []\n",
    "        for target in np.unique(targets):\n",
    "            class_indices = (torch.tensor(targets) == target).nonzero().flatten()\n",
    "            class_predictions = predictions[class_indices]\n",
    "            top1 = topk_accuracy(class_predictions, torch.tensor(targets[class_indices]), topk=(1,))[0]\n",
    "\n",
    "            accuracy_per_class.append(top1)\n",
    "\n",
    "        accuracy_per_class = torch.cat(accuracy_per_class)\n",
    "    return accuracy_per_class\n",
    "\n",
    "# get the training data\n",
    "train_df = pd.read_csv(TRAIN_HASH_DIR)\n",
    "# get the validation data\n",
    "val_df = pd.read_csv(VAL_HASH_DIR)\n",
    "val_dataset = get_dataset(VAL_DATASET)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_targets = val_df['target'].to_numpy()\n",
    "val_vecs = np.array([[int(bit) for bit in list(hash)] for hash in val_df['hash_bin'].to_numpy()])\n",
    "if USE_NORMALIZED_BIT_VALUES:\n",
    "    val_vecs = (val_vecs - 0.5) * 2\n",
    "val_indices = np.concatenate([np.where(val_targets == i) for i in range(NUM_CLASSES)], axis=1).flatten()\n",
    "val_set = torch.utils.data.TensorDataset(torch.tensor(val_vecs).float(), torch.tensor(val_targets))\n",
    "val_loader = torch.utils.data.DataLoader(val_set, batch_size=256, num_workers=8, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the pretrained classifier\n",
    "# for the vanilla hash classifier a dropout of 0.2 was used\n",
    "neuron_nums = [96, 2048, 4096, 2048, NUM_CLASSES]\n",
    "layers = []\n",
    "for i in range(1, len(neuron_nums)):\n",
    "    layers.extend([\n",
    "        nn.Linear(neuron_nums[i - 1], neuron_nums[i]),\n",
    "        nn.BatchNorm1d(neuron_nums[i]),\n",
    "        nn.Dropout(p=0),\n",
    "        nn.ReLU()\n",
    "    ])\n",
    "\n",
    "nn_classifier = nn.Sequential(\n",
    "    *layers\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the predictions for the validation set of each model trained with a different seed\n",
    "class_classifier_prediction_dict = {}\n",
    "for seed in SEEDS:\n",
    "    model_file_name = f'imagenet_train_hash_classifier_stratified{\"_balanced\" if BALANCE_CLASSES else \"\"}_seed{seed}.pt'\n",
    "    model_path = os.path.join(MODEL_BASE_PATH_CLASS_MODELS, model_file_name)\n",
    "    nn_classifier.load_state_dict(torch.load(model_path))\n",
    "    nn_classifier = nn_classifier.cuda()\n",
    "    nn_classifier.eval()\n",
    "    class_classifier_prediction_dict[seed] = get_prediction_vecs(nn_classifier, val_loader)\n",
    "    nn_classifier.cpu()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class_classifier_accs =  {\n",
    "    'top1': [],\n",
    "    'top5': [],\n",
    "    'top10': []\n",
    "}\n",
    "for seed in SEEDS:\n",
    "    # calculate the top-1, top-5 and top-10 accuracy\n",
    "    class_classifier_accs['top1'].append(topk_accuracy(class_classifier_prediction_dict[seed], torch.tensor(val_targets), topk=[1])[0].item())\n",
    "    class_classifier_accs['top5'].append(topk_accuracy(class_classifier_prediction_dict[seed], torch.tensor(val_targets), topk=[5])[0].item())\n",
    "    class_classifier_accs['top10'].append(topk_accuracy(class_classifier_prediction_dict[seed], torch.tensor(val_targets), topk=[10])[0].item())\n",
    "    print(f'Seed {seed}: ({class_classifier_accs[\"top1\"][-1]}, {class_classifier_accs[\"top5\"][-1]}, {class_classifier_accs[\"top10\"][-1]})')\n",
    "\n",
    "for key in class_classifier_accs.keys():\n",
    "    class_classifier_accs[key] = np.array(class_classifier_accs[key])\n",
    "\n",
    "print(f'Top1-Mean-Accuracy: {class_classifier_accs[\"top1\"].mean():5.4f}%')\n",
    "print(f'Top5-Mean-Accuracy: {class_classifier_accs[\"top5\"].mean():5.4f}%')\n",
    "print(f'Top10-Mean-Accuracy: {class_classifier_accs[\"top10\"].mean():5.4f}%')\n",
    "print(f'Top1-Mean-Standard-Deviation: {class_classifier_accs[\"top1\"].std():5.4f}%')\n",
    "print(f'Top5-Mean-Standard-Deviation: {class_classifier_accs[\"top5\"].std():5.4f}%')\n",
    "print(f'Top10-Mean-Standard-Deviation: {class_classifier_accs[\"top10\"].std():5.4f}%')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "accuracy_per_class = get_acc_per_class(val_targets, class_classifier_prediction_dict[SEEDS[0]])\n",
    "\n",
    "# print the 5 classes with the best accuracy\n",
    "top5_classes_acc, top5_classes_idx = torch.topk(accuracy_per_class, 10)\n",
    "\n",
    "print('Classes with the best prediction:')\n",
    "for acc, idx in zip(top5_classes_acc, top5_classes_idx):\n",
    "    print(f'{val_dataset.classes[idx]}: {acc:5.4f}%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Group the Imagenet Classes in Categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "categories = pd.read_csv('./imagenet_categories_modified.csv')\n",
    "\n",
    "category_val_df = val_df.copy(deep=True)\n",
    "\n",
    "use_imagenet_categories(val_dataset, train_df)\n",
    "use_imagenet_categories(val_dataset, category_val_df, modify_dataset=True)\n",
    "\n",
    "category_val_targets = category_val_df['target'].to_numpy()\n",
    "category_val_vecs = np.array([[int(bit) for bit in list(hash)] for hash in category_val_df['hash_bin'].to_numpy()])\n",
    "if USE_NORMALIZED_BIT_VALUES:\n",
    "    category_val_vecs = (category_val_vecs - 0.5) * 2\n",
    "\n",
    "def get_category_data_indices(train_df, val_df, targets):\n",
    "    if BALANCE_CLASSES:\n",
    "        train_indices = balance_classes(train_df)\n",
    "        val_indices = balance_classes(val_df)\n",
    "    else:\n",
    "        val_indices = [np.where(targets == i)[0] for i in range(NUM_CLASSES)]\n",
    "\n",
    "    val_indices = np.concatenate(val_indices).flatten()\n",
    "\n",
    "    return val_indices\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the pretrained category classifier\n",
    "# for the category hash classifier a dropout of 0.3 was used\n",
    "neuron_nums = [96, 2048, 4096, 2048, 85]\n",
    "layers = []\n",
    "for i in range(1, len(neuron_nums)):\n",
    "    layers.extend([\n",
    "        nn.Linear(neuron_nums[i - 1], neuron_nums[i]),\n",
    "        nn.BatchNorm1d(neuron_nums[i]),\n",
    "        nn.Dropout(p=0),\n",
    "        nn.ReLU()\n",
    "    ])\n",
    "\n",
    "nn_category_classifier = nn.Sequential(\n",
    "    *layers\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# get the predictions for the validation set of each model trained with a different seed\n",
    "category_classifier_prediction_dict = {}\n",
    "for seed in SEEDS:\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    val_indices = get_category_data_indices(train_df, category_val_df, category_val_targets)\n",
    "    current_category_val_set = torch.utils.data.TensorDataset(torch.tensor(category_val_vecs[val_indices]).float(), torch.tensor(category_val_targets[val_indices]))\n",
    "    current_category_val_loader = torch.utils.data.DataLoader(current_category_val_set, batch_size=256, num_workers=8, shuffle=False)\n",
    "\n",
    "    model_file_name = f'imagenet_train_categories_hash_classifier_stratified{\"_balanced\" if BALANCE_CLASSES else \"\"}_seed{seed}.pt'\n",
    "    model_path = os.path.join(MODEL_BASE_PATH_CATEGORY_MODELS, model_file_name)\n",
    "    nn_category_classifier.load_state_dict(torch.load(model_path))\n",
    "    nn_category_classifier = nn_category_classifier.cuda()\n",
    "    nn_category_classifier.eval()\n",
    "    category_classifier_prediction_dict[seed] = get_prediction_vecs(nn_category_classifier, current_category_val_loader)\n",
    "    nn_category_classifier.cpu()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "category_classifier_accs =  {\n",
    "    'top1': [],\n",
    "    'top5': [],\n",
    "    'top10': []\n",
    "}\n",
    "for seed in SEEDS:\n",
    "    # calculate the top-1, top-5 and top-10 accuracy\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    val_indices = get_category_data_indices(train_df, category_val_df, category_val_targets)\n",
    "\n",
    "    category_classifier_accs['top1'].append(topk_accuracy(category_classifier_prediction_dict[seed], torch.tensor(category_val_targets[val_indices]), topk=[1])[0].item())\n",
    "    category_classifier_accs['top5'].append(topk_accuracy(category_classifier_prediction_dict[seed], torch.tensor(category_val_targets[val_indices]), topk=[5])[0].item())\n",
    "    category_classifier_accs['top10'].append(topk_accuracy(category_classifier_prediction_dict[seed], torch.tensor(category_val_targets[val_indices]), topk=[10])[0].item())\n",
    "    print(f'Seed {seed}: ({category_classifier_accs[\"top1\"][-1]}, {category_classifier_accs[\"top5\"][-1]}, {category_classifier_accs[\"top10\"][-1]})')\n",
    "\n",
    "for key in category_classifier_accs.keys():\n",
    "    category_classifier_accs[key] = np.array(category_classifier_accs[key])\n",
    "\n",
    "print(f'Top1-Mean-Accuracy: {category_classifier_accs[\"top1\"].mean():5.4f}%')\n",
    "print(f'Top5-Mean-Accuracy: {category_classifier_accs[\"top5\"].mean():5.4f}%')\n",
    "print(f'Top10-Mean-Accuracy: {category_classifier_accs[\"top10\"].mean():5.4f}%')\n",
    "print(f'Top1-Mean-Standard-Deviation: {category_classifier_accs[\"top1\"].std():5.4f}%')\n",
    "print(f'Top5-Mean-Standard-Deviation: {category_classifier_accs[\"top5\"].std():5.4f}%')\n",
    "print(f'Top10-Mean-Standard-Deviation: {category_classifier_accs[\"top10\"].std():5.4f}%')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = SEEDS[0]\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "val_indices = get_category_data_indices(train_df, category_val_df, category_val_targets)\n",
    "accuracy_per_category = get_acc_per_class(category_val_targets[val_indices], category_classifier_prediction_dict[seed])\n",
    "\n",
    "# print the 5 classes with the best accuracy\n",
    "top5_category_acc, top5_category_idx = torch.topk(accuracy_per_category, 10)\n",
    "\n",
    "print('Categories with the best prediction:')\n",
    "for acc, idx in zip(top5_category_acc, top5_category_idx):\n",
    "    print(f'{val_dataset.classes[idx]}: {acc:5.4f}%')\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "d4d1e4263499bec80672ea0156c357c1ee493ec2b1c70f0acce89fc37c4a6abe"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}