Differential_Privacy_Face_Models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Differential Privacy for Vision Tasks"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Settings und Imports"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# suppress warnings\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "#autoreload other packages when code changed\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "torch.manual_seed(20)  #Reproduzierbarkeit\n",
    "from torch import nn\n",
    "from torch.utils.data import DataLoader\n",
    "import torchvision\n",
    "\n",
    "import opacus\n",
    "from opacus import PrivacyEngine\n",
    "from opacus.validators import ModuleValidator\n",
    "from opacus.utils.batch_memory_manager import BatchMemoryManager\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "import gc\n",
    "import os"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "#Own Code\n",
    "from privacyflow.configs import path_configs\n",
    "from privacyflow.datasets import faces_dataset\n",
    "from privacyflow.models import face_models"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "#Check if GPU is available\n",
    "if torch.cuda.is_available():\n",
    "    print(\"GPU will be used\")\n",
    "    device = torch.device('cuda')\n",
    "else:\n",
    "    print(\"No GPU available\")\n",
    "    device = torch.device('cpu')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Data Prep"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "label_columns = 'all'  #40 attributes\n",
    "\n",
    "data_augmentation_train = torchvision.transforms.Compose([\n",
    "    #torchvision.transforms.AutoAugment(),\n",
    "    torchvision.transforms.ToTensor(),\n",
    "])\n",
    "\n",
    "data_augmentation_test = torchvision.transforms.Compose([\n",
    "    torchvision.transforms.ToTensor()\n",
    "])\n",
    "\n",
    "data_augmentation_train_with_resize = torchvision.transforms.Compose([\n",
    "    torchvision.transforms.Resize((224, 224)),\n",
    "    #torchvision.transforms.AutoAugment(),\n",
    "    torchvision.transforms.ToTensor(),\n",
    "])\n",
    "\n",
    "data_augmentation_test_with_resize = torchvision.transforms.Compose([\n",
    "    torchvision.transforms.Resize((224, 224)),\n",
    "    torchvision.transforms.ToTensor()\n",
    "])\n",
    "\n",
    "\n",
    "train_dataset_cnn = faces_dataset.FacesDataset(label_cols=label_columns, mode=\"train\",transform=data_augmentation_train)\n",
    "val_dataset_cnn = faces_dataset.FacesDataset(label_cols=label_columns, mode=\"val\", transform=data_augmentation_test)\n",
    "test_dataset_cnn = faces_dataset.FacesDataset(label_cols=label_columns, mode=\"test\", transform=data_augmentation_test)\n",
    "\n",
    "train_dataset_vit = faces_dataset.FacesDataset(label_cols=label_columns, mode=\"train\",transform=data_augmentation_train_with_resize)\n",
    "val_dataset_vit = faces_dataset.FacesDataset(label_cols=label_columns, mode=\"val\",transform=data_augmentation_test_with_resize)\n",
    "test_dataset_vit = faces_dataset.FacesDataset(label_cols=label_columns, mode=\"test\",transform=data_augmentation_test_with_resize)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Models - No DP "
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "The following section contains code for training and testing the base models, which are trained without DPSGD"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def train_model(model:nn.Module,\n",
    "                criterion:nn.Module,\n",
    "                optimizer:torch.optim.Optimizer,\n",
    "                epochs:int,\n",
    "                train_ds:torch.utils.data.Dataset,\n",
    "                val_ds:torch.utils.data.Dataset,\n",
    "                batch_size:int =32,\n",
    "                num_workers:int=0,\n",
    "                amount_labels:int=40,\n",
    "                val:bool=True):\n",
    "    torch.cuda.empty_cache()\n",
    "    train_dl = DataLoader(train_ds, batch_size=batch_size,shuffle=True,num_workers=num_workers)\n",
    "    for epoch in range(epochs):\n",
    "        model.train()\n",
    "        epoch_loss = 0.0\n",
    "        for model_inputs, labels in tqdm(train_dl, leave=False):\n",
    "            model_inputs = model_inputs.to(device)\n",
    "            labels = labels.to(device)\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            model_outputs = model(model_inputs)\n",
    "            loss = criterion(model_outputs, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            epoch_loss += loss.item()\n",
    "        print(f\"Epoch: {epoch + 1:2}\",\n",
    "              f\"Train Loss: {epoch_loss / len(train_dl):.5f}\")\n",
    "\n",
    "        if val:\n",
    "            torch.cuda.empty_cache()\n",
    "            gc.collect() #free cuda memory from train_dataloader\n",
    "            val_dl = DataLoader(val_ds, batch_size=batch_size, shuffle=False,num_workers=num_workers)\n",
    "            val_loss = 0.0\n",
    "            num_corrects = 0\n",
    "            model.eval()\n",
    "            for model_inputs, labels in val_dl:\n",
    "                model_inputs = model_inputs.to(device)\n",
    "                labels = labels.to(device)\n",
    "                model_outputs = model(model_inputs)\n",
    "                loss = criterion(model_outputs, labels)\n",
    "                val_loss += loss.item()\n",
    "                num_corrects += int((model_outputs.round() == labels).sum())\n",
    "            print(f\"Val Loss: {val_loss / len(val_dl):.5f}\",\n",
    "                  f\"Val Accuracy (all attributes): {num_corrects / (len(val_ds) * amount_labels)}\")\n",
    "\n",
    "@torch.no_grad()\n",
    "def test_model(model:nn.Module,\n",
    "               test_ds:torch.utils.data.Dataset,\n",
    "               batch_size:int,\n",
    "               num_workers:int =0,\n",
    "               amount_labels=40):\n",
    "    test_dl = DataLoader(test_ds, batch_size=batch_size,num_workers=num_workers,shuffle=False)\n",
    "    model.eval()\n",
    "    num_corrects = 0\n",
    "    for model_inputs, labels in tqdm(test_dl, leave=False):\n",
    "        model_inputs = model_inputs.to(device)\n",
    "        labels = labels.to(device)\n",
    "        model_outputs = model(model_inputs)\n",
    "        num_corrects += int((model_outputs.round() == labels).sum())\n",
    "    print(f\"Test Accuracy (all attributes): {num_corrects / (len(test_ds) * amount_labels)}\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "#ResNet\n",
    "pretrained = False\n",
    "\n",
    "model_cnn_base = face_models.get_FaceModelResNet(40, pretrained=pretrained).to(device)\n",
    "criterion = nn.BCELoss()\n",
    "optimizer = torch.optim.Adam(model_cnn_base.parameters(), lr=0.01)\n",
    "\n",
    "train_model(model=model_cnn_base,\n",
    "            criterion=criterion,\n",
    "            optimizer=optimizer,\n",
    "            epochs=4,\n",
    "            train_ds=train_dataset_cnn,\n",
    "            val_ds=val_dataset_cnn,\n",
    "            batch_size=128,\n",
    "            num_workers=8,\n",
    "            amount_labels=40)\n",
    "\n",
    "test_model(model=model_cnn_base, test_ds=test_dataset_cnn, batch_size=128, num_workers=8)\n",
    "torch.save(model_cnn_base.state_dict(), path_configs.FACE_CNN_MODEL)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "#Vision Transformer\n",
    "pretrained = False\n",
    "\n",
    "model_vit_base = face_models.get_FaceVisionTransformer(40, pretrained=pretrained).to(device)\n",
    "criterion = nn.BCELoss()\n",
    "optimizer = torch.optim.Adam(model_vit_base.heads.parameters(), lr=0.01)\n",
    "\n",
    "train_model(model=model_vit_base,\n",
    "            criterion=criterion,\n",
    "            optimizer=optimizer,\n",
    "            epochs=10,\n",
    "            train_ds=train_dataset_vit,\n",
    "            val_ds=train_dataset_vit,\n",
    "            batch_size=32,\n",
    "            num_workers=8,\n",
    "            amount_labels=40)\n",
    "\n",
    "torch.save(model_vit_base.state_dict(), path_configs.FACE_VIT_MODEL)\n",
    "test_model(model=model_vit_base, test_ds=test_dataset_vit, batch_size=64,num_workers=4)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Model + DP-SGD"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2023-08-28T16:08:09.157996700Z",
     "start_time": "2023-08-28T16:08:09.097122900Z"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "The following code contains code for training models with DPSGD. \n",
    "The ResNet-18 and the ViT Models iterates through lists of different param values and train a model for each of the param combination.\n",
    "The model are stored and the testing is done in a seperate cell. \n",
    "\n",
    "The reason why training in testing are exceuted in different cells is because of the memory on the GPU. Having the training dataloader and the test dataloader in memory may cause a CUDA OOM."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def train_model_dpsgd(model:nn.Module,\n",
    "                criterion:nn.Module,\n",
    "                optimizer: opacus.optimizers.optimizer.DPOptimizer,\n",
    "                train_dl:torch.utils.data.DataLoader,\n",
    "                privacy_engine:opacus.PrivacyEngine,\n",
    "                max_physical_batch_size:int = 16,\n",
    "                val_dl:torch.utils.data.DataLoader = None,\n",
    "                len_val_ds:int = 1,\n",
    "                epochs:int = 5,\n",
    "                amount_labels:int=40,\n",
    "                max_epsilon:int= 10,\n",
    "                val:bool=True):\n",
    "    epsilon_reached = False\n",
    "    for epoch in range(epochs):\n",
    "        if epsilon_reached:\n",
    "            break\n",
    "        print(f\"Start Training Epoch: {epoch + 1:2}\")\n",
    "        model.train()\n",
    "        epoch_loss = 0.0\n",
    "        with (BatchMemoryManager(\n",
    "                data_loader=train_dl,\n",
    "                max_physical_batch_size=max_physical_batch_size, \n",
    "                optimizer=optimizer) \n",
    "        as memory_safe_data_loader):\n",
    "            for model_inputs, labels in tqdm(memory_safe_data_loader,leave=False):\n",
    "                model_inputs = model_inputs.to(device)\n",
    "                labels = labels.to(device)\n",
    "                #Forward + Backprop\n",
    "                optimizer.zero_grad()\n",
    "                model_outputs = model(model_inputs)\n",
    "                loss = criterion(model_outputs, labels)\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "                epoch_loss += loss.item()\n",
    "                #Check if epsilon exceeds threshold after each batch\n",
    "                if max_epsilon < privacy_engine.accountant.get_epsilon(delta=1e-6):\n",
    "                    print(f\"ε Value {max_epsilon:2} reached in Epoch {epoch+1:2}\")\n",
    "                    epsilon_reached = True\n",
    "                    break\n",
    "\n",
    "        print(f\"Finished Training Epoch: {epoch + 1:2}\",\n",
    "              f\"Train Loss: {epoch_loss / len(train_dl):.5f}\",\n",
    "              f\"ε:{privacy_engine.accountant.get_epsilon(delta=1e-6):.5f}\")\n",
    "\n",
    "        if val:\n",
    "            torch.cuda.empty_cache()\n",
    "            gc.collect() #free cuda memory from train_dataloader\n",
    "            val_loss = 0.0\n",
    "            num_corrects = 0\n",
    "            model.eval()\n",
    "            for model_inputs, labels in val_dl:\n",
    "                model_inputs = model_inputs.to(device)\n",
    "                labels = labels.to(device)\n",
    "                model_outputs = model(model_inputs)\n",
    "                loss = criterion(model_outputs, labels)\n",
    "                val_loss += loss.item()\n",
    "                num_corrects += int((model_outputs.round() == labels).sum())\n",
    "            print(f\"Val Loss: {val_loss / len(val_dl):.5f}\",\n",
    "                  f\"Val Accuracy (all attributes): {num_corrects / (len_val_ds * amount_labels)}\")\n",
    "        print(\"-------------------------------------------------------------\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "#ResNet-18 Parameters\n",
    "cnn_clipping_threshold = 0.00001\n",
    "cnn_pretrained = [True, False]\n",
    "cnn_target_epsilon = [1,5,10]\n",
    "cnn_num_epochs = [1,3,5,10]\n",
    "cnn_batch_size = 256\n",
    "\n",
    "\n",
    "#Dataloader for ResNet with DPSGD\n",
    "data_augmentation_train = torchvision.transforms.Compose([\n",
    "    #torchvision.transforms.AutoAugment(),\n",
    "    torchvision.transforms.ToTensor(),\n",
    "])\n",
    "\n",
    "train_dataset_cnn = faces_dataset.FacesDataset(label_cols=label_columns, mode=\"train\",transform=data_augmentation_train)\n",
    "#The spezified Batch Sizes are Virtual Batch Sizes\n",
    "train_dl_cnn_dpsgd = DataLoader(\n",
    "    dataset=train_dataset_cnn,\n",
    "    batch_size=cnn_batch_size\n",
    ")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "#train different ResNet dpsgd models\n",
    "for pretrained in cnn_pretrained:\n",
    "    for target_epsilon in cnn_target_epsilon:\n",
    "        for num_epochs in cnn_num_epochs:\n",
    "            torch.cuda.empty_cache()\n",
    "            gc.collect() #free cuda memory from train_dataloader\n",
    "            model_cnn_dpsgd = face_models.get_FaceModelResNet(40,pretrained=pretrained).to(device)\n",
    "            model_cnn_dpsgd = ModuleValidator.fix(model_cnn_dpsgd)\n",
    "            criterion = nn.BCELoss()\n",
    "            optimizer = torch.optim.Adam(model_cnn_dpsgd.parameters(), lr=0.01)\n",
    "            privacy_engine= PrivacyEngine(accountant='rdp')\n",
    "            model_cnn_dpsgd, optimizer, train_dl = privacy_engine.make_private_with_epsilon(\n",
    "                module=model_cnn_dpsgd,\n",
    "                optimizer=optimizer,\n",
    "                data_loader=train_dl_cnn_dpsgd,\n",
    "                epochs=num_epochs,\n",
    "                target_epsilon=target_epsilon,\n",
    "                target_delta=1e-6,\n",
    "                max_grad_norm=cnn_clipping_threshold #Gradienten größer als dieser Wert werden geclippt\n",
    "            )\n",
    "            print(f\"Training ResNet Model\\npretrained={pretrained}\\nNum Epochs = {num_epochs}\\ntarget_epsilon={target_epsilon}\\nNoise Mult={optimizer.noise_multiplier:.4f}\")\n",
    "            torch.cuda.empty_cache()\n",
    "            gc.collect() #free cuda memory from train_dataloader\n",
    "            train_model_dpsgd(model=model_cnn_dpsgd,\n",
    "                              criterion=criterion,\n",
    "                              optimizer=optimizer,\n",
    "                              train_dl=train_dl,\n",
    "                              privacy_engine=privacy_engine,\n",
    "                              max_physical_batch_size=64,\n",
    "                              max_epsilon=target_epsilon,\n",
    "                              epochs=num_epochs,\n",
    "                              val=False)\n",
    "            torch.save(model_cnn_dpsgd._module.state_dict(), f\"{path_configs.MODELS_TRAINED_BASE_PATH}/cnn_{'pretrained_' if pretrained else ''}epsilon{target_epsilon}_epochs{num_epochs}_clipp{str(cnn_clipping_threshold).replace('.','')}_batch256_ohneAA.pl\" )"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "#eval dpsgd models\n",
    "for pretrained in cnn_pretrained:\n",
    "    for epsilon in cnn_target_epsilon:\n",
    "        for clipping_threshold in [cnn_clipping_threshold]:\n",
    "            for num_epochs in cnn_num_epochs:\n",
    "                model_path = f\"{path_configs.MODELS_TRAINED_BASE_PATH}/cnn_{'pretrained_' if pretrained else ''}epsilon{epsilon}_epochs{num_epochs}_clipp{str(clipping_threshold).replace('.','')}_batch{cnn_batch_size}.pl\"\n",
    "                if not os.path.isfile(model_path):\n",
    "                    continue\n",
    "                #Load DPSGD Models\n",
    "                model_dgsgd_testing = face_models.get_FaceModelResNet(40)\n",
    "                model_dgsgd_testing = ModuleValidator.fix(model_dgsgd_testing)\n",
    "                model_dgsgd_testing.load_state_dict(torch.load(model_path))\n",
    "                model_dgsgd_testing = model_dgsgd_testing.to(device)\n",
    "        \n",
    "        \n",
    "                print(f\"Testing ResNet DPSGD Model with params:\\npretrained={pretrained}\\nNum Epochs = {num_epochs}\\nepsilon={epsilon}\\nclip={clipping_threshold}\")\n",
    "                test_model(model_dgsgd_testing,test_dataset_cnn,batch_size=64,num_workers=8)\n",
    "                print(\"-------------------------------------\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "#ViT Parameters\n",
    "vit_clipping_threshold = 0.00001\n",
    "vit_pretrained = [True, False]\n",
    "vit_target_epsilon = [1,5,10]\n",
    "vit_num_epochs = [1,3,5]\n",
    "vit_batch_size = 128\n",
    "\n",
    "#Dataloader for ViT with DPSGD\n",
    "data_augmentation_train_with_resize = torchvision.transforms.Compose([\n",
    "    torchvision.transforms.Resize((224, 224)),\n",
    "    #torchvision.transforms.AutoAugment(),\n",
    "    torchvision.transforms.ToTensor()\n",
    "])\n",
    "\n",
    "train_dataset_vit = faces_dataset.FacesDataset(label_cols=label_columns, mode=\"train\",transform=data_augmentation_train_with_resize)\n",
    "#The spezified Batch Sizes are Virtual Batch Sizes\n",
    "train_dl_vit_dpsgd = DataLoader(\n",
    "    dataset=train_dataset_vit,\n",
    "    batch_size=vit_batch_size\n",
    ")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "#train different Vit dpsgd models\n",
    "for pretrained in vit_pretrained:\n",
    "    for target_epsilon in vit_target_epsilon:\n",
    "        for num_epochs in vit_num_epochs:\n",
    "            torch.cuda.empty_cache()\n",
    "            gc.collect() #free cuda memory from train_dataloader\n",
    "            model_vit_dpsgd = face_models.get_FaceVisionTransformer(40,pretrained=pretrained).to(device)\n",
    "            model_vit_dpsgd = ModuleValidator.fix(model_vit_dpsgd)\n",
    "            criterion = nn.BCELoss()\n",
    "            optimizer = torch.optim.Adam(model_vit_dpsgd.parameters(), lr=0.01)\n",
    "            privacy_engine= PrivacyEngine(accountant=\"rdp\")\n",
    "            model_vit_dpsgd, optimizer, train_dl = privacy_engine.make_private_with_epsilon(\n",
    "                module=model_vit_dpsgd,\n",
    "                optimizer=optimizer,\n",
    "                data_loader=train_dl_vit_dpsgd,\n",
    "                epochs=num_epochs,\n",
    "                target_epsilon=target_epsilon,\n",
    "                target_delta=1e-6,\n",
    "                max_grad_norm=vit_clipping_threshold #Gradienten größer als dieser Wert werden geclippt\n",
    "            )\n",
    "            print(f\"Training ViT Model\\npretrained={pretrained}\\nNum Epochs = {num_epochs}\\ntarget_epsilon={target_epsilon}\\n\")\n",
    "            torch.cuda.empty_cache()\n",
    "            gc.collect() #free cuda memory from train_dataloader\n",
    "            train_model_dpsgd(model=model_vit_dpsgd,\n",
    "                              criterion=criterion,\n",
    "                              optimizer=optimizer,\n",
    "                              train_dl=train_dl,\n",
    "                              privacy_engine=privacy_engine,\n",
    "                              max_physical_batch_size=16,\n",
    "                              max_epsilon=target_epsilon,\n",
    "                              epochs=num_epochs,\n",
    "                              val=False)\n",
    "            torch.save(model_vit_dpsgd._module.state_dict(), f\"{path_configs.MODELS_TRAINED_BASE_PATH}/vit_{'pretrained_' if pretrained else ''}epsilon{target_epsilon}_epochs{num_epochs}_clip{str(vit_clipping_threshold).replace('.','')}_batch{vit_batch_size}.pl\" )"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# eval vit dpsgd models\n",
    "for pretrained in vit_pretrained:\n",
    "    for epsilon in vit_target_epsilon:\n",
    "        for clipping_threshold in [vit_clipping_threshold]:\n",
    "            for num_epochs in vit_num_epochs:\n",
    "                model_path = f\"{path_configs.MODELS_TRAINED_BASE_PATH}/vit_{'pretrained_' if pretrained else ''}epsilon{epsilon}_epochs{num_epochs}_clip{str(clipping_threshold).replace('.','')}_batch{vit_batch_size}.pl\"\n",
    "                if not os.path.isfile(model_path):\n",
    "                    continue\n",
    "                # Load DPSGD Models\n",
    "                model_vit_testing = face_models.get_FaceVisionTransformer(40)\n",
    "                model_vit_testing = ModuleValidator.fix(model_vit_testing)\n",
    "                model_vit_testing.load_state_dict(torch.load(model_path))\n",
    "                model_vit_testing = model_vit_testing.to(device)\n",
    "\n",
    "                print(f\"Testing ViT DPSGD Model with params:\\npretrained={pretrained}\\nNum Epochs = {num_epochs}\\nepsilon={epsilon}\\nclip={str(clipping_threshold).replace('.','')}\")\n",
    "                test_model(model_vit_testing,test_dataset_vit,batch_size=32,num_workers=4)\n",
    "                print(\"-------------------------------------\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}