ood_simmim_finetuning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Base settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# %env CUDA_VISIBLE_DEVICES=1\n",
    "\n",
    "import time\n",
    "from pprint import pprint\n",
    "from pathlib import Path\n",
    "from collections import OrderedDict\n",
    "from tqdm import tqdm\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create ConfigDict from config yaml file\n",
    "import yaml\n",
    "from ml_collections import config_dict\n",
    "\n",
    "config_path = Path('configs') / 'deit_tiny-pretrained-cifar100.yaml'\n",
    "with config_path.open('r') as f:\n",
    "    config = yaml.safe_load(f)\n",
    "    config = config_dict.ConfigDict(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# frequently used variables\n",
    "model_name = config.model.name\n",
    "patch_size = config.model.patch_size\n",
    "summary = config.summary\n",
    "\n",
    "# log directories\n",
    "log_root = Path('./logs') / model_name / summary\n",
    "checkpoint_path = log_root / 'checkpoints'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize ViT model and load pretrained weights\n",
    "from ood_with_vit.models.vit import ViT\n",
    "\n",
    "def initialize_vit_model(config, verbose=0):\n",
    "    n_class = config.dataset.n_class\n",
    "    if config.model.pretrained:\n",
    "        model = torch.hub.load(\n",
    "            repo_or_dir=config.model.repo,\n",
    "            model=config.model.pretrained_model,\n",
    "            pretrained=False,\n",
    "        )\n",
    "        model.head = nn.Linear(model.head.in_features, n_class)\n",
    "    else:\n",
    "        model = ViT(\n",
    "            image_size=config.model.img_size,\n",
    "            patch_size=config.model.patch_size,\n",
    "            num_classes=n_class,\n",
    "            dim=config.model.dim_head,\n",
    "            depth=config.model.depth,\n",
    "            heads=config.model.n_heads,\n",
    "            mlp_dim=config.model.dim_mlp,\n",
    "            dropout=config.model.dropout,\n",
    "            emb_dropout=config.model.emb_dropout,\n",
    "            visualize=True,\n",
    "        )\n",
    "\n",
    "    model = model.to(device=device)\n",
    "    if verbose:\n",
    "        print(model)\n",
    "\n",
    "    checkpoint = torch.load(checkpoint_path / f'{summary}_best.pt')\n",
    "\n",
    "    state_dict = checkpoint['model_state_dict']\n",
    "    trimmed_keys = []\n",
    "    for key in state_dict.keys():\n",
    "        # remove prefix 'module.' for each key (in case of DataParallel)\n",
    "        trimmed_keys.append(key[7:])\n",
    "    trimmed_state_dict = OrderedDict(list(zip(trimmed_keys, state_dict.values())))\n",
    "\n",
    "    model.load_state_dict(trimmed_state_dict)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'apple': 0, 'aquarium_fish': 1, 'baby': 2, 'bear': 3, 'beaver': 4, 'bed': 5, 'bee': 6, 'beetle': 7, 'bicycle': 8, 'bottle': 9, 'bowl': 10, 'boy': 11, 'bridge': 12, 'bus': 13, 'butterfly': 14, 'camel': 15, 'can': 16, 'castle': 17, 'caterpillar': 18, 'cattle': 19, 'chair': 20, 'chimpanzee': 21, 'clock': 22, 'cloud': 23, 'cockroach': 24, 'couch': 25, 'crab': 26, 'crocodile': 27, 'cup': 28, 'dinosaur': 29, 'dolphin': 30, 'elephant': 31, 'flatfish': 32, 'forest': 33, 'fox': 34, 'girl': 35, 'hamster': 36, 'house': 37, 'kangaroo': 38, 'keyboard': 39, 'lamp': 40, 'lawn_mower': 41, 'leopard': 42, 'lion': 43, 'lizard': 44, 'lobster': 45, 'man': 46, 'maple_tree': 47, 'motorcycle': 48, 'mountain': 49, 'mouse': 50, 'mushroom': 51, 'oak_tree': 52, 'orange': 53, 'orchid': 54, 'otter': 55, 'palm_tree': 56, 'pear': 57, 'pickup_truck': 58, 'pine_tree': 59, 'plain': 60, 'plate': 61, 'poppy': 62, 'porcupine': 63, 'possum': 64, 'rabbit': 65, 'raccoon': 66, 'ray': 67, 'road': 68, 'rocket': 69, 'rose': 70, 'sea': 71, 'seal': 72, 'shark': 73, 'shrew': 74, 'skunk': 75, 'skyscraper': 76, 'snail': 77, 'snake': 78, 'spider': 79, 'squirrel': 80, 'streetcar': 81, 'sunflower': 82, 'sweet_pepper': 83, 'table': 84, 'tank': 85, 'telephone': 86, 'television': 87, 'tiger': 88, 'tractor': 89, 'train': 90, 'trout': 91, 'tulip': 92, 'turtle': 93, 'wardrobe': 94, 'whale': 95, 'willow_tree': 96, 'wolf': 97, 'woman': 98, 'worm': 99}\n",
      "{'airplane': 0, 'automobile': 1, 'bird': 2, 'cat': 3, 'deer': 4, 'dog': 5, 'frog': 6, 'horse': 7, 'ship': 8, 'truck': 9}\n"
     ]
    }
   ],
   "source": [
    "from torchvision.datasets import CIFAR10, CIFAR100\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "\n",
    "dataset_mean, dataset_std = config.dataset.mean, config.dataset.std\n",
    "dataset_root = config.dataset.root\n",
    "img_size = config.model.img_size\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.Resize(img_size),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(dataset_mean, dataset_std),\n",
    "])      \n",
    "\n",
    "id_dataset = CIFAR100(\n",
    "    root=dataset_root, \n",
    "    train=False, \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "id_test_dataloader = DataLoader(\n",
    "    dataset=id_dataset, \n",
    "    batch_size=config.eval.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "id_dataset_name = 'cifar100'\n",
    "\n",
    "ood_dataset = CIFAR10(\n",
    "    root=dataset_root, \n",
    "    train=False, \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "ood_test_dataloader = DataLoader(\n",
    "    dataset=ood_dataset, \n",
    "    batch_size=config.eval.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "ood_dataset_name = 'cifar10'\n",
    "\n",
    "print(id_dataset.class_to_idx)\n",
    "print(ood_dataset.class_to_idx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Utility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set seeds\n",
    "import random\n",
    "import numpy as np\n",
    "\n",
    "def set_seed(seed):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "seed = 1234"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "\n",
    "def apply_mask_on_img(img, mask):\n",
    "    img_h, img_w, _ = img.shape\n",
    "    mask = cv2.resize(mask, (img_h, img_w), interpolation=cv2.INTER_NEAREST)\n",
    "    mask = np.expand_dims(mask, axis=-1)\n",
    "    masked_img = img * (1 - mask)\n",
    "    return masked_img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: patch_embed.norm\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.simmim import SimMIMHooker\n",
    "from ood_with_vit.visualizer.feature_extractor import FeatureExtractor\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "# add hooks for patch masking\n",
    "mask_ratio = 0.2\n",
    "simmim = SimMIMHooker(\n",
    "    config=config,\n",
    "    model=model,\n",
    "    patch_embedding_layer_name='patch_embed.norm',\n",
    "    mask_ratio=mask_ratio,\n",
    ")\n",
    "simmim.hook()\n",
    "\n",
    "# add hooks for feature extraction\n",
    "feature_extractor = FeatureExtractor(\n",
    "    model=model,\n",
    "    layer_name=config.model.layer_name.penultimate,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:00<00:00, 79.56it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 96.99it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 78.29it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 94.04it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 95.81it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 96.87it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 94.79it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 96.80it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 98.12it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 97.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:00<00:00, 98.04it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 97.88it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 98.48it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 98.48it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 98.43it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 98.40it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 98.03it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 93.73it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 98.40it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 98.44it/s]\n"
     ]
    }
   ],
   "source": [
    "# gather penultimate features\n",
    "from ood_with_vit.utils import compute_penultimate_features\n",
    "\n",
    "set_seed(seed)\n",
    "num_samples = 20\n",
    "num_class = 10\n",
    "\n",
    "print('processing in-distribution samples...')\n",
    "X_in, y_in = [], []\n",
    "in_img_paths = []\n",
    "cls_indices = random.sample(range(len(id_dataset.classes)), num_class)\n",
    "for cls_idx in cls_indices:\n",
    "    img_indices = np.where(np.array(id_dataset.targets) == cls_idx)[0]\n",
    "    img_indices = random.sample(list(img_indices), num_samples)\n",
    "    for i in tqdm(img_indices):\n",
    "        img, _ = id_dataset[i]\n",
    "        img = img.to(device)\n",
    "        penultimate_features = compute_penultimate_features(\n",
    "            config=config, \n",
    "            model=model, \n",
    "            imgs=img.unsqueeze(0),\n",
    "            feature_extractor=feature_extractor,    \n",
    "        )\n",
    "        X_in.append(penultimate_features.squeeze().numpy())\n",
    "        y_in.append((cls_idx, id_dataset.classes[cls_idx]))\n",
    "        # CHW => HWC\n",
    "        original_img_path = f'./assets/{id_dataset_name}/test/{i:05d}.jpg'\n",
    "        original_img = cv2.imread(original_img_path)\n",
    "        mask = simmim.get_mask()\n",
    "        masked_img = apply_mask_on_img(original_img, mask)\n",
    "        masked_img_path = f'./assets/{id_dataset_name}/test/masked/{i:05d}.jpg'\n",
    "        cv2.imwrite(masked_img_path, masked_img)\n",
    "        in_img_paths.append(masked_img_path)\n",
    "\n",
    "print('processing out-of-distribution samples...')\n",
    "X_out, y_out = [], []\n",
    "out_img_paths = []\n",
    "cls_indices = random.sample(range(len(ood_dataset.classes)), num_class)\n",
    "for cls_idx in cls_indices:\n",
    "    img_indices = np.where(np.array(ood_dataset.targets) == cls_idx)[0]\n",
    "    img_indices = random.sample(list(img_indices), num_samples)\n",
    "    for i in tqdm(img_indices):\n",
    "        img, _ = ood_dataset[i]\n",
    "        img = img.to(device)\n",
    "        penultimate_features = compute_penultimate_features(\n",
    "            config=config, \n",
    "            model=model, \n",
    "            imgs=img.unsqueeze(0),\n",
    "            feature_extractor=feature_extractor,    \n",
    "        )\n",
    "        X_out.append(penultimate_features.squeeze().numpy())\n",
    "        y_out.append((cls_idx, ood_dataset.classes[cls_idx]))\n",
    "        # CHW => HWC\n",
    "        original_img_path = f'./assets/{ood_dataset_name}/test/{i:05d}.jpg'\n",
    "        original_img = cv2.imread(original_img_path)\n",
    "        mask = simmim.get_mask()\n",
    "        masked_img = apply_mask_on_img(original_img, mask)\n",
    "        masked_img_path = f'./assets/{ood_dataset_name}/test/masked/{i:05d}.jpg'\n",
    "        cv2.imwrite(masked_img_path, masked_img)\n",
    "        out_img_paths.append(masked_img_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature visualization by T-SNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ipywidgets import HTML, VBox\n",
    "\n",
    "def interactive_plot(partitioned_info, fig, event=\"click\") :\n",
    "    \"\"\"\n",
    "    Make a plot react on hover or click of a data point and update a HTML preview below it.    \n",
    "    \"\"\"\n",
    "    html = HTML(\"\")\n",
    "    fig = go.FigureWidget(data=fig.data, layout=fig.layout)\n",
    "\n",
    "    if event == \"hover\" :\n",
    "        for data in fig.data:\n",
    "            def update(trace, points, state):\n",
    "                if len(points.point_inds) == 0:\n",
    "                    return\n",
    "                trace_idx, point_idx = points.trace_index, points.point_inds[0]\n",
    "                df_row = partitioned_info[trace_idx].iloc[point_idx]\n",
    "                img_url = df_row['img']\n",
    "                label = df_row['label']\n",
    "                html.value = f'\\\n",
    "                    <body>\\\n",
    "                        <img src=\"{img_url}\" width=\"240\"><br>\\\n",
    "                        <b>Image: {img_url}</b><br>\\\n",
    "                        <b>Label: {label}</b>\\\n",
    "                    </body>'\n",
    "            data.on_hover(update)\n",
    "    else :\n",
    "        for data in fig.data:\n",
    "            def update(trace, points, state):\n",
    "                if len(points.point_inds) == 0:\n",
    "                    return\n",
    "                trace_idx, point_idx = points.trace_index, points.point_inds[0]\n",
    "                df_row = partitioned_info[trace_idx].iloc[point_idx]\n",
    "                img_url = df_row['img']\n",
    "                label = df_row['label']\n",
    "                html.value = f'\\\n",
    "                    <body>\\\n",
    "                        <img src=\"{img_url}\" width=\"240\"><br>\\\n",
    "                        <b>Image: {img_url}</b><br>\\\n",
    "                        <b>Label: {label}</b>\\\n",
    "                    </body>'\n",
    "            data.on_click(update)\n",
    "\n",
    "    return VBox([fig, html])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:780: FutureWarning:\n",
      "\n",
      "The default initialization in TSNE will change from 'random' to 'pca' in 1.2.\n",
      "\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:790: FutureWarning:\n",
      "\n",
      "The default learning rate in TSNE will change from 200.0 to 'auto' in 1.2.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f15adabdf0984e57b595d0b14ed81745",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/cifar100/test/masked/01191.jpg'],\n",
       "…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "\n",
    "tsne = TSNE(n_components=2, random_state=seed)\n",
    "X_tsne = tsne.fit_transform(X_in)\n",
    "\n",
    "df_embed = pd.DataFrame(X_tsne)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_in)\n",
    "df_embed = df_embed.assign(img=in_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_in:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "\n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title=f'ImageNet pretrained, CIFAR100 finetuned ViT {mask_ratio} masked CIFAR100 T-SNE',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:780: FutureWarning:\n",
      "\n",
      "The default initialization in TSNE will change from 'random' to 'pca' in 1.2.\n",
      "\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:790: FutureWarning:\n",
      "\n",
      "The default learning rate in TSNE will change from 200.0 to 'auto' in 1.2.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61d7b0d3d7a948b1b8ceb0a80b8dbe9a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/cifar10/test/masked/02593.jpg'],\n",
       " …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tsne = TSNE(n_components=2, random_state=seed)\n",
    "X_tsne = tsne.fit_transform(X_out)\n",
    "\n",
    "df_embed = pd.DataFrame(X_tsne)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_out)\n",
    "df_embed = df_embed.assign(img=out_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_out:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "        \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title=f'ImageNet pretrained, CIFAR100 finetuned ViT {mask_ratio} masked CIFAR10 T-SNE',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature visualization by PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8898ce55feed41a9acdac1018d779ddf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/cifar100/test/masked/01191.jpg'],\n",
       "…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "\n",
    "pca = PCA()\n",
    "components = pca.fit_transform(X_in)\n",
    "\n",
    "df_embed = pd.DataFrame(components)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_in)\n",
    "df_embed = df_embed.assign(img=in_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_in:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "\n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title=f'ImageNet pretrained, CIFAR100 finetuned ViT {mask_ratio} masked CIFAR100 PCA',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c56e2a706c6e4a1f8181520df1c208cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/cifar10/test/masked/02593.jpg'],\n",
       " …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pca = PCA()\n",
    "components = pca.fit_transform(X_out)\n",
    "\n",
    "df_embed = pd.DataFrame(components)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_out)\n",
    "df_embed = df_embed.assign(img=out_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_out:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "        \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title=f'ImageNet pretrained, CIFAR100 finetuned ViT {mask_ratio} masked CIFAR10 PCA',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OOD detection after SimMIM masking applied"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ood_with_vit.utils import compute_ood_scores\n",
    "from ood_with_vit.utils.ood_metrics import auroc, aupr, fpr_at_95_tpr\n",
    "from ood_with_vit.utils.visualization import plot_roc_curve, plot_precision_recall_curve\n",
    "\n",
    "mask_ratio = 0.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### OOD detection by MSP (Maximum over Softmax Probability)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.metrics.msp import MSP\n",
    "\n",
    "model = initialize_vit_model(config, verbose=0)\n",
    "msp = MSP(config, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: patch_embed.norm\n",
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 16.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:05<00:00, 16.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id mean score: -0.62900597\n",
      "ood mean score: -0.51125354\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# hook mask_patch\n",
    "simmim = SimMIMHooker(\n",
    "    config=config,\n",
    "    model=model,\n",
    "    patch_embedding_layer_name='patch_embed.norm',\n",
    "    mask_ratio=mask_ratio,\n",
    ")\n",
    "simmim.hook()\n",
    "\n",
    "test_y, ood_scores, id_ood_scores, ood_ood_scores = compute_ood_scores(\n",
    "    metric=msp,\n",
    "    in_dist_dataloader=id_test_dataloader,\n",
    "    out_of_dist_dataloader=ood_test_dataloader,    \n",
    ")\n",
    "print('id mean score:', np.mean(id_ood_scores))\n",
    "print('ood mean score:', np.mean(ood_ood_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABC80lEQVR4nO3dd3gU1dfA8e8hdOnNRu9dSkQRBBSlFxsKVhRBpKmg/lSs2AtWuogUUVFsSBF4UcSC9A6CiFRBIPSect4/7kTXGJKFZDO7yfk8T57szszOnNkyZ+69M/eKqmKMMSbryuZ3AMYYY/xlicAYY7I4SwTGGJPFWSIwxpgszhKBMcZkcZYIjDEmi7NEkMmJyBoRaeZ3HOFCRB4TkdE+bXusiDznx7bTm4jcIiKzzvK19p0MM5YIMpCIbBaR4yJyRER2eQeGfKHcpqrWUNW5odxGIhHJJSIvishWbz9/E5GHREQyYvvJxNNMRLYHTlPVF1T17hBtT0Skn4isFpGjIrJdRD4VkVqh2N7ZEpGnReSDtKxDVSeqaosgtvWf5He230kRyenF/pv3/m4WkTEiUvZM12X+zRJBxmuvqvmAOkBd4FF/wzlzIpL9NLM+BZoDbYD8wG1AD+CtEMQgIhJu39+3gPuAfkARoDLwJdA2vTeUwmcQcj5uezLQAbgZKAhcBCzBfefOiJ/vX1hSVfvLoD9gM3BVwPNXgGkBzy8FfgYOACuAZgHzigDvA38C+4EvA+a1A5Z7r/sZqJ10m8AFwHGgSMC8usBeIIf3/C5gnbf+mUCZgGUV6A38BvyRzL41B04ApZJMvwSIByp6z+cCLwILgUPAV0liSuk9mAs8D/zk7UtF4E4v5sPAJuAeb9lzvGUSgCPe3wXA08AH3jJlvf26A9jqvRcDA7aXBxjnvR/rgIeB7af5bCt5+9kghc9/LDAUmObFuwCoEDD/LWCb974sAS4PmPc07kD4gTf/bqABMN97r3YCQ4CcAa+pAcwG9gF/AY8BrYBTQKz3nqzwli0IvOetZwfwHBDlzevqvedvADHevK7Aj9588ebt9mJbBdTEnQTEets7Anyd9HcARHlx/e69J0tI8h3ylrvK+zz/My+F31dyn3U377OeB8wA+iRZxwrgOu9x1YD3bz1wo9/HkJAdm/wOICv9JfkBlPR+MG95zy/0fmRtcCW1q73nxb3504BJQGEgB9DUm17X+wFe4v2o7vC2kyuZbX4LdA+I51VghPe4I7ARqAZkBx4Hfg5YVr0fRREgTzL79hLw/Wn2ewv/HKDnegeamriD9WcBP9bU3oO53o+4hhdjDtzZdgXcwagpcAyo5y3fjCQH7tMcHN7FHfQvAk4C1QL3yXvPSwIrk64vYL09gS2pfP5jvf1p4MU/Efg4YP6tQFFv3gBgF5A7IO5Y4BrvvckD1MclzuzevqwD7veWz487qA8AcnvPL0n6HgRs+wtgpPeZlMAl6sTPrCsQB/T1tpWHfyeClrgDeCHvc6gGnB+wz8+l8Dt4CPc7qOK99iKg6Jl8v5Jbbwqf9XhvH/MAtwM/BSxfHZdUc3nLbMOdaGTnn5Om6n4fR0LxF25F66zgSxE5jPuS7Qae8qbfCkxX1emqmqCqs4HFQBsROR9oDfRU1f2qGquq33uv6wGMVNUFqhqvquNwB7NLk9n2h0AXcFUrQGdvGrgD2Yuquk5V44AXgDoiUibg9S+q6j5VPZ7MuovhDjzJ2enNTzRBVVer6lHgCeBGEYlK6T0IeO1YVV2jqnHe+zBNVX9X53tgFnD5aeI4nWdU9biqrsCdEV7kTb8ReMF7z7cDb6ewjqIp7H+gL1R1ofceT8RVEQKgqh+oaoy3b4NxB6QqAa+dr6pfeu/NcVVdoqq/eMtvxh3Im3rLtgN2qepgVT2hqodVdUFyAYnIubj3+H5VPaqqu3Fn+J0DFvtTVd/xtpX084/FJZqqgHjfoWDeC3Alm8dVdb33Ga5Q1Zhklgv2/U3N094+Hsclv8Dv+C3A56p6Evf+bVbV9719XoY7aemUDjGEHUsEGe8aVc2PO1utyj8HyDJAJxE5kPgHNAbOB0oB+1R1fzLrKwMMSPK6UrhqkKQ+Axp6iaUJrtrkh4D1vBWwjn24M7QLA16/LYX92uvFmpzzvfnJrWcL7sy+GCm/B8nGICKtReQXEdnnLd+GfyedYOwKeHwMSGzAvyDJ9lLa/xhOv//BbAsReVBE1onIQW9fCvLvfUm675VFZKp34cEhXPJOXL4UrrolGGVwn8HOgPd9JK5kkOy2A6nqt7hqqaHAbhEZJSIFgtx2sHEG+/6m5u/9UNXDuJJ2YsLrgkvO4N6TS5J8F28BzkuHGMKOJQKfeGevY4HXvEnbcGfKhQL+zlHVl7x5RUSkUDKr2gY8n+R1eVX1o2S2uR93xnwTrsHtY1XVgPXck2Q9eVT158BVpLBL/4f74ZQKnCgil+B+7N8GTA5cpjTujHJvKu/Bf2IQkVy45PYacK6qFgKm4xJYavEGYyeuSii5uJOaA5QUkeiz2ZCIXI5rg7gRKOzty0H+2Rf47/4MB34FKqlqAVxde+Ly24Dyp9lc0vVsw5UiiwW87wVUtUYKr/n3ClXfVtX6uOqVyrgqn1Rf5227QirLgPt+NRCRkikscxTIG/A8uYN20ng+ArqISENcFdp3AXF9n+S7mE9V7w0i1ohjicBfbwJXi8hFuEbA9iLSUkSiRCS3d/ljSa+YPQMYJiKFRSSHiDTx1vEu0FNELvGupDlHRNqKSP7TbPNDXN3oDfxTLQQwAnhURGoAiEhBEQm6GKyq/4c7GH4mIjW8fbjU26/hqvpbwOK3ikh1EckLDAImq2p8Su/BaTabE1d9sgeIE5HWQOAljX8BRUWkYLD7kcQnuPeksIhcCPQ53YLe/g0DPvJizunF31lEHgliW/lx9fB7gOwi8iSQ2ll1flzj7BERqQoEHqSmAueLyP3iLuvN7yVlcO9L2cSrrrzv1yxgsIgUEJFsIlJBRJoSBBG52Pv+5cAdjE/gSpuJ2zpdQgIYDTwrIpW8729tESmadCHv+zUb+EJE6otIdm+feorIXd5iy4HO3u8jGvcdT8103Nn/IGCSqibGPRWoLCK3eevL4e1ntSDWGXEsEfhIVffgGq+eVNVtuAbbx3AHg224s6rEz+g23Jnzr7i2hfu9dSwGuuOK5vtxDb5dU9jsFNwVLru8OvHEWL4AXgY+9qoZVuPaJc7E9bgzqm9wV4l8gLsSpW+S5SbgSkO7cGdh/bwYUnsP/sUr2vfDHbD340o5UwLm/4o749vkFe+Tqy5LySBgO/AH7ox0Mu7M+XT68U8VyQFclce1wNdBbGsm7n3bgKsuO0HKVVEAD+L2+TDuhGBS4gzvvbkaaI97n38DrvBmf+r9jxGRpd7j23GJdS3uvZxM8FUxBbzt7/dij8FdiADu86/uvf9fJvPa13Gf3yxcUnsP15CbnBtwB+5JuNLSaiAa99mAa2+q4MXxDP8+0UmW1x7wOe6qpA8Dph/GnVR0xl2ptwv3+8iV2jojkfxTM2BM6InIXNyVHL7c3ZsWInIv0FlVgzpTNiZSWInAmNMQkfNFpJFXVVIFdynmF37HZUx6s7vrjDm9nLirZ8rhqno+xrUDGJOpWNWQMcZkcVY1ZIwxWVzEVQ0VK1ZMy5Yt63cYxhgTUZYsWbJXVYsnNy/iEkHZsmVZvHix32EYY0xEEZEtp5tnVUPGGJPFWSIwxpgszhKBMcZkcRHXRpCc2NhYtm/fzokTJ/wOJWRy585NyZIlyZEjh9+hGGMymUyRCLZv307+/PkpW7Ys4s/wuCGlqsTExLB9+3bKlSvndzjGmEwmZFVD4gaV3i0iq08zX0TkbRHZKCIrRaTe2W7rxIkTFC1aNFMmAQARoWjRopm6xGOM8U8o2wjG4sZHPZ3WuF4wK+FG2Rqelo1l1iSQKLPvnzHGPyGrGlLVeSJSNoVFOgLjvYFRfhGRQiJy/hkMcWeMMZmLKuzbB4cOQUwM7NoFmzdz9MAe9sQeoGy7W+Hii9N9s362EVzIv/tb3+5N+08iEJEeuFIDpUuXzpDgzlRUVBS1atUiLi6OcuXKMWHCBAoVKgTAmjVr6Nu3Lzt27CAhIYHbb7+dxx9//O+z/BkzZvDEE09w7NgxcuXKxZVXXsngwYN93BtjTEicOgV79sDatfDXXzB/Pvz+O2TPDn/+6aaf/PeQF9+Wg+7toeBJWLy4CtkyWSIImqqOAkYBREdHh2UveXny5GH58uUA3HHHHQwdOpSBAwdy/PhxOnTowPDhw2nRogXHjh3j+uuvZ9iwYfTu3ZvVq1fTp08fpk2bRtWqVYmPj2fUqFH+7owx5uzFxsKGDe7Avns3LFwIX3wBcXGw8zQVHuefD7VqQa9eULo0FCrEgXOieGj/J4zeOZWKhSvwRvvRZCvXLCQh+5kIdvDvMWBLetMiXsOGDVm5ciUAH374IY0aNaJFCzeCYt68eRkyZAjNmjWjd+/evPLKKwwcOJCqVasCrmRx772ZclhUYzKnU6fg++9h0iT44QeXBALlygU1akDevHDrre5AX7YslCjhDv55/jsgW3xCPJcNr8X6mPU8fNnDPN3safLkON3AbWnnZyKYAvQRkY+BS4CD6dI+cP/94J2Zp5s6deDNN4NaND4+njlz5tCtWzfAVQvVr1//X8tUqFCBI0eOcOjQIVavXs2AAQPSN15jTOj8/jvMmgWffw4rVriqnkQ5ckDLltCpE1x4IRQqBPXqQc6cQa065lgMRfIUISpbFM9f+TylCpYi+oLo0OxHgJAlAhH5CGgGFBOR7cBTQA4AVR2BG3u0DW6M3WPAnaGKJSMcP36cOnXqsGPHDqpVq8bVV1/td0jGmLOlCn/8AV9/7apz5syBpUshIeGfZQoWhFKloHNnd4bfpYur4jmrzSkTV03kvm/u46XmL9G9fneurXZt+uxLEEJ51VCXVOYr0DvdNxzkmXt6S2wjOHbsGC1btmTo0KH069eP6tWrM2/evH8tu2nTJvLly0eBAgWoUaMGS5Ys4aKLLvIlbmOMZ88emD0bBgxwV+sEuuACqFkTzj0XrrwS2rSB2rXTZbPbDm6j57SeTP9tOpeWvJRGpRuly3rPiKpG1F/9+vU1qbVr1/5nWkY755xz/n68dOlSLV26tMbGxuqxY8e0XLlyOnv2bFVVPXbsmLZt21bffvttVVVdsWKFVqhQQdevX6+qqvHx8Tp8+PBktxEO+2lMphEfrzpnjurtt6vWq6fqygGqJUqo1q2r+tRTqrNnqx45ErIQPlz5oeZ/Ib/mfT6vvjn/TY2LjwvZtoDFeprjakRcNRRp6tatS+3atfnoo4+47bbb+Oqrr+jbty+9e/cmPj6e2267jT59+gBQu3Zt3nzzTbp06cKxY8cQEdq1a+fzHhiTScXGwuTJ8NFHsGjRP2f+ZcrAwIGuPbBjR1fXnwEK5ynMJSUvYVS7UZQr7F/3MRE3ZnF0dLQmHZhm3bp1VKtWzaeIMk5W2U9j0t3hw/C//8GHH8LBg25ahw7QuLGr4y9VKuXXp5O4hDjemP8Gp+JPMbDJQMDVymREzwEiskRVk215thKBMSZzSkiAMWNgyhT48UfYvx9atHCXcLZtC0WKZGg4K3atoNuUbizZuYQba9z4dwIIh+5jLBEYYzKX+Hh45hkYMsQd/AGio+Gpp8CHateTcSd5bt5zvPTTSxTJU4RPO33K9dWuD4sEkCjTJIKMKl75JdKq8IzJUAkJsG6dKwF8/jls3uyu8nnpJVcCyJvXt9B+2/cbL//0MjfXupnXW7xO0bxFfYvldDJFIsidOzcxMTGZtitq9cYjyJ07t9+hGBNetmyB1193df9797pp+fLBiBHQowf4dDw4cuoIX/36FbfUvoWaJWrya59fKV+4vC+xBCNTJIKSJUuyfft29gTe4ZfJJI5QZkyWd+CAu95//HiYOtVNa97c3c3bqJErCfho9u+z6TG1B1sObKHe+fWoVrxaWCcByCSJIEeOHDZylzGZ3dKl8Nprrurn5EnInx9uvhn69YNLLvE7OvYf38+Dsx5kzPIxVC5ame+7fk+14pFxlV+mSATGmExq1Sp46y147z33PHt2uOkmuPtuaNDA17r/QPEJ8TQa04gNMRt4tPGjPNn0SXJnj5yqXEsExpjwMm2au+Hr66/dAC3gbvBq3hyGDYMwKv3vPbb3707iXmj+AqULlqbe+Wc96q5vQjlUpTHGBO+HH1wdf7t2MHEilCwJ/fvDtm2uq+cZM8ImCagq41eMp/I7lRm9dDQA11S9JiKTAFiJwBjjp5Mn3RU+w4b9049/v37w4IMZdrfvmdpyYAv3TL2Hmb/P5LJSl9GkTBO/Q0ozSwTGmIwXF+du+nruOfe8aFF44AF46KGz7so5I3yw8gPunXYvqso7rd+h18W9yCaRX7FiicAYk3Hi491IXoldPZcpAy+84Pr7yRb+B9TieYvTqFQjRrYbSZlCZfwOJ91YIjDGhN6qVfDii/Dxx66z51Kl3JVAd97p201fwYiNj2Xw/MHExsfyRNMnaFmxJS0qtMh0N65aIjDGhE5MDHTv7gZvB6hUyd3x269f0MM3+mXZzmV0m9KNZbuW0blm57DqJC69WSIwxqS//ftdg+8nn8CRI67Xz+HDoXx432ELcCLuBIO+H8QrP71CsbzF+OzGz7iu2nV+hxVSlgiMMelnxw43XOzw4XD0qDvr//RTuOEGvyML2sZ9G3nt59e4/aLbGdxiMIXzFPY7pJAL/9YZY0z4W74cWrd21/6/9pq76/fnn93loRGQBI6cOsKEFRMAqFmiJuv7rGdMxzFZIgmAlQiMMWmxcSP06uU6gQOoVQtGj3aJIELM3DiTHlN7sO3gNqIviKZa8Wq+DhvpBysRGGPO3O7drp//SpVcErj6ati6FVaujJgkEHMshju+vINWE1uRN0defrjzh4jpJC69WYnAGBO848fdIO9vvOGet2oFgwbBxRf7G9cZSuwkbuO+jQy8fCCPN3k8ojqJS2+WCIwxqTtxAu66y3UGB1C2LLzzji9DP6bFnqN7KJq3KFHZonj5qpcpU6gMdc6r43dYvrOqIWPM6anChAlQoIBLAg0buv+bNkVUElBV3l/2PpWHVObdJe8C0LFqR0sCHisRGGOSN3cu3H8/rFjhun946in3F2E3VG0+sJkeX/dg9qbZXF76cq4od4XfIYUdSwTGmH9bvx7uuQe+/949f+45uO8+NxZwhJmwYgL3TrsXEWFYm2HcE31PpugkLr1ZIjDGOKdOwSuvwBNPuOcPPeQSwIUX+htXGpyb71yalGnCiHYjKF2wtN/hhC1LBMYYdzVQgQKue+hKlWDkSLgi8qpQYuNjeeWnV4jXeJ5s+iQtKrSgRYUWfocV9qyMZExWN2ECFCvmkkC3bm6AmAhMAkt3LuXidy/m8e8eZ33MelTV75AihiUCY7KqefOgaVO4/Xa44AI3POS77/od1Rk7HnucR/7vERq824C/jv7FFzd9wcTrJmbKXkJDJaSJQERaich6EdkoIo8kM7+0iHwnIstEZKWItAllPMYYXGdwnTq5JDBvnrtBbO1auPnmiLsiCGDT/k28Pv91utbpytpea7mm6jV+hxRxQtZGICJRwFDgamA7sEhEpqjq2oDFHgc+UdXhIlIdmA6UDVVMxmR5P/zgSgCbN0Pjxm6gmAhsDD508hCfr/ucrnW6UqNEDX7r+1umGjEso4WyRNAA2Kiqm1T1FPAx0DHJMgoU8B4XBP4MYTzGZF07d7r+gJo0gb174csvXVKIwCQw/bfp1BxWk25TurFuzzoASwJpFMpEcCGwLeD5dm9aoKeBW0VkO6400De5FYlIDxFZLCKL9+zZE4pYjcmcvv4arrnGtQH83/9Bx47wxx/uf4TZe2wvt31xG20/bEv+XPn56a6fsmwncenN78tHuwBjVXWwiDQEJohITVVNCFxIVUcBowCio6PtUgBjUhITA599BmPHwvz5btrNN7sRw+rW9TW0s5XYSdym/Zt4ssmTPHb5Y+TKnsvvsDKNUCaCHUCpgOclvWmBugGtAFR1vojkBooBu0MYlzGZ0/79MGKE6w30xAnInh2eeQb69IEiRfyO7qz8deQvip9TnKhsUbx29WuUKVSG2ufW9jusTCeUVUOLgEoiUk5EcgKdgSlJltkKNAcQkWpAbsDqfow5E0uXwt13Q4kS8NhjULs2zJjhrg568smITAKqyntL36PKkCqMWjIKgPZV2lsSCJGQlQhUNU5E+gAzgShgjKquEZFBwGJVnQIMAN4VkQdwDcdd1e4CMSY4y5bBgAHw3XfueZcuro+gpk39jSuNNu3fRPevu/PtH9/StExTrip/ld8hZXohbSNQ1em4RuDAaU8GPF4LNAplDMZkSuPHQ9eurlfQNm3c2ADly/sdVZqNWz6OXtN7ESVRjGg7gu71u1sncRnA78ZiY8yZ2LcPqleHv/6C0qXh22+hQgW/o0o3F+S/gCvLXcnwtsMpWaCk3+FkGZYIjIkUq1dD27YuCVxyiRsrOH9+v6NKk1Pxp3jpx5dI0ASebvY0V1e4mqsrXO13WFmOlbmMCXe7d0P37nDRRbBjB3zwAfzyS8QngUU7FlF/VH2emvsUm/Zvsk7ifGSJwJhw9sUXbnzg0aPdTWAbNsAtt/gdVZociz3Gg7Me5NL3LmX/8f1M6TyF8deOt07ifGRVQ8aEox073FVAP/wApUq5XkFbtvQ7qnTxx/4/eGfhO3Sv152Xr3qZgrkL+h1SlmclAmPCydGjMHiwGxzmxx/h5Zfd0JERngQOnjjI+8veB6BGiRps7LuREe1GWBIIE1YiMCZcLFgAzZu7ZFCoEMycCZdf7ndUaTZtwzTumXoPO4/spGGphlQtVpVSBUul/kKTYaxEYIzf9u6Fvn3h0kvh2DFXIoiJifgksOfoHm75/BbafdSOwnkKM7/bfKoWq+p3WCYZViIwxk/ffQcdOsCRI6765513XLVQhItPiKfx+435Y/8fPNPsGR5p/Ag5o3L6HZY5DUsExvghNhbuvNMND1mkCLz/vrtTOMLtOrKLEueUICpbFINbDKZsobLULFHT77BMKoKuGhKRvKEMxJgsY+1ad0/AxInQogVs2hTxSSBBExi5eCSV36nMyMUjAWhXuZ0lgQiRaiIQkctEZC3wq/f8IhEZFvLIjMlsTp50pYA6dWDbNpgwAb75BgpG9pUzG/dtpPn45vSc1pOLL7yYlhUj+wqnrCiYqqE3gJZ4XUir6goRaRLSqIzJbJYsgeuvhy1boFEjGDcuU/QR9P6y9+k1vRc5o3Lybvt36Va3m90YFoGCaiNQ1W1JPtz40IRjTCZz8KAbLH7KFMidG4YMgd69/Y4q3ZQuWJqWFVoytM1QLiwQeeMfGyeYRLBNRC4DVERyAPcB60IbljGZwJYt0K6d6yzuppvgrbfg3HP9jipNTsad5MUfXyRBExh0xSCal29O8/LN/Q7LpFEwjcU9gd64ged3AHWAXiGMyZjIt2yZ6yF03TrXSdzHH0d8EliwfQH1R9Xnme+fYevBrdZJXCYSTCKooqq3qOq5qlpCVW8FqoU6MGMi1pAhUK8eHDoEc+dGfCdxR08dpf/M/jR8ryEHTx5kapepjL1mrLUFZCLBJIJ3gpxmTNb2yy/u7uC+faF+fdi4ERo39juqNNtycAvDFg2jZ3RP1vRaQ9vKbf0OyaSz07YRiEhD4DKguIj0D5hVADcGsTEG4MQJd/AfPRpy5nRjB7z5JuSN3FtvDpw4wOS1k7m73t1UL16djf022ohhmVhKjcU5gXzeMoEjYBwCbghlUMZEBFWYNQv69HFn/+3bu2RQooTfkaXJV79+xb3T7mX30d00Lt2YqsWqWhLI5E6bCFT1e+B7ERmrqlsyMCZjwt++fe6y0GnToHBh10dQnz5+R5Umu4/upt+MfkxaM4na59ZmSpcp1klcFhHM5aPHRORVoAaQO3Giql4ZsqiMCVeqrmuI++93PYT26QOvvuruEYhg8QnxNBrTiK0Ht/LcFc/xcKOHyRGVw++wTAYJJhFMBCYB7XCXkt4B7AllUMaEpb/+cl1EzJjhegj95BO4MrLPh/48/Cfn5TuPqGxRvNXqLcoWKkv14tX9DstksGCuGiqqqu8Bsar6vareBUT2t9+YMxEXB48+Cued55JA//6wZk1EJ4EETWD4ouFUHVKVEYtHANCmUhtLAllUMCWCWO//ThFpC/wJFAldSMaEiWPH4I473L0Ae/dCtWrw2mvQpo3fkaXJhpgNdP+6O/O2zOOq8lfRumJrv0MyPgsmETwnIgWBAbj7BwoA94cyKGN8t307XHstLF7segsdPtx1GhfhN1G9t/Q9+szoQ+7suRnTYQxd63S1G8NM6olAVad6Dw8CVwCISKNQBmWMrz7/HG69FY4fdwmgZ0+/I0o3ZQuVpXXF1gxtM5Tz85/vdzgmTKR0Q1kUcCOuj6FvVHW1iLQDHgPyAHUzJkRjMogqPP00DBrkbgb7v/9zg8lHsJNxJ3l23rMAPHflc9ZJnElWSiWC94BSwELgbRH5E4gGHlHVLzMgNmMyTmys6xPo00+hbVs3dGTx4n5HlSY/b/uZblO68eveX7mrzl2oqlUDmWSllAiigdqqmiAiuYFdQAVVjcmY0IzJIHv3QuvWrj2gf393X0C2oEdxDTtHTh1h4JyBvLPwHUoVLMU3t3xjo4aZFKX0bT+lqgkAqnoC2HSmSUBEWonIehHZKCKPnGaZG0VkrYisEZEPz2T9xqTZzz+7nkIXL4aXX4bBgyM6CQBsPbiVkUtG0vvi3qy+d7UlAZOqlEoEVUVkpfdYgArecwFUVWuntGKvjWEocDWwHVgkIlNUdW3AMpWAR4FGqrpfRCK7kxYTOVTh7rthzBj3fPp0VyqIUPuP7+fTtZ/So34Pqhevzqb7NnFB/gv8DstEiJQSQVrHHGgAbFTVTQAi8jHQEVgbsEx3YKiq7gdQ1d1p3KYxqdu+3V0KunAhlCwJ8+ZBuXJ+R3XWvlj3Bb2m92LP0T00LdOUKsWqWBIwZ+S0ZWBV3ZLSXxDrvhDYFvB8uzctUGWgsoj8JCK/iEir5FYkIj1EZLGILN6zx3q3MGmweDHUru2SQJ8+sHlzxCaBXUd20enTTlz3yXWcl+88FnZfSJViVfwOy0SgoAavD/H2KwHNgJLAPBGppaoHAhdS1VHAKIDo6GgbH8+cnWnT4MYb3aWhixZBdLTfEZ21+IR4Ln//crYd3MYLV77Ag5c9aJ3EmbMWykSwA3f5aaKS3rRA24EFqhoL/CEiG3CJYVEI4zJZzcmTcM89MG6ce75kCVSNzO6Vtx/azgX5LyAqWxRvt3qbcoXLWVfRJs2CujxCRPKIyJmWORcBlUSknIjkBDoDU5Is8yWuNICIFMNVFW06w+0YkzxVN3xk+/YuCdSv79oHIjAJJGgC7yx4h6pDqjJ80XAAWldqbUnApItUE4GItAeWA994z+uISNID+n+oahzQB5gJrAM+UdU1IjJIRDp4i80EYkRkLfAd8JDdp2DSxf79rpO4hg3h++/h8cdd+8CFSZupwt+ve3+lyftN6PdNPxqXbky7yu38DslkMqKacpW7iCzBdTs9V1XretNWqWqtDIjvP6Kjo3Xx4sV+bNpEgoQEeP55eOEFVyXUqxc88wwULep3ZGdl9NLR9Jneh7w58vJmqze5rfZtdnewOSsiskRVk20YC6obalU9mOTLZw22JvzEx8Ntt8FHH0GrVq7PoIsv9juqNKlQuALtq7RnSOshnJvvXL/DMZlUMIlgjYjcDER5N4D1A34ObVjGnKFdu+Dqq2H1avd/+vSI7DL6RNwJBn0/CIAXmr/AFeWu4IpyV/gclcnsgmks7osbr/gk8CGuO+r7QxiTMcHbvBk6d3b3AqxZ47qJmDkzIpPAT1t/os6IOrz444vsObqH1KptjUkvwZQIqqrqQGBgqIMx5oyMGgX33QcnTkCLFjBwIDRp4ndUZ+zwycM8Nucxhi4aSplCZZh560xaVGjhd1gmCwkmEQwWkfOAycAkVV0d4piMSdmxY/DSS/Cs62efH36Axo39jSkNth/azuhlo+nboC/PN3+efDnz+R2SyWJSrRpS1StwI5PtAUaKyCoReTzkkRmTlCp8/LEbRP7ZZ6FDB9ixIyKTQMyxmL/vB6hWvBqb+m3irdZvWRIwvgjqhjJV3aWqbwM9cfcUPBnKoIz5j/XrXSNwly6QKxfMmAFffQUXRFbnaqrK5LWTqT6sOv2+6cf6vesBbNhI46tgbiirJiJPi8gq3OD1P+O6izAm9GJioFs3dzfwd9/Bww/Dpk3u8tAIs/PwTq7/5Ho6fdqJUgVKsbj7YuskzoSFYNoIxgCTgJaq+meI4zHmH9u2QZ067i7hHj3gkUcitqfQxE7idhzewStXvcIDDR8geza/+3w0xkn1m6iqDTMiEGP+RdW1AezbB19+CR07+h3RWdl2cBsXFriQqGxRDG0zlHKFy1G5aGW/wzLmX05bNSQin3j/V4nIyoC/VQEjlxmT/lasgNKlYfly6N07IpNAfEI8by94m6pD/+kkrmXFlpYETFhKqURwn/ffergyGUMV3n3XdRkN7kaxt97yN6azsG7POrpN6cb87fNpXbE17au09zskY1KU0ghlO72HvZIZnaxXxoRnsoyTJ92gMffcAxUrwtq1rs+gqCi/Izsjo5aMos7IOmyI2cCEaycw7eZplC5Y2u+wjElRMJePXp3MtMgd5duEnz/+gLp1YfJkGDDAdRVRLa1DZvujUpFKXFv1Wtb2XsuttW+1nkJNRDht1ZCI3Is78y+fpE0gP/BTqAMzWcSuXXDZZe7/Cy/Ao4/6HdEZOR57nKfnPo2I8NJVL1kncSYipdRG8CEwA3gReCRg+mFV3RfSqEzWcOQI3HKLSwJffeWuEoog87bM4+4pd/Pbvt/oWb8nqmolABORUkoEqqqbRaR30hkiUsSSgUmT7dtdR3Hr1rnRwyIoCRw6eYhH/u8Rhi8eTvnC5Zlz+xyuLHel32EZc9ZSKxG0A5bgBqIJPNVRoHwI4zKZ2dq10KwZ7NkDw4dDz55+R3RG/jz8J2OXj6X/pf0ZdMUgzsl5jt8hGZMmp00EqtrO+x+Zt3Ka8DRsmOs6OmdOmDYN2rTxO6Kg7D22l0/WfEKvi3tRtVhV/rjvDxsxzGQawfQ11EhEzvEe3yoir4uIXQ9nzkxcHNxwg7tBrEYNWLUqIpKAqjJp9SSqD63O/d/cz4aYDQCWBEymEszlo8OBYyJyETAA+B2YENKoTOayYAHkyweffeZ6D124EMqHf83in4f/5JpJ19D5s86UKVSGJT2W2J3BJlMKJhHEqRszryMwRFWH4i4hNSZ1c+a4UcNOnoSnnoIPP3TVQmEuPiGeJu83Ydbvs3jt6teY320+tc6t5XdYxoREMN0fHhaRR4HbgMtFJBuQI7RhmYh3/Li7L+C556B4cZg6FRo08DuqVG05sIWSBUoSlS2KYW2HUb5weSoWqeh3WMaEVDAlgptwA9ffpaq7cGMRvBrSqExkW7cOatZ0SaBJE9iwIeyTQHxCPK/Pf51qQ6sxfLHrJK5FhRaWBEyWEMxQlbuAiUBBEWkHnFDV8SGPzESm7793Ywj8+Se89x7MnQuFCvkcVMpW717NZWMuY8CsATQv35xrql7jd0jGZKhgrhq6EVgIdAJuBBaIyA2hDsxEmFOnXPcQzZq5oSTnzoW77oIwv9N2xOIR1BtZj037N/HhdR8ypfMUShawAfhM1hJMG8FA4GJV3Q0gIsWB/wMmhzIwE0H273elgK1boV49GD/eXSIaxhK7g6hWrBqdanTizZZvUvyc4n6HZYwvgkkE2RKTgCeGIAe9N1nA779D69YuCTz/vCsVhHEp4FjsMZ787kmiJIqXr36ZpmWb0rRsU7/DMsZXwRzQvxGRmSLSVUS6AtOA6aENy4Q9VXdfQIMGrt+gSZPgscfCOgnM3TyX2sNrM3j+YI6cOoK7KtoYE8yYxQ+JyHVAY2/SKFX9IrRhmbDXty8MHQoFCsDXX0Pz5n5HdFoHTxzk4dkPM2rpKCoUrsC3t39rXUUbEyCl8QgqAa8BFYBVwIOquiOjAjNh7J57YNQoN6LYxImQPZgaRv/sPLKTD1Z9wIMNH+SZK54hb468fodkTFhJqWpoDDAVuB7XA+k7Z7pyEWklIutFZKOIPJLCcteLiIpI9Jluw2SgxP6CRo1y9wmMGxe2SWDP0T28s8B9ZasWq8rm+zbzaotXLQkYk4yUfsX5VfVd7/F6EVl6JisWkShgKG6oy+3AIhGZoqprkyyXH7gPWHAm6zcZLDYWLr0Uli51g8pPnAjZwu+aAVXlo9Uf0W9GPw6dPETLii2pXLSyXRFkTApS+iXnFpG6IlJPROoBeZI8T00DYKOqblLVU8DHuP6KknoWeBk4ccbRm4zTo4dLAnff7QaVD8MksO3gNtp/1J5bPr+FikUqsuyeZdZJnDFBSKlEsBN4PeD5roDnCqQ2JNOFwLaA59uBSwIX8BJKKVWdJiIPnW5FItID6AFQurT1gJ3hHn4Yxo51JYF33011cT/EJcTRbFwzdh3ZxRst36Bvg75EZYvyOyxjIkJKA9OE9LIKr/O614GuqS2rqqOAUQDR0dF2zV9GeuIJePVVKFXKtQ2Emc0HNlOqQCmyZ8vOyHYjKV+4POULh38X18aEk1CW73cApQKel/SmJcoP1ATmishm4FJgijUYh5HBg13HcXXqwK+/Qv7w6X08LiGO135+jWpDqzFs0TAArip/lSUBY85CKC/5WARUEpFyuATQGbg5caaqHgSKJT4Xkbm4S1QXhzAmE4zdu+G66+Cnn6BqVZg/H3Ln9juqv638ayXdpnRj8Z+L6VilI9dXv97vkIyJaCFLBKoaJyJ9gJlAFDBGVdeIyCBgsapOCdW2TRosXgwtW8K+fXDBBWGXBIYtGsZ939xH4dyFmXTDJDpV74SE8d3MxkSCVBOBuF/ZLUB5VR3kjVd8nqouTO21qjqdJN1RqOqTp1m2WVARm9D5+GM3lCSE3cDyiZ3E1SxRk841O/NGyzcolrdY6i80xqQqmBLBMCABd5XQIOAw8BlwcQjjMhlJFSZMgDvugHPOcWMKV6/ud1QAHD11lMe/fZzs2bLzaotXaVKmCU3KNPE7LGMylWAaiy9R1d541/mr6n4g/AedNcH57Td30L/jDihRApYvD5skMGfTHGoNr8WbC97kZPxJ6yTOmBAJJhHEencJK/w9HkFCSKMyGWPOHDd+wIYNcPPNsHEjVPR/aMYDJw5w95S7uWrCVWTPlp15Xefxduu3rS3AmBAJJhG8DXwBlBCR54EfgRdCGpUJvVGj4KqrXClg7VrXZUSYXB7615G/+Hj1x/yv0f9Y0XMFl5e53O+QjMnUgumGeqKILAGaAwJco6rrQh6ZCY19+1yD8KxZ7iaxL7+EKlX8jurvg/99l95HlWJV2Hz/ZmsMNiaDBHPVUGngGPB14DRV3RrKwEwIbN4MlSu7DuT+9z8YNAhy+tvco6pMXDWR+765jyOnjtCmUhsqFa1kScCYDBTMVUPTcO0DAuQGygHrgfAelNb8288/Q/v2kCMHjB4Nt9/ud0RsPbiVnlN7MmPjDBqWbMh7Hd6jUtFKfodlTJYTTNVQrcDnXkdxvUIWkUl/06bBNde40cSmT4em/o/RG5cQR7Oxzdh9dDdvt3qbXhf3sk7ijPHJGd9ZrKpLReSS1Jc0YWH6dLj+ejj3XPjlFyhZ0tdwNu3fRJmCZcieLTvvtn+XCkUqULZQWV9jMiarS/WqIRHpH/D3oIh8CPyZAbGZtHrpJWjb1o0i9umnviaBuIQ4Xv7xZaoPrc7QRUMBaF6+uSUBY8JAMCWCwGsK43BtBp+FJhyTbt5/Hx591FUDTZ4MxfxrfF2+azndpnRj6c6lXFv1WjpV7+RbLMaY/0oxEXg3kuVX1QczKB6THj7/HHr2hEqV4JtvfO00bsjCITww8wGK5inK5E6TradQY8LQaauGRCS7qsYDjTIwHpNWw4a5NoFKlWDuXN+SQGJ3ELXPrc0ttW5hbe+1lgSMCVMplQgWAvWA5SIyBfgUOJo4U1U/D3Fs5kzs3AnPP+8SAcC4ca4b6Qx25NQRBs4ZSI6oHLzW4jXrJM6YCBBMFxO5gRhc76PtgPbefxMufvwR6taFoUOhY0f480+oXz/Dw5j1+yxqDqvJOwvfITY+1jqJMyZCpFQiKCEi/YHV/HNDWSL7hYeLefPgyiuhSBH44Qdo3DjDQ9h/fD/9Z/Vn7PKxVClahXl3zqNx6YyPwxhzdlJKBFFAPv6dABJZIvCbKvTq5e4SPvdcWLDAt8tDdx/dzeS1k3m08aM82fRJcmcPnxHNjDGpSykR7FTVQRkWiQlebCy0bu26ka5e3Zd7BHYd2cVHqz7igYYPuE7i7ttM0bxFMzQGY0z6SKmNwDp/D0cxMXDppS4JDBoEa9Zk6EAyqsq45eOoPrQ6j855lN9ifgOwJGBMBEspETTPsChMcNavh7JlYelSePFFeOKJDN385gObaTWxFV2/6kr14tVZ3nO5dRJnTCZw2qohVd2XkYGYVMTFwQ03wJEjrirohhsydvMJcVwx7gr2HtvL0DZD6Rndk2wSzEVnxphwd8adzhkfHDwInTrB6tUwfHiGJoGN+zZSrlA5smfLzpgOYyhfuDxlCpXJsO0bY0LPTunCmaq7QaxKFZg9Gx580HUdkQFi42N54YcXqDGsxt+dxF1R7gpLAsZkQlYiCGdPPQXPPusef/UVdOiQIZtdunMp3aZ0Y/mu5XSq3ombatyUIds1xvjDEkG4mj7dJYHGjeG771xX0hng7QVv039mf4qfU5zPb/yca6tdmyHbNcb4xxJBOJo9240jkCOHaxjOgCSgqogIdc+ry+0X3c7gFoMpnKdwyLdrjPGfJYJw8/77cNddrsuIhQvhvPNCurnDJw/z6JxHyRWVi8EtB3N5mcu5vMzlId2mMSa8WGNxOPn6a5cEwJUKKlQI6ea+2fgNNYfXZNiiYShqncQZk0VZiSBcTJvmGoMLFnR3C194Ycg2FXMshv6z+jN+xXiqFavGT3f9RMNSDUO2PWNMeLMSQTj45JN/rgj68suQJgGAmOMxfLHuC55o8gTL7llmScCYLC6kiUBEWonIehHZKCKPJDO/v4isFZGVIjJHRLLWReqqMH483H47lCkDO3ZAs2Yh2dTOwzt57efXUFUqF63Mlvu3MOiKQeTKnisk2zPGRI6QJQJvvOOhQGugOtBFRJL2jrYMiFbV2sBk4JVQxRN2VOHaa+GOO6BUKXe5aAhGFFNVxiwbQ7Wh1XjiuyfYuG8jgF0RZIz5WyhLBA2Ajaq6SVVPAR8DHQMXUNXvVPWY9/QXwJ8O9TNafDy0auVuErvtNli7FqpWTffN/LH/D1p80IJuU7px0XkXsaLnCuskzhjzH6FsLL4Q2BbwfDtwSQrLdwNmJDdDRHoAPQBKly6dXvH5I7EkMGsW3H03jBwJ2dI/H8clxHHl+CuJORbD8LbD6VG/h3USZ4xJVlhcNSQitwLRQNPk5qvqKGAUQHR0dGRf4/jgg+4y0WuvhXffTffV/xbzG+ULlyd7tuy83/F9KhSuQKmCpdJ9O8aYzCOUp4g7gMAjUElv2r+IyFXAQKCDqp4MYTz+mzoVXn8dLrsMPvssXVcdGx/Lc/Oeo+bwmgxZOASAZmWbWRIwxqQqlCWCRUAlESmHSwCdgZsDFxCRusBIoJWq7g5hLP775BO46SbXIDx1Kkj6DQC3+M/FdJvSjZV/raRzzc50qdUl3dZtjMn8QpYIVDVORPoAM4EoYIyqrhGRQcBiVZ0CvArkAz4Vd2DcqqoZ08VmRpo/3yWBHDlg5kwonH5X7Lz1y1v0n9Wf8/Kdx1edv6JDlcz39hljQiukbQSqOh2YnmTakwGPrwrl9sPC9Olw/fWQNy8sWpRu4wsndhIXfUE03ep245WrX6FQ7kLpsm5jTNYSFo3Fmdbeva4X0Tx54Pvv0yUJHDp5iP/N/h+5s+fmjVZv0Kh0IxqVbpQOwRpjsiq7njBUTp2CFi3c46FDITo6zauc/tt0agyrwailo8ieLbt1EmeMSRdWIggFVbj5Zli2DEaMgDvvTNPq9h7by/3f3M/EVROpUbwGkztN5pKSKd2SYYwxwbNEEAodO7p7Bfr2hXvuSfPq9h/fz9cbvuappk/x2OWPkTMqZzoEaYwxjiWC9JSQAF26uCTQqZO7Z+As7Ti0g4mrJvLQZQ9RqWgltty/xRqDjTEhYYkgvRw6BFdd5a4M6tABPvzwrIaYVFVGLx3Ng7MfJDY+luuqXUfFIhUtCRhjQsYai9NLYhLo29eNKXAWSeD3fb/TfHxzekztQb3z67Hy3pVULFIx/WM1xpgAViJIq4QEeOghlwSefRYef/ysVhOXEEfz8c3Zd3wfI9uN5O56d1snccaYDGGJIK0aNnSDzLdqBf/73xm/fP3e9VQoUoHs2bIz7ppxVChSgZIFskZv3MaY8GCnnGcrLg46d3ZJ4Nxz3R3EOXIE/fJT8ad4Zu4z1Bpei6ELhwLQtGxTSwLGmAxnJYKzoeruDZg0yQ0tOXPmGXUit3DHQrpN6cbq3au5udbN3FL7ltDFaowxqbBEcKZ27oQ2bWD5crjuOpg8+YySwJu/vMmAWQM4P9/5fN3la9pVbhe6WI0xJgiWCM7Erl1QpQocPQqPPALPPx90EkjsJK7BhQ3oXq87L1/1MgVzFwxxwMYYkzpLBMFKSHC9iB4+7C4P7dgx1ZcAHDxxkIdnP0yeHHl4s9WbXFbqMi4rdVloYzXGmDNgjcXBUHVdRfz8MzzzTNBJ4Ov1X1N9WHVGLxtNrqhc1kmcMSYsWYkgGI8/DqNHQ7t28MQTqS6+5+ge7vvmPj5a/RG1StTiy5u+5OILL86AQI0x5sxZIkjNJ5/ACy+4BuKvvgqqTeDgyYNM/206zzR7hkcaP2KdxBljwpolgpSMHw933OEev/oqZDt9Tdq2g9v4YOUHPNL4ESoWqciW+7dYY7AxJiJYG8HprF37TxJYu/a0o4slaAIjFo+gxrAaPPfDc/y+/3cASwLGmIhhiSA5x4/DjTe6x599BtWqJbvYbzG/ceW4K7l32r00uLABq+5dZZ3EGWMijlUNJeeBB2DNGhg50t00loy4hDiunnA1B04c4L0O73FnnTuRM7ixzBhjwoUlgqQ++sglgPbtoUeP/8xet2cdlYpWInu27Ey4dgIVilTggvwX+BCoMcakD6saCvTee26s4XLl4IMP/jXrZNxJnvruKWqPqM2QhUMAuLzM5ZYEjDERz0oEibZuhe7d3eMFC6BAgb9n/bL9F7pN6cbaPWu5rfZt3Fb7Np+CNMaY9GeJAGD7dmjc2N1BPGUKFC/+96zBPw/modkPUbJASabfPJ3WlVr7GKgxxqQ/SwQLFsCll7rHkya5tgHcZaHZJBsNSzWkZ3RPXrrqJQrkKpDCiowxJjJl7UTw3Xdw5ZXu8RtvwI03cuDEAQbMHEDeHHl5p8071kmcMSbTy7qNxYcOwS23QN688OuvcP/9fPnrl1QfWp1xK8aRP1d+6yTOGJMlZM0SgSrccIMbZGbGDHaXLEyfT2/k07WfUue8Oky9eSr1zq/nd5TGGJMhsmaJYPhwmD0bbr8dWrXi0MlDzN40m+evfJ6Fdy+0JGCMyVKyXolg6VLo3ZutpQsy4a6KPKZKxSIV2Xr/VvLnyu93dMYYk+FCWiIQkVYisl5ENorII8nMzyUik7z5C0SkbCjjYelSEqLrM6yBUOPuk7zw00t/dxJnScAYk1WFLBGISBQwFGgNVAe6iEjSLjy7AftVtSLwBvByqOIBWN+3C826Qu82SsOyl7Om1xrrJM4Yk+WFskTQANioqptU9RTwMZB0jMeOwDjv8WSguYSo57a44UNp2WADq0rl4v2O7zPz1pmULVQ2FJsyxpiIEspEcCGwLeD5dm9assuoahxwECiadEUi0kNEFovI4j179pxVMNlrXcQH+5qxtu86utbpaj2FGmOMJyIai1V1FDAKIDo6+uwu7m/cmMaNv0vPsIwxJlMIZYlgB1Aq4HlJb1qyy4hIdqAgEBPCmIwxxiQRykSwCKgkIuVEJCfQGZiSZJkpgDceJDcA36rdzmuMMRkqZFVDqhonIn2AmUAUMEZV14jIIGCxqk4B3gMmiMhGYB8uWRhjjMlAIW0jUNXpwPQk054MeHwC6BTKGIwxxqQsa3YxYYwx5m+WCIwxJouzRGCMMVmcJQJjjMniJNKu1hSRPcCWs3x5MWBvOoYTCWyfswbb56whLftcRlWLJzcj4hJBWojIYlWN9juOjGT7nDXYPmcNodpnqxoyxpgszhKBMcZkcVktEYzyOwAf2D5nDbbPWUNI9jlLtREYY4z5r6xWIjDGGJOEJQJjjMniMmUiEJFWIrJeRDaKyCPJzM8lIpO8+QtEpKwPYaarIPa5v4isFZGVIjJHRMr4EWd6Sm2fA5a7XkRURCL+UsNg9llEbvQ+6zUi8mFGx5jegvhulxaR70Rkmff9buNHnOlFRMaIyG4RWX2a+SIib3vvx0oRqZfmjapqpvrDdXn9O1AeyAmsAKonWaYXMMJ73BmY5HfcGbDPVwB5vcf3ZoV99pbLD8wDfgGi/Y47Az7nSsAyoLD3vITfcWfAPo8C7vUeVwc2+x13Gve5CVAPWH2a+W2AGYAAlwIL0rrNzFgiaABsVNVNqnoK+BjomGSZjsA47/FkoLlE9iDGqe6zqn6nqse8p7/gRoyLZMF8zgDPAi8DJzIyuBAJZp+7A0NVdT+Aqu7O4BjTWzD7rEAB73FB4M8MjC/dqeo83Pgsp9MRGK/OL0AhETk/LdvMjIngQmBbwPPt3rRkl1HVOOAgUDRDoguNYPY5UDfcGUUkS3WfvSJzKVWdlpGBhVAwn3NloLKI/CQiv4hIqwyLLjSC2eengVtFZDtu/JO+GROab870956qiBi83qQfEbkViAaa+h1LKIlINuB1oKvPoWS07LjqoWa4Ut88Eamlqgf8DCrEugBjVXWwiDTEjXpYU1UT/A4sUmTGEsEOoFTA85LetGSXEZHsuOJkTIZEFxrB7DMichUwEOigqiczKLZQSW2f8wM1gbkishlXlzolwhuMg/mctwNTVDVWVf8ANuASQ6QKZp+7AZ8AqOp8IDeuc7bMKqjf+5nIjIlgEVBJRMqJSE5cY/CUJMtMAe7wHt8AfKteK0yESnWfRaQuMBKXBCK93hhS2WdVPaiqxVS1rKqWxbWLdFDVxf6Emy6C+W5/iSsNICLFcFVFmzIwxvQWzD5vBZoDiEg1XCLYk6FRZqwpwO3e1UOXAgdVdWdaVpjpqoZUNU5E+gAzcVccjFHVNSIyCFisqlOA93DFx424RpnO/kWcdkHu86tAPuBTr118q6p28C3oNApynzOVIPd5JtBCRNYC8cBDqhqxpd0g93kA8K6IPIBrOO4aySd2IvIRLpkX89o9ngJyAKjqCFw7SBtgI3AMuDPN24zg98sYY0w6yIxVQ8YYY86AJQJjjMniLBEYY0wWZ4nAGGOyOEsExhiTxVkiMGFJROJFZHnAX9kUlj2SDtsbKyJ/eNta6t2heqbrGC0i1b3HjyWZ93NaY/TWk/i+rBaRr0WkUCrL14n03jhN6NnloyYsicgRVc2X3sumsI6xwFRVnSwiLYDXVLV2GtaX5phSW6+IjAM2qOrzKSzfFdfrap/0jsVkHlYiMBFBRPJ54ygsFZFVIvKfnkZF5HwRmRdwxny5N72FiMz3XvupiKR2gJ4HVPRe299b12oRud+bdo6ITBORFd70m7zpc0UkWkReAvJ4cUz05h3x/n8sIm0DYh4rIjeISJSIvCoii7w+5u8J4m2Zj9fZmIg08PZxmYj8LCJVvDtxBwE3ebHc5MU+RkQWessm12OryWr87nvb/uwvuT/cXbHLvb8vcHfBF/DmFcPdVZlYoj3i/R8ADPQeR+H6GyqGO7Cf403/H/BkMtsbC9zgPe4ELADqA6uAc3B3Za8B6gLXA+8GvLag938u3pgHiTEFLJMY47XAOO9xTlwvknmAHsDj3vRcwGKgXDJxHgnYv0+BVt7zAkB27/FVwGfe467AkIDXvwDc6j0uhOuL6By/P2/78/cv03UxYTKN46paJ/GJiOQAXhCRJkAC7kz4XGBXwGsWAWO8Zb9U1eUi0hQ3WMlPXtcaOXFn0sl5VUQex/VT0w3Xf80XqnrUi+Fz4HLgG2CwiLyMq0764Qz2awbwlojkAloB81T1uFcdVVtEbvCWK4jrLO6PJK/PIyLLvf1fB8wOWH6ciFTCdbOQ4zTbbwF0EJEHvee5gdLeukwWZYnARIpbgOJAfVWNFdejaO7ABVR1npco2gJjReR1YD8wW1W7BLGNh1R1cuITEWme3EKqukHcWAdtgOdEZI6qDgpmJ1T1hIjMBVoCN+EGWgE32lRfVZ2ZyiqOq2odEcmL63+nN/A2bgCe71T1Wq9hfe5pXi/A9aq6Pph4TdZgbQQmUhQEdntJ4ArgP2MuixuH+S9VfRcYjRvu7xegkYgk1vmfIyKVg9zmD8A1IpJXRM7BVev8ICIXAMdU9QNcZ37JjRkb65VMkjMJ11FYYukC3EH93sTXiEhlb5vJUjfaXD9ggPzTlXpiV8RdAxY9jKsiSzQT6Cte8Uhcr7Qmi7NEYCLFRCBaRFYBtwO/JrNMM2CFiCzDnW2/pap7cAfGj0RkJa5aqGowG1TVpbi2g4W4NoPRqroMqAUs9KpongKeS+blo4CViY3FSczCDQz0f+qGXwSXuNYCS8UNWj6SVErsXiwrcQOzvAK86O174Ou+A6onNhbjSg45vNjWeM9NFmeXjxpjTBZnJQJjjMniLBEYY0wWZ4nAGGOyOEsExhiTxVkiMMaYLM4SgTHGZHGWCIwxJov7f4A2bzMov1HkAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSP AUROC score: 0.62397265\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, auroc_score = auroc(test_y, ood_scores)\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "print('MSP AUROC score:', auroc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAiWklEQVR4nO3deZhcZZ328e+dTsgiCZAFBRJIJGEJCQRohIgvsm+GZHBBEFSUFwbHyDjjMIMgi8CrCJfLMITBOCAKAoqoEzWABIkMIyAdCZBFIEYIHXAICYEJSSDL7/3jOUVXN510dadOVbrP/bmuuvpsdep3OlB3P89zFkUEZmZWXL3qXYCZmdWXg8DMrOAcBGZmBecgMDMrOAeBmVnBOQjMzArOQWDdhqTTJf2mgu1ukHRxLWqqBUnPSTo6m75M0q31rsl6FgeBVUX2ZbVG0ipJ/yPpZknbVvMzIuJHEXFsBdudGxFXVPOzSySFpDey41wq6VuSGvL4rK6QNEjSdyQtyWr8czY/tN612dbLQWDVdFJEbAscADQCX2m7gaTeNa+q+vbLjvODwMeBz9a5HgAkbQPcD+wDHA8MAiYCy4H3dWF/PeHfyirgILCqi4ilwN3AOHj7r+jPS3oWeDZbNknSXEkrJf1e0r6l90saIelnkpZJWi7pumz5mZIeyqYl6duSXpb0uqSnJJU+72ZJV5bt72xJiyStkDRD0s5l60LSuZKezWqZJkkVHuci4L+BCWX768px7S7pt9myVyT9SNL2nfy1A3wK2BU4OSIWRMTGiHg5Iq6IiJllxzu6rKa3f1eSDpfULOlfJP0V+L6khZImlW3fO6v/gGz+kOw4V0p6QtLhXajb6sxBYFUnaQRwIvB42eK/AQ4GxkraH7gJ+FtgCPBdYIakvlk3y6+A54GRwC7AHe18zLHAYcAewHbAKaS/fNvWciTw9Wz9Ttl+2+5vEnAQsG+23XEVHudewP8BFmXzXT0uZTXuDOwNjAAuq6SGNo4G7omIVV14b8l7gMHAbsA5wO3AaWXrjwNeiYg/StoF+DVwZfaefwLukjRsCz7f6sBBYNX0C0krgYeA3wFfK1v39YhYERFrSF8w342IRyNiQ0T8AHgTOITUhbEzcH5EvBERayPioXY+ax0wENgLUEQsjIiX2tnudOCmiPhjRLwJfBmYKGlk2TZXRcTKiFgCPEDZX/ib8EdJbwALgdnA9dnyLh1XRCyKiPsi4s2IWAZ8i9Tt1FlDgPZ+B52xEbg0q2UNcBswWdKAbP0nSOEAcAYwMyJmZq2P+4Am0h8B1o04CKya/iYito+I3SLi77IvkpIXyqZ3A76UdSeszMJjBOmLcgTwfESs39wHRcRvgeuAacDLkqZLGtTOpjuT/govvW8VqeWwS9k2fy2bXg1sCyBpfjbgukrS/ynb5oBsm4+TWjnv2pLjkvRuSXdkg8+vA7cCXRncXU5q9WyJZRGxtjSTdX8tBE7KwmAyKRwgHe/H2hzvB6pQg9WYg8Bqpfw2ty8A/y8LjdJrQETcnq3btZKByoi4NiIOBMaSuojOb2ezF0lfWABIehfpL+elFex/n4jYNnv9V5t1ERE/AR4GLtnC4/oa6fczPiIGkf7Srmicoo1ZwHHZMW7KamBA2fx72qxv73bEpe6hKcCCLBwgHdMtbY73XRFxVRdqtzpyEFg9fA84V9LB2aDvuyR9SNJA4A+k7o2rsuX9JB3adgeSDsre3wd4A1hL6tZo63bgM5ImSOpL+tJ9NCKeq9KxXAWcLek9W3BcA4FVwGtZv3t7gVaJW0hfzndJ2ktSL0lDJF0oqdRdMxf4hKQGScdTWRfUHaQxmc/R0hqA1HI5SdJx2f76ZQPOw7tYv9WJg8BqLiKagLNJXTuvkgZbz8zWbQBOAkYDS4BmUhdMW4NIX7yvkrp+lgPXtPNZs4CLgbtIX8S7A6dW8VieAh4k9f139bi+Supueo00+PqzLtbyJmnA+E/AfcDrpAAaCjyabfb3WR0rSeMnv6hgvy+RWj7vB35ctvwFUivhQmAZKYTOx98r3Y78YBozs2JzcpuZFZyDwMys4BwEZmYF5yAwMyu4bndTqaFDh8bIkSPrXYaZWbcyZ86cVyKi3dt/dLsgGDlyJE1NTfUuw8ysW5H0/KbWuWvIzKzgHARmZgXnIDAzK7huN0ZgZtZV69ato7m5mbVr13a8cTfVr18/hg8fTp8+fSp+j4PAzAqjubmZgQMHMnLkSCp8EF23EhEsX76c5uZmRo0aVfH7cusaknST0mME521ivSRdq/QIwSdLj74zM8vL2rVrGTJkSI8MAQBJDBkypNMtnjzHCG4mPUB7U04AxmSvc4B/z7EWMzOAHhsCJV05vtyCICIeBFZsZpMpwA+zB3w8AmwvKb8nGz30EFxyCbz1Vm4fYWbWHdXzrKFdaP34wmZaPz7wbZLOkdQkqWnZsmVd+7SHH4YrroB167r2fjOzKmhoaGDChAmMGzeOk046iZUrV769bv78+Rx55JHsueeejBkzhiuuuILyRwXcfffdNDY2MnbsWPbff3++9KUvVaWmbnH6aERMj4jGiGgcNqzdK6TNzLqF/v37M3fuXObNm8fgwYOZNm0aAGvWrGHy5MlccMEFPP300zzxxBP8/ve/5/rrrwdg3rx5TJ06lVtvvZUFCxbQ1NTE6NGjq1JTPYNgKemB3iXDqeA5smZmPcXEiRNZujR97d12220ceuihHHvssQAMGDCA6667jquuSo+Avvrqq7nooovYa6+9gNSy+NznPleVOup5+ugMYKqkO4CDgdeyR+KZmeXvi1+EuXOru88JE+A736lo0w0bNnD//fdz1llnAalb6MADD2y1ze67786qVat4/fXXmTdvXtW6gtrKLQgk3Q4cDgyV1AxcCvQBiIgbgJnAiaTnuq4GPpNXLWZmW4s1a9YwYcIEli5dyt57780xxxxT75LyC4KIOK2D9QF8Pq/PNzPbrAr/cq+20hjB6tWrOe6445g2bRrnnXceY8eO5cEHH2y17eLFi9l2220ZNGgQ++yzD3PmzGG//farek3dYrDYzKynGTBgANdeey3f/OY3Wb9+PaeffjoPPfQQs2bNAlLL4bzzzuOf//mfATj//PP52te+xjPPPAPAxo0bueGGG6pSi4PAzKxO9t9/f/bdd19uv/12+vfvz3/+539y5ZVXsueeezJ+/HgOOuggpk6dCsC+++7Ld77zHU477TT23ntvxo0bx+LFi6tSh+81ZGZWQ6tWrWo1/8tf/vLt6fHjxzN79uxNvnfSpElMmjSp6jW5RWBmVnAOAjOzgnMQmFmhlN+yoSfqyvE5CMysMPr168fy5ct7bBiUnkfQr1+/Tr3Pg8VmVhjDhw+nubmZLt+8shsoPaGsMxwEZlYYffr06dSTu4rCXUNmZgXnIDAzKzgHgZlZwTkIzMwKzkFgZlZwDgIzs4JzEJiZFZyDwMys4BwEZmYF5yAwMys4B4GZWcE5CMzMCs5BYGZWcA4CM7OCcxCYmRWcg8DMrOAcBGZmBecgMDMrOAeBmVnBOQjMzArOQWBmVnAOAjOzgss1CCQdL+lpSYskXdDO+l0lPSDpcUlPSjoxz3rMzOydcgsCSQ3ANOAEYCxwmqSxbTb7CvCTiNgfOBW4Pq96zMysfXm2CN4HLIqIxRHxFnAHMKXNNgEMyqa3A17MsR4zM2tHnkGwC/BC2XxztqzcZcAZkpqBmcAX2tuRpHMkNUlqWrZsWR61mpkVVr0Hi08Dbo6I4cCJwC2S3lFTREyPiMaIaBw2bFjNizQz68nyDIKlwIiy+eHZsnJnAT8BiIiHgX7A0BxrMjOzNvIMgseAMZJGSdqGNBg8o802S4CjACTtTQoC9/2YmdVQbkEQEeuBqcC9wELS2UHzJV0uaXK22ZeAsyU9AdwOnBkRkVdNZmb2Tr3z3HlEzCQNApcvu6RsegFwaJ41mJnZ5tV7sNjMzOrMQWBmVnAOAjOzgitOEPzwh+nn0rZnsJqZFVtxgmDevPTzT3+qbx1mZluZ4gSBmZm1q3hBsH49rFhR7yrMzLYaxQuCj3wEhgyBf/1XB4KZGUUMgpIvfjEFgplZwRU3CMzMDCh6EPQq9uGbmUHRg2DjxnpXYGZWd8UOAjMzcxCYmRWdg8CPPzCzgituEJQGitetq28dZmZ1VswgaGiAq65K0+vWgZReGzfChg1p+qMfrW+NZmY1kusTyrZahx0G22yTpidNalne0NAyfdddta3JzKxOitkiuPxy+N3v0vTs2Zve7uCDa1KOmVk9FTMItt8edtyx4+3+8IfcSzEzq7diBsGYMXD00a2XffvbcM896Syi8jOJ1qypbW1mZjVWzDGCvn3hvvta5lesgB12aL3NqFHwl7/AgAGbP8U0Ig0uAzz3HLznPdCvX9VLNjPLSzFbBADvfW/LdNsQABg/vmVagrfeSuMJhx8OTz0F11yTlvfq1XLW0ahR0L8/fO97LctOPx3OOqtl/oMfzPvIzMw6RdHNLqhqbGyMpqamzr+x9Fc7pL/iN2yA3r1b5tu64gq45JKuFVmJbvZ7N7PuTdKciGhsb11xWwSlU0XPOaf99RdcALfd1n5roaRPH5gzJ3UJLV0Kb77Zsu7gg2G77VpvXzplFVpaCBLMmgVTprSMT7zxhm+IZ2Y1U9wWQWfMmgUXXwwXXggnnQQrV6Yzj7ri/vvfOVC9Kb16wYsvwh13wMknw667wiuvpAC65hr4+c/hoYdSaPTuDYMGpUdxvvxyen+fPqnWMWNaH7+ZFc7mWgQOgnrYuBGWL0+tiP33r+1n//znqcVxwgkweHBtP9vM6sZdQ1ubXr1g2DCYMKGlO2jdOliwIE2/9RZMn976Sue2OvoLv2/f9peffDKccUZ6TGd599SmXqedlsZTStau9f2ZzHoYB8HWondv2HvvNN2nD5x9durmKQVF29fGjZteF5G+sMvnN2xIXUeddccdqbZSMPTvn8Y6ysPiuOPg1Vehubn+rS0z6zQHQVH06gWvvbb58CgFTClknngCjjmm433/5jepm2nEiJbTaSdPTu///e/hxhvTOMsXvgD/9m8pNDZsgCVLWrc2zKwuch0jkHQ88K9AA/AfEXFVO9ucAlwGBPBERHxic/vsEWMEPcn69an7aM6c1N1VjdtyTJmSXiNGwAc+4Av0zKqgLoPFkhqAZ4BjgGbgMeC0iFhQts0Y4CfAkRHxqqQdI+Llze3XQdANLF0KX/0q/PCHcMAB8PDD8OEPw047pbC4/vo0PXgwPPBA5/Z97LHw7nenlsqUKbBqVRqz2LAh/Xz9dZg3L7VMSmMZo0al7qwVK9IFfV0948usG6tXEEwELouI47L5LwNExNfLtrkaeCYi/qPS/ToIerCI1LL4zW/S/PTp8Pzz+X/uUUel03pPPjmFx6mnpgDbc8+WcRCzbq5eQfBR4PiI+L/Z/CeBgyNiatk2vyC1Gg4ldR9dFhH3tLOvc4BzAHbdddcDn+/Kl4ODoGdYuRKefRZOOQV23jkNgP/5z+lushMmpK6qPfZIrY0NG+DBB9OykSNh4UJ4+ul0dlZn7bhjuj5jjz3SxYJLl6azuvr0SYPpq1enbrFttkl1jRsHxx8PBx6Ytlm6NA2m9+mTrkUZMKDKvxizzdviIJB0KKkffzfSjeoERES8dzPvqSQIfgWsA04BhgMPAuMjYuWm9usWgVVdaZB81ap0P6kZM1LLZP58GDgwDW5Dusq8NF0tn/xkuuDviCNScO2zDwwdWt3PMGPzQVDp3UdvBP4BmANUeprHUmBE2fzwbFm5ZuDRiFgH/EXSM8AY0niCWW1I6a/77bZrGajenPJrP5YtS+MdDQ0tgbJ8OTzyCLz0UrqA76CD0nUd++4LL7wAV1+dWhGLF8Mtt7xz/wMHptN0Bw9OF//tvHNqhfTtCxddlD5r223z+V1YIVXaIng0Ijr1uC5JvUndPkeRAuAx4BMRMb9sm+NJA8ifljQUeByYEBHLN7VftwisR3nppdRl1dycbnu+eHG6u+3jj6f122yTLjCsVEND6r5auDDdyuSxx9JpwyeckAbQ3/3uFEZTpqRuqwkTfFZWQVSjRfCApGuAnwFv31ktIv64qTdExHpJU4F7Sf3/N0XEfEmXA00RMSNbd6ykBaSWxvmbCwGzHmenndJrc9avT2c83XRTujZj2TJ49NF0BtSvf92y3ZAhKTgWLkyth1mz0vI+fdIAfPk1G+XP4ygZNiwNku++ewqPceNgt91gl11gv/1SIA0dmlpCDQ0tt2G3bq/SFkF75/hFRBxZ/ZI2b4tbBJ/+NNx8c1VrMusWItIdct94Iw18/+hH6aaFCxakcIF0Y8MlSzq/72HDWsY2Fi5MtzHZaacUFI2NKaRGjUr7BwdIHWxxiyAijqhuSXW0ufv3mPVkUuoG6tcvfTFfeWX720XAokUpMJqa0h1wX3ghvX/VqvSF39CQBtSl1I21bFlLmADceuvmaxk8OLVyGhpSUAweDP/7v2mfvXqlU3oPOwwmToRDDnFw5KyiIJC0HXApcFi26HfA5RHxWl6FmVmdSOlMJkhjCF2xYUMa/yi91qyBu+5KA/IbN8Ltt6cB87594ckn0xf9G2+0vH/GjPQq2XPP1O3V2JguThw+PN2bq6Gh5QFT1mWV/gZvAuaRTvME+CTwfeDDeRSVK/9lYZa/hob0ZT18eMuyj3+8ZfrGGzf//hUrUrfVkiWpq+n669Pyp56C739/0+/bc8/U7bXDDunnqlVpoHzNmnRV+nvfm1pD48algXNfLAhUHgS7R8RHyua/KmluDvWYmaWuosmTW+anTUs/58+H665LF/D16pUGyw85BP7nf9JFerNnp/GKJUvSIDvA3Xenn7Nnt/9Z++yTQmPixDQdAR/6UDr76l3vSq8ertIgWCPpAxHxELx9gdma/MrKkVsEZt3XPvvAv/97596zbh389a/wpz+llsHLL6fTde+6K3WBPfhg6pYq7466qs39MbfbLrU2XnklnZa7YQN89rPpDKsdd+z2LYtKzxqaAPwA2I50VfEK4MyIeCLX6tqxxWcNnXtu5/9DMrOeLyJdDPjnP8Mzz6T7XP3qV+n2JPfdl7qrNmX06PS0wVNOSYExfnzNyq5UNc4amgvsJ2lQNv969cqrsW6e3GaWEymdETV0aLqSG+ArX3nndq++CnPnwh//mFobTU3ws5+lM63uvLNlu/79061Dxo9PPwcPToPvffrU4mg6ZbNBIOmMiLhV0j+2WQ5ARHwrx9ry4dNHzWxL7LBD+mI/os1Z9c8/nwLilltSy2LVKpg5M72+8Y3W2/btm8YwLrkEPvWp1Oqoo45aBKVRkoF5F1IzHiMwszzstlt6ld+ravXq9JS+N99M3UuzZqUuqNIdcC+9NL0g3ZX2qqtg7Nial77ZIIiI72Y/v1qbcmrAQWBmtTJgQBpchnQmUrn169PNCa+8MgXDL3+ZXpBuVjhlSs26siv6VpR0taRBkvpIul/SMkln5F1cLhwEZrY16N07PYr1nnvS6a5z57bcVfbkk9N31cSJqVWRs0q/FY/NBognAc8Bo4Hz8yoqVw4CM9sa7bdfus3Giy/C5ZenZY88kq5j2NQ1EFVS6bdiqQvpQ8Cd3frWEg4CM9ua7bQTXHxxOiOpdAX2EUfAhRfm9pGVfiv+StKfgAOB+yUNA9bmVlWeHARm1h307p0uWnv22XTn1q9/HW64IZePquhbMSIuAN4PNGZPE3sD6OAxTlupj32s3hWYmVVu9Oh09fOkSWnMIAcdXUdwZET8VtKHy5aVb/KzXKrKw8iR8Nxz6RxgM7PuZPjwljOKctDRdQQfBH4LnNTOuqA7BYGZmbWro+sILs1+fqY25ZiZWa1Veh3B1yRtXza/g6RNPN7IzMy6k0pPoTkhIlaWZiLiVeDEXCoyM7OaqjQIGiT1Lc1I6g/03cz2ZmbWTVT6YJofka4fKD0j7jOk5xOYmVk3V+nzCL4h6Qkgu3sSV0TEvfmVlYMKHsBjZlZElbYIABYC6yNilqQBkgZGxP/mVVhu/GAaM7NWKj1r6Gzgp8B3s0W7AL/IqSYzM6uhSgeLPw8cCrwOEBHPAjvmVZSZmdVOpUHwZkS8VZqR1Jt0ZbGZmXVzlQbB7yRdCPSXdAxwJ5DfjS/MzKxmKg2CfwGWAU8BfwvMBL6SV1FmZlY7HZ41JKkBmB8RewHfy78kMzOrpQ5bBBGxAXha0q41qCc/vo7AzKxdlXYN7QDMzx5cP6P06uhNko6X9LSkRZIu2Mx2H5EUkhorLbzLfB2BmVkrlV5QdnFnd5x1KU0DjgGagcckzYiIBW22Gwj8PfBoZz/DzMy2XEdPKOsHnAuMJg0U3xgR6yvc9/uARRGxONvXHaTHWy5os90VwDeA8ztRt5mZVUlHXUM/ABpJIXAC8M1O7HsX4IWy+eZs2dskHQCMiIhfb25Hks6R1CSpadmyZZ0owczMOtJR19DYiBgPIOlG4A/V+mBJvYBvAWd2tG1ETAemAzQ2NnrU18ysijpqEawrTXSiS6hkKTCibH54tqxkIDAOmC3pOeAQYEZNBozNzOxtHbUI9pP0ejYt0pXFr2fTERGDNvPex4AxkkaRAuBU4BOllRHxGjC0NC9pNvBPEdHU6aMwM7Mu6+jh9Q1d3XFErJc0FbgXaABuioj5ki4HmiKiw9NPq8rXEZiZtaszzyPotIiYSbodRfmySzax7eF51vI2X0dgZtZKpReUmZlZD+UgMDMrOAeBmVnBOQjMzArOQWBmVnAOAjOzgnMQmJkVXHGCwBeUmZm1qzhBUOILyszMWileEJiZWSsOAjOzgnMQmJkVnIPAzKzgHARmZgXnIDAzK7jiBIGvIzAza1dxgqDE1xGYmbVSvCAwM7NWHARmZgXnIDAzKzgHgZlZwTkIzMwKzkFgZlZwxQkCX0dgZtau4gRBia8jMDNrpXhBYGZmrTgIzMwKzkFgZlZwDgIzs4JzEJiZFVyuQSDpeElPS1ok6YJ21v+jpAWSnpR0v6Td8qzHzMzeKbcgkNQATANOAMYCp0ka22azx4HGiNgX+ClwdV71+DoCM7P25dkieB+wKCIWR8RbwB3AlPINIuKBiFidzT4CDM+xnsTXEZiZtZJnEOwCvFA235wt25SzgLvbWyHpHElNkpqWLVtWxRLNzGyrGCyWdAbQCFzT3vqImB4RjRHROGzYsNoWZ2bWw/XOcd9LgRFl88OzZa1IOhq4CPhgRLyZYz1mZtaOPFsEjwFjJI2StA1wKjCjfANJ+wPfBSZHxMs51mJmZpuQWxBExHpgKnAvsBD4SUTMl3S5pMnZZtcA2wJ3SporacYmdmdmZjnJs2uIiJgJzGyz7JKy6aPz/HwzM+vYVjFYXBO+jsDMrF3FCYISX0dgZtZK8YLAzMxacRCYmRWcg8DMrOAcBGZmBecgMDMrOAeBmVnBOQjMzAquOEHgC8rMzNpVnCAo8QVlZmatFC8IzMysFQeBmVnBOQjMzArOQWBmVnAOAjOzgnMQmJkVXHGCwNcRmJm1qzhBUOLrCMzMWileEJiZWSsOAjOzgnMQmJkVnIPAzKzgHARmZgXnIDAzK7jiBIGvIzAza1dxgqDE1xGYmbVSvCAwM7NWHARmZgXnIDAzKzgHgZlZweUaBJKOl/S0pEWSLmhnfV9JP87WPyppZJ71mJnZO+UWBJIagGnACcBY4DRJY9tsdhbwakSMBr4NfCOveszMrH15tgjeByyKiMUR8RZwBzClzTZTgB9k0z8FjpJyOr9zwID006ePmpm10jvHfe8CvFA23wwcvKltImK9pNeAIcAr5RtJOgc4B2DXXXftWjWzZsGdd8KwYV17v5lZD9UtBosjYnpENEZE47CufpGPHg1f/nJ1CzMz6wHyDIKlwIiy+eHZsna3kdQb2A5YnmNNZmbWRp5B8BgwRtIoSdsApwIz2mwzA/h0Nv1R4LcRvimQmVkt5TZGkPX5TwXuBRqAmyJivqTLgaaImAHcCNwiaRGwghQWZmZWQ3kOFhMRM4GZbZZdUja9FvhYnjWYmdnmdYvBYjMzy4+DwMys4BwEZmYF5yAwMys4dbezNSUtA57v4tuH0uaq5QLwMReDj7kYtuSYd4uIdq/I7XZBsCUkNUVEY73rqCUfczH4mIshr2N215CZWcE5CMzMCq5oQTC93gXUgY+5GHzMxZDLMRdqjMDMzN6paC0CMzNrw0FgZlZwPTIIJB0v6WlJiyRd0M76vpJ+nK1/VNLIOpRZVRUc8z9KWiDpSUn3S9qtHnVWU0fHXLbdRySFpG5/qmElxyzplOzfer6k22pdY7VV8N/2rpIekPR49t/3ifWos1ok3STpZUnzNrFekq7Nfh9PSjpgiz80InrUi3TL6z8D7wW2AZ4AxrbZ5u+AG7LpU4Ef17vuGhzzEcCAbPpzRTjmbLuBwIPAI0Bjveuuwb/zGOBxYIdsfsd6112DY54OfC6bHgs8V++6t/CYDwMOAOZtYv2JwN2AgEOAR7f0M3tii+B9wKKIWBwRbwF3AFPabDMF+EE2/VPgKKlbP9W+w2OOiAciYnU2+wjpiXHdWSX/zgBXAN8A1tayuJxUcsxnA9Mi4lWAiHi5xjVWWyXHHMCgbHo74MUa1ld1EfEg6fksmzIF+GEkjwDbS9ppSz6zJwbBLsALZfPN2bJ2t4mI9cBrwJCaVJePSo653Fmkvyi6sw6POWsyj4iIX9eysBxV8u+8B7CHpP+W9Iik42tWXT4qOebLgDMkNZOef/KF2pRWN539/71DuT6YxrY+ks4AGoEP1ruWPEnqBXwLOLPOpdRab1L30OGkVt+DksZHxMp6FpWz04CbI+KbkiaSnno4LiI21ruw7qIntgiWAiPK5odny9rdRlJvUnNyeU2qy0clx4yko4GLgMkR8WaNastLR8c8EBgHzJb0HKkvdUY3HzCu5N+5GZgREesi4i/AM6Rg6K4qOeazgJ8ARMTDQD/Szdl6qor+f++MnhgEjwFjJI2StA1pMHhGm21mAJ/Opj8K/DayUZhuqsNjlrQ/8F1SCHT3fmPo4Jgj4rWIGBoRIyNiJGlcZHJENNWn3Kqo5L/tX5BaA0gaSuoqWlzDGqutkmNeAhwFIGlvUhAsq2mVtTUD+FR29tAhwGsR8dKW7LDHdQ1FxHpJU4F7SWcc3BQR8yVdDjRFxAzgRlLzcRFpUObU+lW85So85muAbYE7s3HxJRExuW5Fb6EKj7lHqfCY7wWOlbQA2ACcHxHdtrVb4TF/CfiepH8gDRyf2Z3/sJN0OynMh2bjHpcCfQAi4gbSOMiJwCJgNfCZLf7Mbvz7MjOzKuiJXUNmZtYJDgIzs4JzEJiZFZyDwMys4BwEZmYF5yAwa4ekDZLmSpon6ZeStq/y/p/LzvNH0qpq7tussxwEZu1bExETImIc6VqTz9e7ILO8OAjMOvYw2U29JO0u6R5JcyT9l6S9suXvlvRzSU9kr/dny3+RbTtf0jl1PAazTepxVxabVZOkBtLtC27MFk0Hzo2IZyUdDFwPHAlcC/wuIk7O3rNttv1nI2KFpP7AY5Lu6s5X+lrP5CAwa19/SXNJLYGFwH2StgXeT8ttOgD6Zj+PBD4FEBEbSLc2BzhP0snZ9AjSDeAcBLZVcRCYtW9NREyQNIB0n5vPAzcDKyNiQiU7kHQ4cDQwMSJWS5pNuiGa2VbFYwRmm5E91e080o3NVgN/kfQxePvZsftlm95PegQokhokbUe6vfmrWQjsRboVttlWx0Fg1oGIeBx4kvQAlNOBsyQ9Acyn5bGJfw8cIekpYA7p2bn3AL0lLQSuIt0K22yr47uPmpkVnFsEZmYF5yAwMys4B4GZWcE5CMzMCs5BYGZWcA4CM7OCcxCYmRXc/wfpd1F42JYlEwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSP AUPR score: 0.5944642299032242\n"
     ]
    }
   ],
   "source": [
    "precision, recall, aupr_score = aupr(test_y, ood_scores)\n",
    "# aupr_score = average_precision_score(test_y, ood_scores)\n",
    "\n",
    "plot_precision_recall_curve(precision, recall)\n",
    "print('MSP AUPR score:', aupr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSP fpr95 score: 0.8366\n"
     ]
    }
   ],
   "source": [
    "fpr95 = fpr_at_95_tpr(test_y, ood_scores)\n",
    "print('MSP fpr95 score:', fpr95)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OOD detection by Mahalanobis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n",
      "100%|██████████| 98/98 [00:25<00:00,  3.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "covariance norm: 14.235700347525052\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.metrics.mahalanobis import Mahalanobis\n",
    "from ood_with_vit.visualizer.feature_extractor import FeatureExtractor\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.Resize(img_size),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(dataset_mean, dataset_std),\n",
    "])      \n",
    "cifar100 = CIFAR100(\n",
    "    root=dataset_root, \n",
    "    train=True, \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "id_train_dataloader = DataLoader(\n",
    "    dataset=cifar100, \n",
    "    batch_size=config.train.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "feature_extractor = FeatureExtractor(\n",
    "    model=model,\n",
    "    layer_name=config.model.layer_name.penultimate,\n",
    ")\n",
    "mahalanobis = Mahalanobis(config, model, id_train_dataloader, feature_extractor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: patch_embed.norm\n",
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:07<00:00, 13.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:07<00:00, 13.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id mean score: 388.8107 id variance: 7660.762\n",
      "ood mean score: 433.76346 ood variance: 4476.076\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# hook mask_patch\n",
    "simmim = SimMIMHooker(\n",
    "    config=config,\n",
    "    model=model,\n",
    "    patch_embedding_layer_name='patch_embed.norm',\n",
    "    mask_ratio=mask_ratio,\n",
    ")\n",
    "simmim.hook()\n",
    "\n",
    "test_y, ood_scores, id_ood_scores, ood_ood_scores = compute_ood_scores(\n",
    "    metric=mahalanobis,\n",
    "    in_dist_dataloader=id_test_dataloader,\n",
    "    out_of_dist_dataloader=ood_test_dataloader,    \n",
    ")\n",
    "print('id mean score:', np.mean(id_ood_scores), 'id variance:', np.var(id_ood_scores))\n",
    "print('ood mean score:', np.mean(ood_ood_scores), 'ood variance:', np.var(ood_ood_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABC50lEQVR4nO3dd3gUVffA8e8hdOlNkd47IkYR6SJVFH0RRRFFeUGaoKK+KqjIix0UlSIoYAERRVEEFPmpiK9I71UQqYJAKAKhJTm/P+5E1xiSBbI7SfZ8nmef7M7MzpzZ3cyZe+/MvaKqGGOMiVxZ/A7AGGOMvywRGGNMhLNEYIwxEc4SgTHGRDhLBMYYE+EsERhjTISzRJDJicg6EWnqdxzphYg8ISJv+7Ttd0RkqB/bTmsi0llEvj7P99pvMp2xRBBGIrJNRE6IyDER2esdGPKEcpuqWkNV54VyG4lEJIeIPC8iO7z93Cwij4iIhGP7ycTTVER2BU5T1edU9d8h2p6ISD8RWSsix0Vkl4h8LCK1QrG98yUig0Vk0oWsQ1Unq2rLILb1j+R3vr9JEcnuxb7Z+3y3icgEESl7rusyf2eJIPxuUNU8QB3gcuBxf8M5dyKS9SyzPgaaA22BvEAXoAfwWghiEBFJb7/f14D+QD+gEFAZ+Ay4Pq03lMJ3EHI+bnsacCNwB5AfuAxYhvvNnRM/P790SVXtEaYHsA24LuD1S8CsgNdXAwuAw8AqoGnAvELAROA34BDwWcC8dsBK730LgNpJtwlcCpwACgXMuxw4AGTzXt8LbPDWPwcoE7CsAn2AzcCvyexbc+AkUCrJ9HpAPFDRez0PeB5YDPwBfJ4kppQ+g3nAs8CP3r5UBO7xYj4KbAXu85a9yFsmATjmPS4FBgOTvGXKevt1N7DD+ywGBmwvF/Cu93lsAB4Fdp3lu63k7edVKXz/7wCjgFlevIuACgHzXwN2ep/LMqBRwLzBuAPhJG/+v4GrgJ+8z2oPMBLIHvCeGsBc4CDwO/AE0Bo4DZzxPpNV3rL5gfHeenYDQ4Eob15X7zN/FYjx5nUF/ufNF2/ePi+2NUBN3EnAGW97x4Avkv4fAFFeXL94n8kykvyGvOWu877Pf8xL4f8rue+6m/ddzwe+BPomWccq4F/e86oBn98m4Fa/jyEhOzb5HUAkPZL8A5T0/mFe816X8P7J2uJKai2810W9+bOAqUBBIBvQxJt+ufcPWM/7p7rb206OZLb5LdA9IJ6XgTe95+2BLUA1ICswCFgQsKx6/xSFgFzJ7NsLwPdn2e/t/HWAnucdaGriDtafBPyzpvYZzPP+iWt4MWbDnW1XwB2MmgCxQF1v+aYkOXCf5eDwFu6gfxlwCqgWuE/eZ14SWJ10fQHr7QlsT+X7f8fbn6u8+CcDHwbMvxMo7M0bAOwFcgbEfQa4yftscgFX4BJnVm9fNgAPeMvnxR3UBwA5vdf1kn4GAdueDoz1vpNiuESd+J11BeKA+71t5eLviaAV7gBewPseqgHFA/Z5aAr/B4/g/g+qeO+9DCh8Lr+v5Nabwnf9nrePuYC7gB8Dlq+OS6o5vGV24k40svLXSVN1v48joXikt6J1JPhMRI7ifmT7gKe96XcCs1V1tqomqOpcYCnQVkSKA22Anqp6SFXPqOr33vt6AGNVdZGqxqvqu7iD2dXJbPsD4HZwVStAJ28auAPZ86q6QVXjgOeAOiJSJuD9z6vqQVU9kcy6i+AOPMnZ481P9L6qrlXV48CTwK0iEpXSZxDw3ndUdZ2qxnmfwyxV/UWd74GvgUZnieNsnlHVE6q6CndGeJk3/VbgOe8z3wW8nsI6Cqew/4Gmq+pi7zOejKsiBEBVJ6lqjLdvw3EHpCoB7/1JVT/zPpsTqrpMVRd6y2/DHcibeMu2A/aq6nBVPamqR1V1UXIBicjFuM/4AVU9rqr7cGf4nQIW+01V3/C2lfT7P4NLNFUB8X5DwXwW4Eo2g1R1k/cdrlLVmGSWC/bzTc1gbx9P4JJf4G+8M/Cpqp7CfX7bVHWit88rcCctHdMghnTHEkH43aSqeXFnq1X56wBZBugoIocTH0BDoDhQCjioqoeSWV8ZYECS95XCVYMk9QlQ30ssjXHVJj8ErOe1gHUcxJ2hlQh4/84U9uuAF2tyinvzk1vPdtyZfRFS/gySjUFE2ojIQhE56C3flr8nnWDsDXgeCyQ24F+aZHsp7X8MZ9//YLaFiDwsIhtE5Ii3L/n5+74k3ffKIjLTu/DgD1zyTly+FK66JRhlcN/BnoDPfSyuZJDstgOp6re4aqlRwD4RGSci+YLcdrBxBvv5pubP/VDVo7iSdmLCux2XnMF9JvWS/BY7A5ekQQzpjiUCn3hnr+8Aw7xJO3FnygUCHhep6gvevEIiUiCZVe0Enk3yvtyqOiWZbR7CnTHfhmtw+1BVNWA99yVZTy5VXRC4ihR26f9w/zilAieKSD3cP/u3AZMDlymNO6M8kMpn8I8YRCQHLrkNAy5W1QLAbFwCSy3eYOzBVQklF3dS3wAlRST6fDYkIo1wbRC3AgW9fTnCX/sC/9yfMcBGoJKq5sPVtScuvxMof5bNJV3PTlwpskjA555PVWuk8J6/r1D1dVW9Ale9UhlX5ZPq+7xtV0hlGXC/r6tEpGQKyxwHcge8Tu6gnTSeKcDtIlIfV4X2XUBc3yf5LeZR1V5BxJrhWCLw1wighYhchmsEvEFEWolIlIjk9C5/LOkVs78ERotIQRHJJiKNvXW8BfQUkXrelTQXicj1IpL3LNv8AFc3egt/VQsBvAk8LiI1AEQkv4gEXQxW1f/DHQw/EZEa3j5c7e3XGFXdHLD4nSJSXURyA0OAaaoan9JncJbNZsdVn+wH4kSkDRB4SePvQGERyR/sfiTxEe4zKSgiJYC+Z1vQ27/RwBQv5uxe/J1E5LEgtpUXVw+/H8gqIk8BqZ1V58U1zh4TkapA4EFqJlBcRB4Qd1lvXi8pg/tcyiZedeX9vr4GhotIPhHJIiIVRKQJQRCRK73fXzbcwfgkrrSZuK2zJSSAt4H/ikgl7/dbW0QKJ13I+33NBaaLyBUiktXbp54icq+32Eqgk/f/EY37jadmNu7sfwgwVVUT454JVBaRLt76snn7WS2IdWY4lgh8pKr7cY1XT6nqTlyD7RO4g8FO3FlV4nfUBXfmvBHXtvCAt46lQHdc0fwQrsG3awqbnYG7wmWvVyeeGMt04EXgQ6+aYS2uXeJcdMCdUX2Fu0pkEu5KlPuTLPc+rjS0F3cW1s+LIbXP4G+8on0/3AH7EK6UMyNg/kbcGd9Wr3ifXHVZSoYAu4BfcWek03BnzmfTj7+qSA7jqjxuBr4IYltzcJ/bz7jqspOkXBUF8DBun4/iTgimJs7wPpsWwA24z3kz0Myb/bH3N0ZElnvP78Il1vW4z3IawVfF5PO2f8iLPQZ3IQK477+69/l/lsx7X8F9f1/jktp4XENucm7BHbin4kpLa4Fo3HcDrr2pghfHM/z9RCdZXnvAp7irkj4ImH4Ud1LRCXel3l7c/0eO1NaZEclfNQPGhJ6IzMNdyeHL3b0XQkR6AZ1UNagzZWMyCisRGHMWIlJcRBp4VSVVcJdiTvc7LmPSmt1dZ8zZZcddPVMOV9XzIa4dwJhMxaqGjDEmwlnVkDHGRLgMVzVUpEgRLVu2rN9hGGNMhrJs2bIDqlo0uXkZLhGULVuWpUuX+h2GMcZkKCKy/WzzrGrIGGMinCUCY4yJcJYIjDEmwmW4NoLknDlzhl27dnHy5Em/QwmZnDlzUrJkSbJly+Z3KMaYTCZTJIJdu3aRN29eypYti/gzPG5IqSoxMTHs2rWLcuXK+R2OMSaTCVnVkLhBpfeJyNqzzBcReV1EtojIahGpe77bOnnyJIULF86USQBARChcuHCmLvEYY/wTyjaCd3Djo55NG1wvmJVwo2yNuZCNZdYkkCiz758xxj8hqxpS1fkiUjaFRdoD73kDoywUkQIiUvwchrgzxpjMQRUOHoSYGNi/H44ehZMnYds2yJ6d48cPs3/7esre3R+uvDLNN+9nG0EJ/t7f+i5v2j8SgYj0wJUaKF26dFiCO1dRUVHUqlWLuLg4ypUrx/vvv0+BAgUAWLduHffffz+7d+8mISGBu+66i0GDBv15lv/ll1/y5JNPEhsbS44cObj22msZPny4j3tjjElTx4/D7t2wcSP8+CP89ps78P/6K8TGwvaz3uvFt+Wg+w2QPw6WzqlClkyWCIKmquOAcQDR0dHpspe8XLlysXLlSgDuvvtuRo0axcCBAzlx4gQ33ngjY8aMoWXLlsTGxtKhQwdGjx5Nnz59WLt2LX379mXWrFlUrVqV+Ph4xo0b5+/OGGPOzZkzsGULbNjgHqtWwfz5kCMH7Nhx9veVLg3FikHz5nDRRVC5MpQpA4ULczhrHI9seJ23t31CxQLlebXdW2SpcG1IwvczEezm72PAlvSmZXj169dn9erVAHzwwQc0aNCAli3dCIq5c+dm5MiRNG3alD59+vDSSy8xcOBAqlatCriSRa9emXJYVGMyh99+g8WL4euvYdEiWLMGEhIgPv6vZYoVg9q1XYJo2xayZIEKFaBePaheHQoWTHET8QnxXDOmFptiNvHoNY8yuOlgcmU728BtF87PRDAD6CsiHwL1gCNp0j7wwAPgnZmnmTp1YMSIoBaNj4/nm2++oVu3boCrFrriiiv+tkyFChU4duwYf/zxB2vXrmXAgAFpG68xJm3s3u3O7DduhKVLYfbsv8+vWBGio6F8eWjWDEqWhMsug0suOa/NxcTGUChXIaKyRPHstc9SKn8poi+NToMdSVnIEoGITAGaAkVEZBfwNJANQFXfxI092hY3xm4scE+oYgmHEydOUKdOHXbv3k21atVo0aKF3yEZY87H5s0wdKir3lm16q/pBQtCx45QvDg0bQrXXgv586fJJlWVyWsm0/+r/rzQ/AW6X9Gdm6vdnCbrDkYorxq6PZX5CvRJ8w0Heeae1hLbCGJjY2nVqhWjRo2iX79+VK9enfnz5/9t2a1bt5InTx7y5ctHjRo1WLZsGZdddpkvcRsT0U6ccFU7334LEyfCvn1w+PBf8/v3h1tugbJloUQJCMFl3DuP7KTnrJ7M3jybq0teTYPSDdJ8G6lS1Qz1uOKKKzSp9evX/2NauF100UV/Pl++fLmWLl1az5w5o7GxsVquXDmdO3euqqrGxsbq9ddfr6+//rqqqq5atUorVKigmzZtUlXV+Ph4HTNmTLLbSA/7aUyGtn+/6mefqd5zj2qzZqruwk33KFVKtUYN1UGDVL/4QjUhIeThfLD6A837XF7N/WxuHfHTCI2LjwvZtoClepbjaoa4aiijufzyy6lduzZTpkyhS5cufP7559x///306dOH+Ph4unTpQt++fQGoXbs2I0aM4Pbbbyc2NhYRoV27dj7vgTGZREKCa9T97juYNg22bv1rXqFCcP/9rhG3fXt31h9mBXMVpF7JeoxrN45yBf3rPibDjVkcHR2tSQem2bBhA9WqVfMpovCJlP005oKsXAkffgiTJ8OuXW5aVBRUq+Yadm+4AVq0gLx5wx5aXEIcr/70KqfjTzOw8UDA1cqEo+cAEVmmqsm2PFuJwBiT8W3bBv/9r7u6Z8ECN61GDXfgb94cunWDXKG7/DIYq/auotuMbizbs4xba9z6ZwJID93HWCIwxmQ8GzbAxx/DwoXuceiQm37JJfDQQ9CzJ1Sq5G+MnlNxpxg6fygv/PgChXIV4uOOH9OhWod0kQASZZpEEK7ilV8yWhWeMWlu+3Z491348kt38Ad3ll+3rjv779IFGjb0N8ZkbD64mRd/fJE7at3BKy1foXDuwn6H9A+ZIhHkzJmTmJiYTNsVtXrjEeTMmdPvUIwJr9OnYfp0mDQJZs500/LmdY28PXpAzZr+xncWx04f4/ONn9O5dmdqFqvJxr4bKV+wvN9hnVWmSAQlS5Zk165d7N+/3+9QQiZxhDJjMrWEBNfQO3YsrF/veuNMLA3fey907uxu5ErH5v4ylx4ze7D98HbqFq9LtaLV0nUSgEySCLJly2YjdxmTkf36K4wZA6+88lefPZUqQaNGcOut0K4d5Mnjb4ypOHTiEA9//TATVk6gcuHKfN/1e6oVzRhX+WWKRGCMyWASEtwdva++Cj/88Nf1/SLw8svQp4/vV/mci/iEeBpMaMDPMT/zeMPHearJU+TMmnGqci0RGGPC5+RJGDYMnnvOde+Q6I474OGH4fLL/YvtPByIPfBnJ3HPNX+O0vlLU7f4eY+66xtLBMaY0FqwwPXgOX++u+IHXJ/7PXq4K31KlUr5/emQqvL+6vd54KsHeOG6F+hxRQ9uqnqT32GdN0sExpjQmDnTXc+/2xtmpEQJ19jboQPcdFNIOnALh+2Ht3PfzPuY88scril1DY3LNPY7pAtmicAYk3ZOnICpU13d/+rVrsvmfv2gd283+lYGPfgnmrR6Er1m9UJVeaPNG/S+sjdZJIvfYV0wSwTGmAt38CD85z/w9tvudeHCrkv4++6DTHT/S9HcRWlQqgFj242lTIEyfoeTZiwRGGPO39at8OKLkDjOdrt2cNttbgCXHDn8jS0NnIk/w/CfhnMm/gxPNnmSVhVb0bJCy0x346olAmPMuVu/Hho0+GsQl5tvhkcfhauv9jWstLRizwq6zejGir0r6FSzU7rqJC6tWSIwxgQnLs517TxnDkyZ4qZ17uzGCY8O/bi64XIy7iRDvh/CSz++RJHcRfjk1k/4V7V/+R1WSFkiMMak7MgRV/f/xBOu7x9wY/aOHOk6e8tkthzcwrAFw7jrsrsY3nI4BXMV9DukkLNEYIxJ3p49MH68awM4dsyN5NWxo0sIPgzqEkrHTh9j+obpdLmsCzWL1WRT302+jhgWbpYIjDF/iYuD7793d/muXOmm1a8PgwdDy5Z+RhYyc7bMocfMHuw8spPoS6OpVrRaRCUBgIx/Aawx5sJt2OC6di5SBK67zr1u2xZ++gl+/DFTJoGY2Bju/uxuWk9uTe5sufnhnh8yTCdxac1KBMZEsjlz3DCOiXf/VqkCQ4bA3XdD/vz+xhZCiZ3EbTm4hYGNBjKo8aAM1UlcWrNEYEykOXjQdfc8fbq7DBTgiivclUDpZHjHUNl/fD+FcxcmKksUL173ImUKlKHOJXX8Dst3VjVkTKQ4ftx19FapEjz7rLsZ7M47XTXQ0qWZOgmoKhNXTKTyyMq8tewtANpXbW9JwGMlAmMyu4MHXR//L73kxgGoXBlmzcpUN3+lZNvhbfT4ogdzt86lUelGNCvXzO+Q0h1LBMZkZj/8ADfc4O4FKFXKdQXRurXfUYXN+6vep9esXogIo9uO5r7o+zJFJ3FpzT4RYzKjbdvg3/+Gxo0hSxbXHrBjR0QlAYCL81xM4zKNWdd7Hb2u7GVJ4CysRGBMZnL6tLsLuE8f9/qaa1wSKFbM37jC5Ez8GV768SXiNZ6nmjxFywotaVkh8136mtYsPRqTGZw549oBihd3SaBGDVi4EP73v4hJAsv3LOfKt65k0HeD2BSzCVX1O6QMw0oExmR0y5e7u39Pn3aJYNo0aN8eskbGv/eJMyd45vtnGLZgGEUvKsr026Zn6GEj/RDSEoGItBaRTSKyRUQeS2Z+aRH5TkRWiMhqEWkbyniMyVQOHHBXAtWrB9myuRvBdu1yQ0FGSBIA2HpoK6/89Apd63Rlfe/1lgTOQ8h+LSISBYwCWgC7gCUiMkNV1wcsNgj4SFXHiEh1YDZQNlQxGZMpJCRA//6u90+AOnXczWBVq/oaVjj9ceoPPt3wKV3rdKVGsRpsvn9zphoxLNxCWSK4CtiiqltV9TTwIdA+yTIK5POe5wd+C2E8xmRscXHwwQcQFeWSQKNGMG+eqxqKoCQwe/Nsao6uSbcZ3diwfwOAJYELFMpEUALYGfB6lzct0GDgThHZhSsN3J/cikSkh4gsFZGl+/fvD0WsxqRfx4/DI4+46p/OnV07wODBLgk0aZLhB4QP1oHYA3SZ3oXrP7ievDny8uO9P0ZsJ3Fpze+KxNuBd1R1uIjUB94XkZqqmhC4kKqOA8YBREdH26UAJjKouvEAHn0UDh1y9f5PPw2PP+5KBREksZO4rYe28lTjp3ii0RPkyJrxx0ROL0KZCHYDpQJel/SmBeoGtAZQ1Z9EJCdQBNgXwriMSf8+/NANAPPrr67aZ/x4uPHGiEsAvx/7naIXFSUqSxTDWgyjTIEy1L64tt9hZTqhrBpaAlQSkXIikh3oBMxIsswOoDmAiFQDcgJW92Mi1+nTrlvo2293SaB3b1izxg0OH0FJQFUZv3w8VUZWYdyycQDcUOUGSwIhErISgarGiUhfYA4QBUxQ1XUiMgRYqqozgAHAWyLyIK7huKvaXSAmEqnCsGHusW+fq/ufORPy5PE7srDbemgr3b/ozre/fkuTMk24rvx1foeU6YW0jUBVZ+MagQOnPRXwfD3QIJQxGJOuxce7K4Eeewx+8y6ae+stVyqIkEbgQO+ufJfes3sTJVG8ef2bdL+iu/UPFAZ+NxYbE7nWroV77nFjARQpAq+95jqKy53b78h8c2neS7m23LWMuX4MJfOV9DuciGGJwJhwU4VPPnHtAHFx7qqgoUPd5aER5nT8aV743wskaAKDmw6mRYUWtKjQwu+wIo4lAmPCafNm6NULvvkGKlRwPYPWquV3VL5YsnsJ9864l7X71tKldhdUFYnA6rD0wCrfjAmHM2fc8JCVK7sk0K+fGyIyApNA7JlYHv76Ya4efzWHThxiRqcZvHfze5YEfGQlAmNCKSHBDQv5xBOuTaB0aZg8GRo29Dsy3/x66FfeWPwG3et258XrXiR/zvx+hxTxrERgTCgkJLhhIUuVcjeC/f47vPmmuzcgApPAkZNHmLhiIgA1itVgy/1beLPdm5YE0gkrERiTllTdeADPPAPr1rlp//0vPPRQxF4NNOvnWdw38z72HNtD/VL1qVqkKqXyl0r9jSZsrERgTFr55BO49FK49Vb4+Wd44w3XNjBoUEQmgf3H99P50860m9KOgrkK8lO3n6haJHJ6Sc1IrERgzIXavt21AXzwgXvduzcMHw45c/obl4/iE+JpOLEhvx76lWeaPsNjDR8je1R2v8MyZ2GJwJgL8cYb7goggHvvdVcGXXKJvzH5aO+xvRS7qBhRWaIY3nI4ZQuUpWaxmn6HZVIRdNWQiERe2daY5Ki6ht+LL3ZJoHx5+PZb10NohCaBBE1g7NKxVH6jMmOXjgWgXeV2lgQyiFQTgYhcIyLrgY3e68tEZHTIIzMmPdq7190I1quX6xxuwAB3k1izZn5H5pstB7fQ/L3m9JzVkytLXEmriq38Dsmco2BKBK8CrYAYAFVdBTQOZVDGpDuxsW6c4OLF/+oeeudO11tolsi95mLiionUGlOL5XuW89YNb/F/Xf6P8gXL+x2WOUdBtRGo6s4kd/3FhyYcY9KhUaOgb1/3PHdud3lomzb+xpROlM5fmlYVWjGq7ShK5Es6Eq3JKIJJBDtF5BpARSQb0B/YENqwjEkHVq2C1q1ddVCdOm6YyPbtI7J76ESn4k7x/P+eJ0ETGNJsCM3LN6d5+eZ+h2UuUDBl2p5AH9zA87uBOkDvEMZkjL9OnoSHH3YH/4MHoU8fWLwYbropopPAol2LuGLcFTzz/TPsOLIDG0Mq8wimRFBFVTsHThCRBsCPoQnJGJ8cOADdu8Nnn7nX1arB119DycjuF//46eM8+d2TjFg4ghL5SjDz9plcX/l6v8MyaSiYEsEbQU4zJuNatAiKFnVJoGhReO8910VEhCcBgO1HtjN6yWh6RvdkXe91lgQyobOWCESkPnANUFREHgqYlQ83BrExmcPEie5msMTnd98d0VVAAIdPHmba+mn8u+6/qV60Olv6bbERwzKxlKqGsgN5vGXyBkz/A7gllEEZExYHDsC//gU//OBKARMnwvV2tvv5xs/pNasX+47vo2HphlQtUtWSQCZ31kSgqt8D34vIO6q6PYwxGRN6//d/bqjImBjo2hXGjoXskd0Xzr7j++j3ZT+mrptK7YtrM+P2GdZJXIQIprE4VkReBmoAf/aiparXhiwqY0Jp+nRXEihaFJYsgSuu8Dsi38UnxNNgQgN2HNnB0GZDebTBo2SLirwxlCNVMIlgMjAVaIe7lPRuYH8ogzImZCZPhjvvdHcDL1oE5cr5HZGvfjv6G5fkuYSoLFG81vo1yhYoS/Wi1f0Oy4RZMFcNFVbV8cAZVf1eVe8FrDRgMp7nn3dJoHRpN15ABCeBBE1gzJIxVB1ZlTeXvglA20ptLQlEqGBKBGe8v3tE5HrgN6BQ6EIyJo2dOAHdusGUKXDllTB7NhQp4ndUvvk55me6f9Gd+dvnc13562hT0brLiHTBJIKhIpIfGIC7fyAf8EAogzImzWze7PoF+uUXN1zkiy9C1sgdhmP88vH0/bIvObPmZMKNE+hapysS4ZfKmiASgarO9J4eAZrBn3cWG5O+jRwJDz7oxg8YNcr1GBrhyhYoS5uKbRjVdhTF8xb3OxyTTqR0Q1kUcCuuj6GvVHWtiLQDngByAZeHJ0RjzpGquyns/fehaVN46y2oWNHvqHxxKu4U/53/XwCGXjvUOokzyUqpRDAeKAUsBl4Xkd+AaOAxVf0sDLEZc+6OHYO2bd1NYjfdBB9+CDly+B2VLxbsXEC3Gd3YeGAj99a5F1W1aiCTrJQSQTRQW1UTRCQnsBeooKox4QnNmHO0fr07+G/e7LqMfvrpiOwq4tjpYwz8ZiBvLH6DUvlL8VXnr2zUMJOilC4fPa2qCQCqehLYeq5JQERai8gmEdkiIo+dZZlbRWS9iKwTkQ/OZf3GAK6r6EGDoHZtlwSGDYPBgyMyCQDsOLKDscvG0ufKPqzttdaSgElVSiWCqiKy2nsuQAXvtQCqqrVTWrHXxjAKaAHsApaIyAxVXR+wTCXgcaCBqh4SkWIXsC8mEk2Z4i4NPXECmjd3jcJVqvgdVdgdOnGIj9d/TI8relC9aHW29t/KpXkv9Tssk0GklAiqXeC6rwK2qOpWABH5EGgPrA9YpjswSlUPAajqvgvcpokUqvDoo+7sH2DmTNc2EIGlgOkbptN7dm/2H99PkzJNqFKkiiUBc05S6nTuQjuaKwHsDHi9C6iXZJnKACLyI65r68Gq+lXSFYlID6AHQOnSpS8wLJMp3HqrGzu4dm2XBEqV8juisNt7bC/3f3k/09ZPo84ldZh1xyyqFIm80pC5cH7fWZMVqAQ0BUoC80WklqoeDlxIVccB4wCio6NtfLxItmEDPPusSwJlysDy5RAVecNjxCfE02hiI3Ye2clz1z7Hw9c8bJ3EmfMWykSwG3f5aaKS3rRAu4BFqnoG+FVEfsYlhiUhjMtkVDt2QKNGruvo9u1h0qSISwK7/tjFpXkvJSpLFK+3fp1yBctZV9HmggXT6RwikktEzrXMuQSoJCLlRCQ70AmYkWSZz3ClAUSkCK6qaOs5bsdEgvffdyWAw4fdPQKffQZ58vgdVdgkaAJvLHqDqiOrMmbJGADaVGpjScCkiVQTgYjcAKwEvvJe1xGRpAf0f1DVOKAvMAfYAHykqutEZIiI3OgtNgeIEZH1wHfAI3afgvmH++6Du+6CSy+FWbOgYUO/IwqrjQc20nhiY/p91Y+GpRvSrnI7v0MymYyoplzlLiLLcN1Oz1PVy71pa1S1Vhji+4fo6GhdunSpH5s2fvjvf+Gpp6BGDViwAPLl8zuisHp7+dv0nd2X3NlyM6L1CLrU7mJ3B5vzIiLLVDU6uXlBdUOtqkeS/PiswdaE1oEDLgGMGQPNmrmuo3PmTP19mUyFghW4ocoNjGwzkovzXOx3OCaTCiYRrBORO4Ao7wawfsCC0IZlItry5X8NH3nzza6/oAgZT/hk3EmGfD8EgOeaP0ezcs1oVq6Zz1GZzC6YxuL7ceMVnwI+wHVH/UAIYzKRbM6cv5LAk0/Cp59GTBL4cceP1HmzDs//73n2H99PatW2xqSVYEoEVVV1IDAw1MGYCPfuu3DPPVCoEHz0kesyIgIcPXWUJ755glFLRlGmQBnm3DmHlhVa+h2WiSDBJILhInIJMA2YqqprQxyTiTSq0KWLG1i+YkX4/nt3hVCE2PXHLt5e8Tb3X3U/zzZ/ljzZI+eyWJM+pFo1pKrNcCOT7QfGisgaERkU8shMZDh+3DUGT54MV18Ny5ZFRBKIiY35836AakWrsbXfVl5r85olAeOLoG4oU9W9qvo60BN3T8FToQzKRIhffnF9BX3/PTzxRERcHqqqTFs/jeqjq9Pvq35sOrAJwIaNNL4K5oayaiIyWETW4AavX4DrLsKY8/fpp3D55fD776494NlnM33PoXuO7qHDRx3o+HFHSuUrxdLuS62TOJMuBNNGMAGYCrRS1d9CHI+JBN98Ax06QPny8MUXUL263xGFXGIncbuP7ual617iwfoPkjWL330+GuOk+ktU1frhCMREiE8/hTvugIIFIyIJ7DyykxL5ShCVJYpRbUdRrmA5Kheu7HdYxvzNWauGROQj7+8aEVkd8FgTMHKZMcF76SVXEihWDBYvztRJID4hntcXvU7VUX91EteqYitLAiZdSqlE0N/7az1cmQv32GPw4otQv74bSKZQIb8jCpkN+zfQbUY3ftr1E20qtuGGKjf4HZIxKTpriUBV93hPe6vq9sAH0Ds84ZkMTxUef9wlgdKlXftAJk4C45aNo87YOvwc8zPv3/w+s+6YRen8NqqeSd+CuXy0RTLT2qR1ICYTOnkSrr8eXngBiheH1ashVy6/owqpSoUqcXPVm1nfZz131r7Tego1GcJZq4ZEpBfuzL98kjaBvMCPoQ7MZHDHj0ObNm4QmQcfhJdfzpSjiZ04c4LB8wYjIrxw3QvWSZzJkFJqI/gA+BJ4HngsYPpRVT0Y0qhMxrZxI1x3HezeDc8/79oHMqH52+fz7xn/ZvPBzfS8oieqaiUAkyGllAhUVbeJSJ+kM0SkkCUDk6zFi6FlSzhxwg0w36GD3xGluT9O/cFj//cYY5aOoXzB8nxz1zdcW+5av8My5rylViJoByzDDUQTeKqjQPkQxmUyonfege7d4eKLYeFCqJo5x9P97ehvvLPyHR66+iGGNBvCRdkv8jskYy5IqkNVpjc2VGU6dfQolCsHMTGwdat7nokciD3AR+s+oveV7oK534/9biOGmQwlpaEqg+lrqIGIXOQ9v1NEXhERux7O/GXlStdZXEwMfPJJpkoCqsrUtVOpPqo6D3z1AD/H/AxgScBkKsFcPjoGiBWRy4ABwC/A+yGNymQcq1a5AWTy5nVdSf/rX35HlGZ+O/obN029iU6fdKJMgTIs67HM7gw2mVIwvV7FqaqKSHtgpKqOF5FuoQ7MZAA//wzXXguHDsHcuZlqRLH4hHgaT2zM7qO7GdZiGP2v7m+dxJlMK5hf9lEReRzoAjQSkSxAttCGZdK9zz+Hrl3h2DH3PJMkge2Ht1MyX0miskQx+vrRlC9YnoqFKvodljEhFUzV0G24gevvVdW9uLEIXg5pVCb9UoVOneCmm1x10KJFcEPG70snPiGeV356hWqjqjFmqeskrmWFlpYETEQIZqjKvcBkIL+ItANOqup7IY/MpD+xsdC5M0ydCq1bw/r1ULeu31FdsLX71nLNhGsY8PUAmpdvzk1Vb/I7JGPCKpirhm4FFgMdgVuBRSJyS6gDM+nM3r1w2WUwZQrcdRfMng15Mv74um8ufZO6Y+uy9dBWPvjXB8zoNIOS+WwAPhNZgmkjGAhcqar7AESkKPB/wLRQBmbSkSNHoFYtOHAA3nwT7rvP74guWGJ3ENWKVKNjjY6MaDWCohcV9TssY3wRTCLIkpgEPDEEOei9yQT27XNjCx84AMOGZfgkEHsmlqe+e4ooieLFFi/SpGwTmpRt4ndYxvgqmAP6VyIyR0S6ikhXYBYwO7RhGd/FxUHv3q67iN9+gxEjYMAAv6O6IPO2zaP2mNoM/2k4x04fI6PdVW9MqAQzZvEjIvIvoKE3aZyqTg9tWMZXv//uOov78Udo0AD+858MfWXQkZNHeHTuo4xbPo4KBSvw7V3fWlfRxgRIaTyCSsAwoAKwBnhYVXeHKzDjE1V3d/DChfDKK24sgQxuz7E9TFoziYfrP8wzzZ4hd7bcfodkTLqSUtXQBGAm0AHXA+kb57pyEWktIptEZIuInLVTehHpICIqIsl2iGTC5MQJN47AggUwaFCGTgL7j+/njUXuJ1u1SFW29d/Gyy1ftiRgTDJSqhrKq6pvec83icjyc1mxiEQBo3BDXe4ClojIDFVdn2S5vEB/YNG5rN+ksZMnITra3RvQpQs89ZTfEZ0XVWXK2in0+7Iff5z6g1YVW1G5cGW7IsiYFKRUIsgpIpeLSF0RqQvkSvI6NVcBW1R1q6qeBj4E2iez3H+BF4GT5xy9SRtLl7qxA9avhyFD4L33MuSwkjuP7OSGKTfQ+dPOVCxUkRX3rbBO4owJQkolgj3AKwGv9wa8ViC1IZlKADsDXu8C6gUu4CWUUqo6S0QeOduKRKQH0AOgdGnrATtNffutG1tYBMaMgZ49/Y7ovMQlxNH03absPbaXV1u9yv1X3U9UloyXzIzxw1kTgaqG9LIKr/O6V4CuqS2rquOAceAGpgllXBHlww/h9tvdWALLlkHFjNevzrbD2yiVrxRZs2RlbLuxlC9YnvIFbfA8Y85FKG8M2w2UCnhd0puWKC9QE5gnItuAq4EZ1mAcBqp/TwLLl2e4JBCXEMewBcOoNqoao5eMBuC68tdZEjDmPISyg/UlQCURKYdLAJ2AOxJnquoRoEjiaxGZh7tE1cahDLXXXnNXBBUp4kYUq1DB74jOyerfV9NtRjeW/raU9lXa06F6B79DMiZDC1kiUNU4EekLzAGigAmquk5EhgBLVXVGqLZtUjBhgksCWbPCpk1QqJDfEZ2T0UtG0/+r/hTMWZCpt0ylY/WOiIjfYRmToaWaCMT9l3UGyqvqEG+84ktUdXFq71XV2STpjkJVk70uUVWbBhWxOX+zZkG3bpAjhxtiMgMlgcRO4moWq0mnmp14tdWrFMldJPU3GmNSJan1tyIiY4AE4FpVrSYiBYGvVfXKcASYVHR0tC5darVH52zdOjd2QI4cbojJSy7xO6KgHD99nEHfDiJrlqy83NLGQzLmfInIMlVNtg02mMbieqraB+86f1U9BGRPw/hMqH3zDTRr5hqJFy/OMEngm63fUGtMLUYsGsGp+FPWSZwxIRJMG8EZ7y5hhT/HI0gIaVQm7Xz6qetALirKDSZTtarfEaXq8MnDPPz1w4xfMZ5KhSoxv+t8GpVp5HdYxmRawSSC14HpQDEReRa4BRgU0qhM2vjpJze0ZM6crjqoVKnU35MO/H7sdz5c+yH/afAfnm7yNLmy5fI7JGMytWC6oZ4sIsuA5oAAN6nqhpBHZi7M669D//5QrBj873/pPgkkHvz7X92fKkWqsO2BbdYYbEyYBHPVUGkgFvgicJqq7ghlYOY8qbpSwJQpri1gzhyoVMnvqM5KVZm8ZjL9v+rPsdPHaFupLZUKV7IkYEwYBVM1NAvXPiBATqAcsAmoEcK4zPl64gmXBIoXh61bXbVQOrXjyA56zuzJl1u+pH7J+oy/cTyVCqffpGVMZhVM1VCtwNdeR3G9QxaROX9LlsALL7juIjZscDeNpVNxCXE0facp+47v4/XWr9P7yt7WSZwxPjnnI4WqLheReqkvacLq7behe3f3fNKkdJsEth7aSpn8ZciaJStv3fAWFQpVoGyBsn6HZUxEC6aN4KGAl1mAusBvIYvInJuEBOjRA8aPd5eITpoE9dJfno5LiGP4guE8Pe9pXmrxEv3q9aN5+eZ+h2WMIbgSQd6A53G4NoNPQhOOOSeqrhQwYQI0bAgzZ0L+/H5H9Q8r966k24xuLN+znJur3kzH6h39DskYEyDFRODdSJZXVR8OUzwmWJs3uxvF1qyBjh1h6lQ3uEw6M3LxSB6c8yCFcxVmWsdp1lOoMenQWROBiGT1ehBtEM6ATBA2b4bK3hCM/frBq6+muySQ2Elc7Ytr07lWZ15p9QqFcmWcTu6MiSQplQgW49oDVorIDOBj4HjiTFX9NMSxmeT88Qc0buyeT5vmSgXpyLHTxxj4zUCyRWVjWMthNC7TmMZlGvsdljEmBcF0OpcTiMGNUdwOuMH7a8LtzBlo2hT27oUBA9JdEvj6l6+pObombyx+gzPxZ6yTOGMyiJRKBMW8K4bW8tcNZYnsPzzcjh2D1q1hxQp49ll341g6cejEIR76+iHeWfkOVQpXYf4982lYuqHfYRljgpRSIogC8vD3BJDIEkE4bd8O7du7wWSeew4ef9zviP5m3/F9TFs/jccbPs5TTZ4iZ9b0ezezMeafUkoEe1R1SNgiMcnbuBFatIDff4d334W77vI7IgD2HtvLlDVTeLD+g66TuP7bKJy7sN9hGWPOQ0qJIH1dhhKJ/vc/lwROnnTDTLZt63dEqCrvrXqPB+c8SOyZWNpVbkelwpUsCRiTgaXUWGy3ffpp1Spo0+avbqTTQRLYdngbrSe3puvnXaletDore660TuKMyQTOWiJQ1YPhDMQEeOst121Enjzw1VdQrZrfERGXEEezd5txIPYAo9qOomd0T7JIMBedGWPSu/TZM1kk+/xzlwQAvvjC9ySw5eAWyhUoR9YsWZlw4wTKFyxPmQJlfI3JGJO27JQuPZk6FW66CXLnhm3b3D0DPjkTf4bnfniOGqNrMGrJKACalWtmScCYTMhKBOmBKowYAQ895EYVmzsXyvh3wF2+ZzndZnRj5d6VdKzekdtq3OZbLMaY0LNE4DdVuPNO+OADKFwYfvjBDSzjk9cXvc5Dcx6i6EVF+fTWT7m52s2+xWKMCQ+rGvJb//4uCdxzD+zb51sSSOwO4vJLLueuy+5ife/1lgSMiRBWIvDT9OkwciRcf70bWMaHHkSPnjrK4988To6oHAxvNZxGZRrRqEyjsMdhjPGPlQj8MmEC/OtfrgTwwQe+JIGvtnxFzTE1Gb1kNIpaJ3HGRCgrEfhh7Fjo2RMqVYLFiyFfvrBuPiY2hoe+foj3Vr1HtSLV+PHeH6lfqn5YYzDGpB+WCMJtwQKXBEqXhmXLIG/e1N+TxmJOxDB9w3SebPwkAxsNJEfWHGGPwRiTfoS0akhEWovIJhHZIiKPJTP/IRFZLyKrReQbEcncF6nHxUGXLu4+gbffDmsS2HN0D8MWDENVqVy4Mtsf2M6QZkMsCRhjQpcIvPGORwFtgOrA7SJSPcliK4BoVa0NTANeClU8vjtyBJo0ga1b4fnnXWdyYaCqTFgxgWqjqvHkd0+y5eAWAArmKhiW7Rtj0r9QlgiuArao6lZVPQ18CLQPXEBVv1PVWO/lQqBkCOPxz8GDcNVVrlpo+HA3znAY/HroV1pOakm3Gd247JLLWNVzlXUSZ4z5h1C2EZQAdga83gXUS2H5bsCXyc0QkR5AD4DSpUunVXzhsW8flCzphpl85RV48MGwbDYuIY5r37uWmNgYxlw/hh5X9LBO4owxyUoXjcUicicQDTRJbr6qjgPGAURHR2ecaxxV4bLLXBIYOjQsSWBzzGbKFyxP1ixZmdh+IhUKVqBU/lIh364xJuMK5SnibiDwCFTSm/Y3InIdMBC4UVVPhTCe8IqPh6uvdgPNd+0KAweGdHNn4s8wdP5Qao6pycjFIwFoWrapJQFjTKpCWSJYAlQSkXK4BNAJuCNwARG5HBgLtFbVfSGMJbxOnXIDySxeDPnzu/sGQmjpb0vpNqMbq39fTaeanbi91u0h3Z4xJnMJWYlAVeOAvsAcYAPwkaquE5EhInKjt9jLQB7gYxFZKSIzQhVPWHXuDN9+65LB4cOQPXvINvXawteo93Y9DsQe4PNOnzOlwxSKXVQsZNszxmQ+IW0jUNXZwOwk054KeH5dKLfvi++/h08+gWbN3DjDIaKqiAjRl0bT7fJuvNTiJQrkLBCy7RljMq900VicaSxcCK1bQ65c8N57IdnEH6f+4D9z/0POrDl5tfWrNCjdgAalG4RkW8aYyGDXE6aV3buhfn3XPjB3rrtkNI3N3jybGqNrMG75OLJmyWqdxBlj0oSVCNLC8ePQuLF7/u230CBtz9APxB7gga8eYPKaydQoWoNpHadRr2RKt2QYY0zwLBFcKFWoWhV27YLJk0MyzvChE4f44ucveLrJ0zzR6AmyR4Wu8dkYE3ksEVyoBx90SeDKK+GOO1JfPki7/9jN5DWTeeSaR6hUuBLbH9hujcHGmJCwNoILMX06vPYalCsHixalySpVlbeWvUX10dUZPG8wvxz6BcCSgDEmZCwRnK+PP3YjjBUsCEuWpMkIY78c/IXm7zWnx8we1C1el9W9VlOxkH8D2RtjIoNVDZ2Pt9+G7t2hRAmYMQMKF77gVcYlxNH8veYcPHGQse3G8u+6/7ZO4owxYWGJ4FzFx8N//gPFi8P69Rc8zOSmA5uoUKgCWbNk5d2b3qVCoQqUzJc5e+M2xqRPdsp5rnr0cOMLDB58QUngdPxpnpn3DLXG1GLU4lEANCnbxJKAMSbsrERwLjZuhIkToVYtVzV0nhbvXky3Gd1Yu28td9S6g861O6dhkMYYc24sEQTr4EF3iagITJt23o3DIxaOYMDXAyiepzhf3P4F7Sq3S+NAjTHm3FgiCMaZM9ChAxw75i4XrVz5nFeR2EncVSWuonvd7rx43Yvkz5k/BMEaY8y5sUSQGlW49VaYNw+efvqcxxs+cvIIj859lFzZcjGi9QiuKXUN15S6JjSxGmPMebDG4tRMmQKffeZGGRs8+Jze+sWmL6g+ujpvr3ibHFE5rJM4Y0y6ZCWClOzYAX37um6l33wz6LftP76f/l/1Z8raKdQqVovPbvuMK0tcGcJAjTHm/FkiOJuDB6FhQ9ez6EcfQY4cQb/1yKkjzN48m2eaPsNjDR+zTuKMMemaJYLkxMVBp06wc6erFmrfPtW37Dyyk0mrJ/FYw8eoWKgi2x/Ybo3BxpgMwdoIktOtmxtcZujQVJNAgibw5tI3qTG6BkN/GPpnJ3GWBIwxGYUlgqS2bnVVQQ0awMCBKS66OWYz1757Lb1m9eKqElexptca6yTOGJPhWNVQoCNHoEULN9xkKo3DcQlxtHi/BYdPHmb8jeO5p849SBr0QGqMMeFmiSDRwYOuFLB1K0yYADVrJrvYhv0bqFS4ElmzZOX9m9+nQqEKXJr30jAHa4wxaceqhhLdcovrS+iVV+Cee/4x+1TcKZ7+7mlqv1mbkYtHAtCoTCNLAsaYDM9KBOCuDPruO6hb1w09mcTCXQvpNqMb6/evp0vtLnSp3SX8MRpjTIhYIvjkE7j9dihZEr766h+zhy8YziNzH6FkvpLMvmM2bSq18SFIY4wJnchOBHv3uiqhSy+FBQugaNE/ZyVoAlkkC/VL1adndE9euO4F8uW4sEFojDEmPYrsRJDYgdykSVCqFACHTx5mwJwB5M6WmzfavmGdxBljMr3IbSyeMsUNQN+iBTRrBsBnGz+j+qjqvLvqXfLmyGudxBljIkJklgiWL4c77oBq1eCjj9h3fB99Z/fl4/UfU+eSOsy8YyZ1i9f1O0pjjAmLyCwR9O3r/n76KRQowB+n/mDu1rk8e+2zLP73YksCxpiIEnklgkWL4Kef2HFlZd7f9wlPVHmCioUqsuOBHeTNkdfv6IwxJuxCWiIQkdYisklEtojIY8nMzyEiU735i0SkbCjj4eRJEhpcw+grocaNu3juf8/92UmcJQFjTKQKWSIQkShgFNAGqA7cLiLVkyzWDTikqhWBV4EXQxUPwKYbrqHpXQn0uR7ql2nAut7rrJM4Y0zEC2WJ4Cpgi6puVdXTwIdA0j6d2wPves+nAc0lRD23xQ17iVY1V7CmZDYmtp/InDvnULZA2VBsyhhjMpRQthGUAHYGvN4F1DvbMqoaJyJHgMLAgcCFRKQH0AOgdOnS5xVM1pq1mbSjPRX6j6B44bLntQ5jjMmMMkRjsaqOA8YBREdHn9/F/a1b07B167QMyxhjMoVQVg3tBkoFvC7pTUt2GRHJCuQHYkIYkzHGmCRCmQiWAJVEpJyIZAc6ATOSLDMDuNt7fgvwrdrtvMYYE1Yhqxry6vz7AnOAKGCCqq4TkSHAUlWdAYwH3heRLcBBXLIwxhgTRiFtI1DV2cDsJNOeCnh+EugYyhiMMcakLDK7mDDGGPMnSwTGGBPhLBEYY0yEs0RgjDERTjLa1Zoish/Yfp5vL0KSu5YjgO1zZLB9jgwXss9lVLVocjMyXCK4ECKyVFWj/Y4jnGyfI4Ptc2QI1T5b1ZAxxkQ4SwTGGBPhIi0RjPM7AB/YPkcG2+fIEJJ9jqg2AmOMMf8UaSUCY4wxSVgiMMaYCJcpE4GItBaRTSKyRUQeS2Z+DhGZ6s1fJCJlfQgzTQWxzw+JyHoRWS0i34hIGT/iTEup7XPAch1EREUkw19qGMw+i8it3ne9TkQ+CHeMaS2I33ZpEflORFZ4v++2fsSZVkRkgojsE5G1Z5kvIvK693msFpG6F7xRVc1UD1yX178A5YHswCqgepJlegNves87AVP9jjsM+9wMyO097xUJ++wtlxeYDywEov2OOwzfcyVgBVDQe13M77jDsM/jgF7e8+rANr/jvsB9bgzUBdaeZX5b4EtAgKuBRRe6zcxYIrgK2KKqW1X1NPAh0D7JMu2Bd73n04DmIiJhjDGtpbrPqvqdqsZ6LxfiRozLyIL5ngH+C7wInAxncCESzD53B0ap6iEAVd0X5hjTWjD7rEA+73l+4LcwxpfmVHU+bnyWs2kPvKfOQqCAiBS/kG1mxkRQAtgZ8HqXNy3ZZVQ1DjgCFA5LdKERzD4H6oY7o8jIUt1nr8hcSlVnhTOwEArme64MVBaRH0VkoYhk9IG6g9nnwcCdIrILN/7J/eEJzTfn+v+eqgwxeL1JOyJyJxANNPE7llASkSzAK0BXn0MJt6y46qGmuFLffBGppaqH/QwqxG4H3lHV4SJSHzfqYU1VTfA7sIwiM5YIdgOlAl6X9KYlu4yIZMUVJ2PCEl1oBLPPiMh1wEDgRlU9FabYQiW1fc4L1ATmicg2XF3qjAzeYBzM97wLmKGqZ1T1V+BnXGLIqILZ527ARwCq+hOQE9c5W2YV1P/7uciMiWAJUElEyolIdlxj8Iwky8wA7vae3wJ8q14rTAaV6j6LyOXAWFwSyOj1xpDKPqvqEVUtoqplVbUsrl3kRlVd6k+4aSKY3/ZnuNIAIlIEV1W0NYwxprVg9nkH0BxARKrhEsH+sEYZXjOAu7yrh64GjqjqngtZYaarGlLVOBHpC8zBXXEwQVXXicgQYKmqzgDG44qPW3CNMp38i/jCBbnPLwN5gI+9dvEdqnqjb0FfoCD3OVMJcp/nAC1FZD0QDzyiqhm2tBvkPg8A3hKRB3ENx10z8omdiEzBJfMiXrvH00A2AFV9E9cO0hbYAsQC91zwNjPw52WMMSYNZMaqIWOMMefAEoExxkQ4SwTGGBPhLBEYY0yEs0RgjDERzhKBSZdEJF5EVgY8yqaw7LE02N47IvKrt63l3h2q57qOt0Wkuvf8iSTzFlxojN56Ej+XtSLyhYgUSGX5Ohm9N04Tenb5qEmXROSYquZJ62VTWMc7wExVnSYiLYFhqlr7AtZ3wTGltl4ReRf4WVWfTWH5rrheV/umdSwm87ASgckQRCSPN47CchFZIyL/6GlURIqLyPyAM+ZG3vSWIvKT996PRSS1A/R8oKL33oe8da0VkQe8aReJyCwRWeVNv82bPk9EokXkBSCXF8dkb94x7++HInJ9QMzviMgtIhIlIi+LyBKvj/n7gvhYfsLrbExErvL2cYWILBCRKt6duEOA27xYbvNinyAii71lk+ux1UQav/vetoc9knvg7opd6T2m4+6Cz+fNK4K7qzKxRHvM+zsAGOg9j8L1N1QEd2C/yJv+H+CpZLb3DnCL97wjsAi4AlgDXIS7K3sdcDnQAXgr4L35vb/z8MY8SIwpYJnEGG8G3vWeZ8f1IpkL6AEM8qbnAJYC5ZKJ81jA/n0MtPZe5wOyes+vAz7xnncFRga8/zngTu95AVxfRBf5/X3bw99HputiwmQaJ1S1TuILEckGPCcijYEE3JnwxcDegPcsASZ4y36mqitFpAlusJIfva41suPOpJPzsogMwvVT0w3Xf810VT3uxfAp0Aj4ChguIi/iqpN+OIf9+hJ4TURyAK2B+ap6wquOqi0it3jL5cd1FvdrkvfnEpGV3v5vAOYGLP+uiFTCdbOQ7SzbbwncKCIPe69zAqW9dZkIZYnAZBSdgaLAFap6RlyPojkDF1DV+V6iuB54R0ReAQ4Bc1X19iC28YiqTkt8ISLNk1tIVX8WN9ZBW2CoiHyjqkOC2QlVPSki84BWwG24gVbAjTZ1v6rOSWUVJ1S1jojkxvW/0wd4HTcAz3eqerPXsD7vLO8XoIOqbgomXhMZrI3AZBT5gX1eEmgG/GPMZXHjMP+uqm8Bb+OG+1sINBCRxDr/i0SkcpDb/AG4SURyi8hFuGqdH0TkUiBWVSfhOvNLbszYM17JJDlTcR2FJZYuwB3UeyW+R0Qqe9tMlrrR5voBA+SvrtQTuyLuGrDoUVwVWaI5wP3iFY/E9UprIpwlApNRTAaiRWQNcBewMZllmgKrRGQF7mz7NVXdjzswThGR1bhqoarBbFBVl+PaDhbj2gzeVtUVQC1gsVdF8zQwNJm3jwNWJzYWJ/E1bmCg/1M3/CK4xLUeWC5u0PKxpFJi92JZjRuY5SXgeW/fA9/3HVA9sbEYV3LI5sW2znttIpxdPmqMMRHOSgTGGBPhLBEYY0yEs0RgjDERzhKBMcZEOEsExhgT4SwRGGNMhLNEYIwxEe7/AUX2OdSH5T+zAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis AUROC score: 0.65204821\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, auroc_score = auroc(test_y, ood_scores)\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "print('Mahalanobis AUROC score:', auroc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAjOUlEQVR4nO3deZgV5Z328e8tIIssRsCorI4QEUHRtBsm4xJXgpjNRKLjmHhpdCTOJL4aM84YX3UyxlzJ5HXEuIxm0aiJMVFMMCRuMRg1NoLKEpWgQpM4IoqILLL83j+eOvbp5kCfbrq66a77c13n6nOq6tT5Fcu5+3meqnoUEZiZWXHt0N4FmJlZ+3IQmJkVnIPAzKzgHARmZgXnIDAzKzgHgZlZwTkIrMOQdJqk31ax3Q2S/r0tamoLkl6RdEz2/HJJt7d3Tda5OAisVWRfVmskrZL0v5J+KKl3a35GRPwkIo6rYrtzI+LK1vzsEkkh6d3sOJdK+q6kLnl8VktI6ivpe5IWZzX+JXs9oL1rs+2Xg8Ba00kR0Rs4EKgB/q3xBpK6tnlVrW//7DiPAD4HfLGd6wFA0o7AQ8C+wAlAX+AwYDlwcAv21xn+rqwKDgJrdRGxFHgAGAPv/xZ9vqSXgJeyZRMlzZG0QtIfJe1Xer+kIZJ+IWmZpOWSrsuWnylpZvZckv5L0uuSVkp6XlLp834o6aqy/Z0taaGkNyVNk7RH2bqQdK6kl7JapkpSlce5EHgcGFe2v5Yc116SHs6WvSHpJ5J2buYfO8AZwFDgkxExPyI2RcTrEXFlREwvO94RZTW9/2cl6UhJdZK+Juk14AeSFkiaWLZ916z+A7PXh2bHuULSs5KObEHd1s4cBNbqJA0BJgCzyxZ/AjgEGC3pAOBW4EtAf+BGYJqk7lk3y6+AV4HhwCDgrgofcxzw98CHgH7AZ0m/+Tau5WjgP7P1u2f7bby/icBBwH7ZdsdXeZyjgI8CC7PXLT0uZTXuAewDDAEur6aGRo4BfhMRq1rw3pLdgF2AYcA5wJ3A5LL1xwNvRMQzkgYBvwauyt7zf4B7JA3chs+3duAgsNZ0r6QVwEzg98A3y9b9Z0S8GRFrSF8wN0bEUxGxMSJ+BKwDDiV1YewBXBQR70bE2oiYWeGz1gN9gFGAImJBRPytwnanAbdGxDMRsQ74OnCYpOFl21wdESsiYjHwCGW/4W/BM5LeBRYAjwLXZ8tbdFwRsTAifhcR6yJiGfBdUrdTc/UHKv0ZNMcm4BtZLWuAO4BJknpl6z9PCgeA04HpETE9a338Dqgl/RJgHYiDwFrTJyJi54gYFhH/lH2RlCwpez4MuDDrTliRhccQ0hflEODViNiwtQ+KiIeB64CpwOuSbpLUt8Kme5B+Cy+9bxWp5TCobJvXyp6vBnoDSJqXDbiukvTRsm0OzLb5HKmVs9O2HJekD0q6Kxt8XgncDrRkcHc5qdWzLZZFxNrSi6z7awFwUhYGk0jhAOl4T2l0vB9phRqsjTkIrK2U3+Z2CfAfWWiUHr0i4s5s3dBqBioj4tqI+DAwmtRFdFGFzf5K+sICQNJOpN+cl1ax/30jonf2+EOjdRERPwOeAC7bxuP6JunPZ2xE9CX9pl3VOEUjDwLHZ8e4JauBXmWvd2u0vtLtiEvdQycD87NwgHRMtzU63p0i4uoW1G7tyEFg7eFm4FxJh2SDvjtJ+rikPsCfSN0bV2fLe0g6vPEOJB2Uvb8b8C6wltSt0didwBckjZPUnfSl+1REvNJKx3I1cLak3bbhuPoAq4C3s373SoFWjdtIX873SBolaQdJ/SX9q6RSd80c4POSukg6geq6oO4ijcmcR31rAFLL5SRJx2f765ENOA9uYf3WThwE1uYiohY4m9S18xZpsPXMbN1G4CRgBLAYqCN1wTTWl/TF+xap62c58O0Kn/Ug8O/APaQv4r2AU1vxWJ4HHiP1/bf0uP4vqbvpbdLg6y9aWMs60oDxn4HfAStJATQAeCrb7J+zOlaQxk/urWK/fyO1fMYDPy1bvoTUSvhXYBkphC7C3ysdjjwxjZlZsTm5zcwKzkFgZlZwDgIzs4JzEJiZFVyHu6nUgAEDYvjw4e1dhplZhzJr1qw3IqLi7T86XBAMHz6c2tra9i7DzKxDkfTqlta5a8jMrOAcBGZmBecgMDMruA43RmBm1lLr16+nrq6OtWvXNr1xB9WjRw8GDx5Mt27dqn6Pg8DMCqOuro4+ffowfPhwqpyIrkOJCJYvX05dXR177rln1e/LrWtI0q1K0wjO3cJ6SbpWaQrB50pT35mZ5WXt2rX079+/U4YAgCT69+/f7BZPnmMEPyRNoL0lJwIjs8c5wPdzrMXMDKDThkBJS44vtyCIiMeAN7eyycnAj7MJPp4EdpaU38xGM2fCZZfBe+/l9hFmZh1Re541NIiG0xfW0XD6wPdJOkdSraTaZcuWtezTnngCrrwS1q9v2fvNzFpBly5dGDduHGPGjOGkk05ixYoV76+bN28eRx99NHvvvTcjR47kyiuvpHyqgAceeICamhpGjx7NAQccwIUXXtgqNXWI00cj4qaIqImImoEDK14hbWbWIfTs2ZM5c+Ywd+5cdtllF6ZOnQrAmjVrmDRpEpdccgkvvPACzz77LH/84x+5/vrrAZg7dy5Tpkzh9ttvZ/78+dTW1jJixIhWqak9g2ApaULvksFUMY+smVlncdhhh7F0afrau+OOOzj88MM57rjjAOjVqxfXXXcdV1+dpoC+5ppruPTSSxk1ahSQWhbnnXdeq9TRnqePTgOmSLoLOAR4O5sSz8wsf//yLzBnTuvuc9w4+N73qtp048aNPPTQQ5x11llA6hb68Ic/3GCbvfbai1WrVrFy5Urmzp3bal1BjeUWBJLuBI4EBkiqA74BdAOIiBuA6cAE0ryuq4Ev5FWLmdn2Ys2aNYwbN46lS5eyzz77cOyxx7Z3SfkFQURMbmJ9AOfn9flmZltV5W/ura00RrB69WqOP/54pk6dygUXXMDo0aN57LHHGmy7aNEievfuTd++fdl3332ZNWsW+++/f6vX1CEGi83MOptevXpx7bXX8p3vfIcNGzZw2mmnMXPmTB588EEgtRwuuOACLr74YgAuuugivvnNb/Liiy8CsGnTJm644YZWqcVBYGbWTg444AD2228/7rzzTnr27Ml9993HVVddxd57783YsWM56KCDmDJlCgD77bcf3/ve95g8eTL77LMPY8aMYdGiRa1Sh+81ZGbWhlatWtXg9f333//+87Fjx/Loo49u8b0TJ05k4sSJrV6TWwRmZgXnIDAzKzgHgZkVSvktGzqjlhyfg8DMCqNHjx4sX76804ZBaT6CHj16NOt9Hiw2s8IYPHgwdXV1tPjmlR1AaYay5nAQmFlhdOvWrVkzdxWFu4bMzArOQWBmVnAOAjOzgnMQmJkVnIPAzKzgHARmZgXnIDAzKzgHgZlZwTkIzMwKzkFgZlZwDgIzs4JzEJiZFZyDwMys4BwEZmYF5yAwMys4B4GZWcE5CMzMCs5BYGZWcA4CM7OCcxCYmRWcg8DMrOAcBGZmBZdrEEg6QdILkhZKuqTC+qGSHpE0W9JzkibkWY+ZmW0utyCQ1AWYCpwIjAYmSxrdaLN/A34WEQcApwLX51WPmZlVlmeL4GBgYUQsioj3gLuAkxttE0Df7Hk/4K851mNmZhXkGQSDgCVlr+uyZeUuB06XVAdMB75caUeSzpFUK6l22bJledRqZlZY7T1YPBn4YUQMBiYAt0narKaIuCkiaiKiZuDAgW1epJlZZ5ZnECwFhpS9HpwtK3cW8DOAiHgC6AEMyLEmMzNrJM8geBoYKWlPSTuSBoOnNdpmMfAxAEn7kILAfT9mZm0otyCIiA3AFGAGsIB0dtA8SVdImpRtdiFwtqRngTuBMyMi8qrJzMw21zXPnUfEdNIgcPmyy8qezwcOz7MGMzPbuvYeLDYzs3bmIDAzKzgHgZlZwRUnCB55JP3s3bt96zAz284UJwgeeKC9KzAz2y4VJwjMzKwiB4GZWcE5CMzMCs5BYGZWcA4CM7OCcxCUrFsHq1e3dxVmZm2umEEgbb6sRw/YaSf4zGfavh4zs3ZUzCBo7Je/rH9+zz31zxctgvHjYd68tq/JzKyNFDcI5s9PP195BT71qYbrrr8+tRr22gueeALGjGnz8szM2kpxg+A3v0k/jz1283Xnn7/5sk2b8q3HzKydFDcIHn4Yfv5zWLgwve7dOy3bki5dUiuh9Fi0qG3qNDPLWXGD4Ne/hlNOqX99991w1FH1r999FyK2fI+ivfaCe++FjRthxIj6gFizJj1awq0OM2sHxQ2CcnvvDSeckJ6vXQvr10OvXun1CSeklkPJJz5R//yTn4SuXeEvf6lf1qtXekiphVHeiviP/4A33oDXXkvb/vnPKVBOPz2tL7U61q3L9XDNzMqpo00RXFNTE7W1tc1/Y6VTRktmzoTDmzFj5pNPwmGHNVx2++3pCz0Pgwal1sLGjXDlldC3L5x6av36iK0fn5kVnqRZEVFTaZ1bBJBaBM1x6KHpy/fll+Gaa1Ir4rTT0rIImDu3ftuhQ9Nv+FubB2HMGHjzTXj77crrly6Fv/0NXn8dvvQlmDy5YUtjhx3Szw98IJ0FNWMGLFiQgsPMrAm5Tl7fITz0EAwY0LL3Dh8OF120+fJ9960PhdJv6u+8k35GwKpV6QK2bt02f2/pfQsWwP33wzHHwAUXpHGIH/946/WsWAF77ll53e67w5IlKaSefx5efTUt++IXqz1aM+ukit019Pjj6YKxzmDxYpgwofUufrvvPpg0qXX2ZWbtzl1DW9JZQgBSF9TcufUtitJj1Sr41rfSNqNGpcFpgCFDtr6/k09u2P0kwa23pm4udzmZdSrFDoIi2GknuPji+u6mhQvT88WLNw+NiDQo/f3vV97XWWelLq2uXRsGxOGHe0zCrAMrXhB89KPp59e+1r51bK8kOPfc+mDYuDENYt92GwweDHvssfl7/vhHGD1684Aof3zlK/Df/93wVFsz2y4Ub4zgqKO2fgWxVS8inTF11VVpIHvjxnR2UzX694fly1Ow/PWv0L176nb69KfT6bIf+hDsthscfTT065fOjCpdcLdD8X5/MdtWWxsjKF4QfOxj8OCDrVuUbdmmTTBtGjz2GPzv/8Idd+TzOV261F/lffrpcOCB6ayuUaNSi2b5chg50iFiheUggPogOPZY+O1vW7coa56I1ArYbbf0BV6ybBn85CfpbK6ePeG999JV3aNHw7Bh6UK6JUvgD39I2++4Y9qmucaPh0MOSS2QceNSl9Y++6TP2GWX1Dox62QcBFAfBMcfX3/nUet85s6FWbNS2L/xRhrcfucdeOsteO656u/nNHhw+jfz2mvpeouxY2H27PTvZ8aMtPyoo1I4DR0KNTXpDK1Ro1J3V+/eKah69Ei3HHFLxNqZgwDqg2DChHTDOSumTZvg2WfTWVMAH/xgumXInDmpdbJgQbqSe82a1EKB+m6nkh490thIS+y0U7qhIaRWye67p26rQYNS+PTvn64QHz06hc/OO6dtfRsR20ZbC4JcryyWdALw/4AuwP9ExNUVtvkscDkQwLMR8fk8a/JvZgW3ww5wwAHpUXLooU2/LyKFQ8+eDb+Q33wTnn46dS+99loaC+ndO92/auHCNNhdur5j6NAUKKWTFRYuhKeeqq7u8vDZY48UHCtXputBBgyA3/0ujX+tWwfXXps+y6xKubUIJHUBXgSOBeqAp4HJETG/bJuRwM+AoyPiLUm7RsTrW9vvNrcIJk1KV82abS82bEitkNdeS/NcdO8OL76Yupr+8AfYb7/0pT99emox7LIL/OlPzbtuo0+f+tucfPzjqfVx8MFpv336pH1u2pS6s/r0Sfff2m03t0I6kfZqERwMLIyIRVkRdwEnA/PLtjkbmBoRbwE0FQKtwi0C29507ZoGqocNS91F1dqwIbU0unVLofH662lsZO7cNFfG0qWp5TN7dn0IQH3X6I03Vv9Zw4alLrKzzkpdVx//eGqNOCg6hTyDYBCwpOx1HdD4X/mHACQ9Tuo+ujwiNhvJlXQOcA7A0G1t8vofrnUWXcv++/bunR7nnpteX3fdlt+3cmU6pXblynT21ptv1ndtDRuWli9ZAvfcAy+8kLrDXn01vffSSxvuqzTmMWQIHHdcujVJTQ0MHNiwPtuuVfU3JelwUj/+sOw9AiIi/q4VPn8kcCQwGHhM0tiIWFG+UUTcBNwEqWtomz7RQWBF17dvekC6U+6WNL76/vXX06D6vfem25136QK/+lVat2QJ3HJLepTr3h0+9al05tSZZ6buqB13bJ3jsFZTbWTfAnwFmAVU2zG5FCi/s9ngbFm5OuCpiFgPvCzpRVIwPF3lZzRfBztLymy7seuu6bf+445ruLx0j6qZM1MrokuXNPve7NnpIsI770zblUKie/c02L3HHumsqfHj03hHv37p9NuxY9NzazPVBsHbEbGFyXu36GlgpKQ9SQFwKtD4jKB7gcnADyQNIHUV5Tsr/IgRue7erHBK06wecUR6NLZ+PTzzTLoVSZ8+UFeXruuYOTOtv/vuzd8zdGgaGJ89OwXDwQenM6cOOihNH/vBD+Z7TAVTbRA8IunbwC+A9yfUjYhntvSGiNggaQowg9T/f2tEzJN0BVAbEdOydcdJmk9qaVwUEctbeCxbN3o0zJ8P//APuezezLagW7c0CH7//Zuve++9dH+qNWvSeMWcOekaj7/8JXU7Sek02+efb/i+XXdNg+UjRqRrLcaPT+8fNiwNkB94YNrGXcFVqer0UUmPVFgcEXF065e0dS0+fXTffVMQPP98mhrSzDqO9evTqbX33ZeuGF+0KLUqqr2BZP/+6XYiS5em27JPmJCuvyi/xUknt82nj0bEUa1bUjvybwhmHU+3bunahosv3nzdpk2pZfHGG+mW6D17pussnngiTd86a1b60n/oobR9aXrW0h1tJ02CiRPTtLBbmuq1k6v2rKF+wDeAv88W/R64IiK2MNu6mVkb2WGHNH4weDB89rNp2UknVd52xYo0096CBfA//5OWTZuWHiW77566nD760TRoXVOTHqUzrTqhascIbgXmAtmfMv8A/AD4VB5FmZnlYued4atfTc9vvjn9XLw4dTEtXpxaFDNmpHGL0l1uGxs/Hj7ykTQGccYZ6ZqJDq7aMYI5ETGuqWVtYZvHCObO3fq502Zm69eni+jefTd1Mb32Whrsnj1781PQpXQm05FHwuc/n1om26HWuMXEGkkfiYiZ2Q4PB9a0VoFtwtcPmFm1unWrP9V8//3Tz8svr1+/cmU6q+mXv4SXX4YHHkiP0kV4RxwBJ54In/kM7LVXm5beEtXeeOc8YKqkVyS9ClwHnJtfWTnyYLGZbau+fdNv/3ffDbW16a6v11+fTl/93Ofg97+HSy5JYSKlU1qvuiqdtbQdatbdRyX1BYiIlblV1IQWdw2NHp0GiObNS8/NzPLyzjvpbrF//nMaiH6m7JKrXXZJd3Y94wz48pfT7TfaQIsnppF0ekTcLumrldZHxHdbqcaqOQjMrMNZty4Fwvz58Oij6VFSUwMXXphaEjn2WGwtCJrqGtop+9lnCw8zM2tK9+5wyinwjW/AI4/A6tXw05+m7qXaWpg8OZ0G268ffP3r9bPYtZHiTFW5zz6pmeYWgZltT95+G267LY03PP54/YRDEybA97/farPNbUuLoLSDayT1ldRN0kOSlkk6vVWqa2seLDaz7Um/fjBlShpgXr8+naZ67LFpjGHYMDj//NxLqPasoeOyAeKJwCvACOCivIoyMyskKd3u4re/TRe5jR2bzkY64ohcT4GvNghK1xt8HLjbt5YwM8vZUUeli9kmToTHHks3zstJtUHwK0l/Bj4MPCRpILA2t6rMzCxNBTptWrow7a234MEHc/mYqoIgIi4BxgM12Wxi75Imou84OtiguJkZkLqLbr4Zhg9PA8s52OotJiQdHREPS/pU2bLyTX6RS1V58mCxmXU0O+8ML70EXau9K1DzNLXXI4CHgUr3dA06YhCYmXVEOYUANBEEEfGN7OcXcqvAzMzaVbXXEXxT0s5lrz8g6arcqjIzszZT7VlDJ0bEitKLiHgLmJBLRWZm1qaqDYIukrqXXkjqCXTfyvbbH581ZGZWUbWjDz8hXT/wg+z1F4Af5VNSznzWkJlZA1UFQUR8S9KzwDHZoisjYkZ+ZZmZWVtpzvlIC4ANEfGgpF6S+kTEO3kVZmZmbaPas4bOBn4O3JgtGgTcm1NNZmbWhqodLD4fOBxYCRARLwG75lVULjxYbGZWUbVBsC4i3iu9kNSVdGVxx+PBYjOzBqoNgt9L+legp6RjgbuB+/Mry8zM2kq1QfA1YBnwPPAlYDrwb3kVZWZmbafJs4YkdQHmRcQo4Ob8SzIzs7bUZIsgIjYCL0hqnRmU24sHi83MKqq2a+gDwLxs4vpppUdTb5J0gqQXJC2UdMlWtvu0pJBUU23hLebBYjOzBqq9oOzfm7vjrEtpKnAsUAc8LWlaRMxvtF0f4J+Bp5r7GWZmtu2amqGsB3AuMII0UHxLRGyoct8HAwsjYlG2r7tI01vOb7TdlcC3gIuaUbeZmbWSprqGfgTUkELgROA7zdj3IGBJ2eu6bNn7JB0IDImIX29tR5LOkVQrqXbZsmXNKMHMzJrSVNfQ6IgYCyDpFuBPrfXBknYAvguc2dS2EXETcBNATU1Ny0Z9PVhsZlZRUy2C9aUnzegSKlkKDCl7PThbVtIHGAM8KukV4FBgWu4Dxh4sNjNroKkWwf6SVmbPRbqyeGX2PCKi71be+zQwUtKepAA4Ffh8aWVEvA0MKL2W9CjwfyKittlHYWZmLdbU5PVdWrrjiNggaQowA+gC3BoR8yRdAdRGRJOnn5qZWf6aMx9Bs0XEdNLtKMqXXbaFbY/MsxYzM6us2gvKOj4PFpuZVVScICjxYLGZWQPFCwIzM2vAQWBmVnAOAjOzgitOEHiw2MysouIEQYkHi83MGiheEJiZWQMOAjOzgnMQmJkVnIPAzKzgihMEPmvIzKyi4gRBic8aMjNroHhBYGZmDTgIzMwKzkFgZlZwxQkCDxabmVVUnCAo8WCxmVkDxQsCMzNrwEFgZlZwDgIzs4IrThB4sNjMrKLiBEGJB4vNzBooXhCYmVkDDgIzs4JzEJiZFVxxgsCDxWZmFRUnCEo8WGxm1kDxgsDMzBpwEJiZFVyuQSDpBEkvSFoo6ZIK678qab6k5yQ9JGlYnvWYmdnmcgsCSV2AqcCJwGhgsqTRjTabDdRExH7Az4Fr8qrHg8VmZpXl2SI4GFgYEYsi4j3gLuDk8g0i4pGIWJ29fBIYnGM9iQeLzcwayDMIBgFLyl7XZcu25CzggUorJJ0jqVZS7bJly1qxRDMz2y4GiyWdDtQA3660PiJuioiaiKgZOHBg2xZnZtbJdc1x30uBIWWvB2fLGpB0DHApcERErMuxHjMzqyDPFsHTwEhJe0raETgVmFa+gaQDgBuBSRHxeo61eLDYzGwLcguCiNgATAFmAAuAn0XEPElXSJqUbfZtoDdwt6Q5kqZtYXetx4PFZmYN5Nk1RERMB6Y3WnZZ2fNj8vx8MzNr2nYxWGxmZu3HQWBmVnDFCQIPFpuZVVScICjxYLGZWQPFCwIzM2vAQWBmVnAOAjOzgnMQmJkVXHGCwGcNmZlVVJwgKPFZQ2ZmDRQvCMzMrAEHgZlZwTkIzMwKrjhB4MFiM7OKihMEJR4sNjNroHhBYGZmDTgIzMwKzkFgZlZwxQkCDxabmVVUnCAo8WCxmVkDxQsCMzNrwEFgZlZwDgIzs4IrThB4sNjMrKLiBEGJB4vNzBooXhCYmVkDDgIzs4JzEJiZFVxxgsCDxWZmFRUnCEo8WGxm1kDxgsDMzBrINQgknSDpBUkLJV1SYX13ST/N1j8laXie9ZiZ2eZyCwJJXYCpwInAaGCypNGNNjsLeCsiRgD/BXwrr3rMzKyyPFsEBwMLI2JRRLwH3AWc3Gibk4EfZc9/DnxMyqkTv1ev9NNjBGZmDXTNcd+DgCVlr+uAQ7a0TURskPQ20B94o3wjSecA5wAMHTq0ZdU8+CDcfTcMHNiy95uZdVIdYrA4Im6KiJqIqBnY0i/yESPg619v3cLMzDqBPINgKTCk7PXgbFnFbSR1BfoBy3OsyczMGskzCJ4GRkraU9KOwKnAtEbbTAP+MXv+GeDhCF/5ZWbWlnIbI8j6/KcAM4AuwK0RMU/SFUBtREwDbgFuk7QQeJMUFmZm1obyHCwmIqYD0xstu6zs+VrglDxrMDOzresQg8VmZpYfB4GZWcE5CMzMCs5BYGZWcOpoZ2tKWga82sK3D6DRVcsF4GMuBh9zMWzLMQ+LiIpX5Ha4INgWkmojoqa962hLPuZi8DEXQ17H7K4hM7OCcxCYmRVc0YLgpvYuoB34mIvBx1wMuRxzocYIzMxsc0VrEZiZWSMOAjOzguuUQSDpBEkvSFoo6ZIK67tL+mm2/ilJw9uhzFZVxTF/VdJ8Sc9JekjSsPaoszU1dcxl231aUkjq8KcaVnPMkj6b/V3Pk3RHW9fY2qr4tz1U0iOSZmf/vie0R52tRdKtkl6XNHcL6yXp2uzP4zlJB27zh0ZEp3qQbnn9F+DvgB2BZ4HRjbb5J+CG7PmpwE/bu+42OOajgF7Z8/OKcMzZdn2Ax4AngZr2rrsN/p5HArOBD2Svd23vutvgmG8CzsuejwZeae+6t/GY/x44EJi7hfUTgAcAAYcCT23rZ3bGFsHBwMKIWBQR7wF3ASc32uZk4EfZ858DH5M69Kz2TR5zRDwSEauzl0+SZozryKr5ewa4EvgWsLYti8tJNcd8NjA1It4CiIjX27jG1lbNMQfQN3veD/hrG9bX6iLiMdL8LFtyMvDjSJ4Edpa0+7Z8ZmcMgkHAkrLXddmyittExAbgbaB/m1SXj2qOudxZpN8oOrImjzlrMg+JiF+3ZWE5qubv+UPAhyQ9LulJSSe0WXX5qOaYLwdOl1RHmv/ky21TWrtp7v/3JuU6MY1tfySdDtQAR7R3LXmStAPwXeDMdi6lrXUldQ8dSWr1PSZpbESsaM+icjYZ+GFEfEfSYaRZD8dExKb2Lqyj6IwtgqXAkLLXg7NlFbeR1JXUnFzeJtXlo5pjRtIxwKXApIhY10a15aWpY+4DjAEelfQKqS91WgcfMK7m77kOmBYR6yPiZeBFUjB0VNUc81nAzwAi4gmgB+nmbJ1VVf/fm6MzBsHTwEhJe0rakTQYPK3RNtOAf8yefwZ4OLJRmA6qyWOWdABwIykEOnq/MTRxzBHxdkQMiIjhETGcNC4yKSJq26fcVlHNv+17Sa0BJA0gdRUtasMaW1s1x7wY+BiApH1IQbCsTatsW9OAM7Kzhw4F3o6Iv23LDjtd11BEbJA0BZhBOuPg1oiYJ+kKoDYipgG3kJqPC0mDMqe2X8Xbrspj/jbQG7g7GxdfHBGT2q3obVTlMXcqVR7zDOA4SfOBjcBFEdFhW7tVHvOFwM2SvkIaOD6zI/9iJ+lOUpgPyMY9vgF0A4iIG0jjIBOAhcBq4Avb/Jkd+M/LzMxaQWfsGjIzs2ZwEJiZFZyDwMys4BwEZmYF5yAwMys4B4FZBZI2Spojaa6k+yXt3Mr7fyU7zx9Jq1pz32bN5SAwq2xNRIyLiDGka03Ob++CzPLiIDBr2hNkN/WStJek30iaJekPkkZlyz8o6ZeSns0e47Pl92bbzpN0Tjseg9kWdbori81ak6QupNsX3JItugk4NyJeknQIcD1wNHAt8PuI+GT2nt7Z9l+MiDcl9QSelnRPR77S1zonB4FZZT0lzSG1BBYAv5PUGxhP/W06ALpnP48GzgCIiI2kW5sDXCDpk9nzIaQbwDkIbLviIDCrbE1EjJPUi3Sfm/OBHwIrImJcNTuQdCRwDHBYRKyW9Cjphmhm2xWPEZhtRTar2wWkG5utBl6WdAq8P3fs/tmmD5GmAEVSF0n9SLc3fysLgVGkW2GbbXccBGZNiIjZwHOkCVBOA86S9Cwwj/ppE/8ZOErS88As0ty5vwG6SloAXE26FbbZdsd3HzUzKzi3CMzMCs5BYGZWcA4CM7OCcxCYmRWcg8DMrOAcBGZmBecgMDMruP8Pzil+z3ONyvIAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis AUPR score: 0.6173494618913611\n"
     ]
    }
   ],
   "source": [
    "precision, recall, aupr_score = aupr(test_y, ood_scores)\n",
    "\n",
    "plot_precision_recall_curve(precision, recall)\n",
    "print('Mahalanobis AUPR score:', aupr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis fpr95 score: 0.8168\n"
     ]
    }
   ],
   "source": [
    "fpr95 = fpr_at_95_tpr(test_y, ood_scores)\n",
    "print('Mahalanobis fpr95 score:', fpr95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n",
      "100%|██████████| 98/98 [00:26<00:00,  3.73it/s]\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.metrics.sml import SML\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "sml = SML(config, model, id_train_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name: patch_embed.norm\n",
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 16.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:05<00:00, 17.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id mean score: 4.8796208895918625 id variance: 1.9578902844789732\n",
      "ood mean score: 5.204622194300016 ood variance: 1.628932299108642\n"
     ]
    }
   ],
   "source": [
    "# hook mask_patch\n",
    "simmim = SimMIMHooker(\n",
    "    config=config,\n",
    "    model=model,\n",
    "    patch_embedding_layer_name='patch_embed.norm',\n",
    "    mask_ratio=mask_ratio,\n",
    ")\n",
    "simmim.hook()\n",
    "\n",
    "test_y, ood_scores, id_ood_scores, ood_ood_scores = compute_ood_scores(\n",
    "    metric=sml,\n",
    "    in_dist_dataloader=id_test_dataloader,\n",
    "    out_of_dist_dataloader=ood_test_dataloader,    \n",
    ")\n",
    "print('id mean score:', np.mean(id_ood_scores), 'id variance:', np.var(id_ood_scores))\n",
    "print('ood mean score:', np.mean(ood_ood_scores), 'ood variance:', np.var(ood_ood_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABC50lEQVR4nO3dd3gUVffA8e8hdOlNkd47IkYR6SJVFH0RRRFFeUGaoKK+KqjIix0UlSIoYAERRVEEFPmpiK9I71UQqYJAKAKhJTm/P+5E1xiSBbI7SfZ8nmef7M7MzpzZ3cyZe+/MvaKqGGOMiVxZ/A7AGGOMvywRGGNMhLNEYIwxEc4SgTHGRDhLBMYYE+EsERhjTISzRJDJicg6EWnqdxzphYg8ISJv+7Ttd0RkqB/bTmsi0llEvj7P99pvMp2xRBBGIrJNRE6IyDER2esdGPKEcpuqWkNV54VyG4lEJIeIPC8iO7z93Cwij4iIhGP7ycTTVER2BU5T1edU9d8h2p6ISD8RWSsix0Vkl4h8LCK1QrG98yUig0Vk0oWsQ1Unq2rLILb1j+R3vr9JEcnuxb7Z+3y3icgEESl7rusyf2eJIPxuUNU8QB3gcuBxf8M5dyKS9SyzPgaaA22BvEAXoAfwWghiEBFJb7/f14D+QD+gEFAZ+Ay4Pq03lMJ3EHI+bnsacCNwB5AfuAxYhvvNnRM/P790SVXtEaYHsA24LuD1S8CsgNdXAwuAw8AqoGnAvELAROA34BDwWcC8dsBK730LgNpJtwlcCpwACgXMuxw4AGTzXt8LbPDWPwcoE7CsAn2AzcCvyexbc+AkUCrJ9HpAPFDRez0PeB5YDPwBfJ4kppQ+g3nAs8CP3r5UBO7xYj4KbAXu85a9yFsmATjmPS4FBgOTvGXKevt1N7DD+ywGBmwvF/Cu93lsAB4Fdp3lu63k7edVKXz/7wCjgFlevIuACgHzXwN2ep/LMqBRwLzBuAPhJG/+v4GrgJ+8z2oPMBLIHvCeGsBc4CDwO/AE0Bo4DZzxPpNV3rL5gfHeenYDQ4Eob15X7zN/FYjx5nUF/ufNF2/ePi+2NUBN3EnAGW97x4Avkv4fAFFeXL94n8kykvyGvOWu877Pf8xL4f8rue+6m/ddzwe+BPomWccq4F/e86oBn98m4Fa/jyEhOzb5HUAkPZL8A5T0/mFe816X8P7J2uJKai2810W9+bOAqUBBIBvQxJt+ufcPWM/7p7rb206OZLb5LdA9IJ6XgTe95+2BLUA1ICswCFgQsKx6/xSFgFzJ7NsLwPdn2e/t/HWAnucdaGriDtafBPyzpvYZzPP+iWt4MWbDnW1XwB2MmgCxQF1v+aYkOXCf5eDwFu6gfxlwCqgWuE/eZ14SWJ10fQHr7QlsT+X7f8fbn6u8+CcDHwbMvxMo7M0bAOwFcgbEfQa4yftscgFX4BJnVm9fNgAPeMvnxR3UBwA5vdf1kn4GAdueDoz1vpNiuESd+J11BeKA+71t5eLviaAV7gBewPseqgHFA/Z5aAr/B4/g/g+qeO+9DCh8Lr+v5Nabwnf9nrePuYC7gB8Dlq+OS6o5vGV24k40svLXSVN1v48joXikt6J1JPhMRI7ifmT7gKe96XcCs1V1tqomqOpcYCnQVkSKA22Anqp6SFXPqOr33vt6AGNVdZGqxqvqu7iD2dXJbPsD4HZwVStAJ28auAPZ86q6QVXjgOeAOiJSJuD9z6vqQVU9kcy6i+AOPMnZ481P9L6qrlXV48CTwK0iEpXSZxDw3ndUdZ2qxnmfwyxV/UWd74GvgUZnieNsnlHVE6q6CndGeJk3/VbgOe8z3wW8nsI6Cqew/4Gmq+pi7zOejKsiBEBVJ6lqjLdvw3EHpCoB7/1JVT/zPpsTqrpMVRd6y2/DHcibeMu2A/aq6nBVPamqR1V1UXIBicjFuM/4AVU9rqr7cGf4nQIW+01V3/C2lfT7P4NLNFUB8X5DwXwW4Eo2g1R1k/cdrlLVmGSWC/bzTc1gbx9P4JJf4G+8M/Cpqp7CfX7bVHWit88rcCctHdMghnTHEkH43aSqeXFnq1X56wBZBugoIocTH0BDoDhQCjioqoeSWV8ZYECS95XCVYMk9QlQ30ssjXHVJj8ErOe1gHUcxJ2hlQh4/84U9uuAF2tyinvzk1vPdtyZfRFS/gySjUFE2ojIQhE56C3flr8nnWDsDXgeCyQ24F+aZHsp7X8MZ9//YLaFiDwsIhtE5Ii3L/n5+74k3ffKIjLTu/DgD1zyTly+FK66JRhlcN/BnoDPfSyuZJDstgOp6re4aqlRwD4RGSci+YLcdrBxBvv5pubP/VDVo7iSdmLCux2XnMF9JvWS/BY7A5ekQQzpjiUCn3hnr+8Aw7xJO3FnygUCHhep6gvevEIiUiCZVe0Enk3yvtyqOiWZbR7CnTHfhmtw+1BVNWA99yVZTy5VXRC4ihR26f9w/zilAieKSD3cP/u3AZMDlymNO6M8kMpn8I8YRCQHLrkNAy5W1QLAbFwCSy3eYOzBVQklF3dS3wAlRST6fDYkIo1wbRC3AgW9fTnCX/sC/9yfMcBGoJKq5sPVtScuvxMof5bNJV3PTlwpskjA555PVWuk8J6/r1D1dVW9Ale9UhlX5ZPq+7xtV0hlGXC/r6tEpGQKyxwHcge8Tu6gnTSeKcDtIlIfV4X2XUBc3yf5LeZR1V5BxJrhWCLw1wighYhchmsEvEFEWolIlIjk9C5/LOkVs78ERotIQRHJJiKNvXW8BfQUkXrelTQXicj1IpL3LNv8AFc3egt/VQsBvAk8LiI1AEQkv4gEXQxW1f/DHQw/EZEa3j5c7e3XGFXdHLD4nSJSXURyA0OAaaoan9JncJbNZsdVn+wH4kSkDRB4SePvQGERyR/sfiTxEe4zKSgiJYC+Z1vQ27/RwBQv5uxe/J1E5LEgtpUXVw+/H8gqIk8BqZ1V58U1zh4TkapA4EFqJlBcRB4Qd1lvXi8pg/tcyiZedeX9vr4GhotIPhHJIiIVRKQJQRCRK73fXzbcwfgkrrSZuK2zJSSAt4H/ikgl7/dbW0QKJ13I+33NBaaLyBUiktXbp54icq+32Eqgk/f/EY37jadmNu7sfwgwVVUT454JVBaRLt76snn7WS2IdWY4lgh8pKr7cY1XT6nqTlyD7RO4g8FO3FlV4nfUBXfmvBHXtvCAt46lQHdc0fwQrsG3awqbnYG7wmWvVyeeGMt04EXgQ6+aYS2uXeJcdMCdUX2Fu0pkEu5KlPuTLPc+rjS0F3cW1s+LIbXP4G+8on0/3AH7EK6UMyNg/kbcGd9Wr3ifXHVZSoYAu4BfcWek03BnzmfTj7+qSA7jqjxuBr4IYltzcJ/bz7jqspOkXBUF8DBun4/iTgimJs7wPpsWwA24z3kz0Myb/bH3N0ZElnvP78Il1vW4z3IawVfF5PO2f8iLPQZ3IQK477+69/l/lsx7X8F9f1/jktp4XENucm7BHbin4kpLa4Fo3HcDrr2pghfHM/z9RCdZXnvAp7irkj4ImH4Ud1LRCXel3l7c/0eO1NaZEclfNQPGhJ6IzMNdyeHL3b0XQkR6AZ1UNagzZWMyCisRGHMWIlJcRBp4VSVVcJdiTvc7LmPSmt1dZ8zZZcddPVMOV9XzIa4dwJhMxaqGjDEmwlnVkDHGRLgMVzVUpEgRLVu2rN9hGGNMhrJs2bIDqlo0uXkZLhGULVuWpUuX+h2GMcZkKCKy/WzzrGrIGGMinCUCY4yJcJYIjDEmwmW4NoLknDlzhl27dnHy5Em/QwmZnDlzUrJkSbJly+Z3KMaYTCZTJIJdu3aRN29eypYti/gzPG5IqSoxMTHs2rWLcuXK+R2OMSaTCVnVkLhBpfeJyNqzzBcReV1EtojIahGpe77bOnnyJIULF86USQBARChcuHCmLvEYY/wTyjaCd3Djo55NG1wvmJVwo2yNuZCNZdYkkCiz758xxj8hqxpS1fkiUjaFRdoD73kDoywUkQIiUvwchrgzxpjMQRUOHoSYGNi/H44ehZMnYds2yJ6d48cPs3/7esre3R+uvDLNN+9nG0EJ/t7f+i5v2j8SgYj0wJUaKF26dFiCO1dRUVHUqlWLuLg4ypUrx/vvv0+BAgUAWLduHffffz+7d+8mISGBu+66i0GDBv15lv/ll1/y5JNPEhsbS44cObj22msZPny4j3tjjElTx4/D7t2wcSP8+CP89ps78P/6K8TGwvaz3uvFt+Wg+w2QPw6WzqlClkyWCIKmquOAcQDR0dHpspe8XLlysXLlSgDuvvtuRo0axcCBAzlx4gQ33ngjY8aMoWXLlsTGxtKhQwdGjx5Nnz59WLt2LX379mXWrFlUrVqV+Ph4xo0b5+/OGGPOzZkzsGULbNjgHqtWwfz5kCMH7Nhx9veVLg3FikHz5nDRRVC5MpQpA4ULczhrHI9seJ23t31CxQLlebXdW2SpcG1IwvczEezm72PAlvSmZXj169dn9erVAHzwwQc0aNCAli3dCIq5c+dm5MiRNG3alD59+vDSSy8xcOBAqlatCriSRa9emXJYVGMyh99+g8WL4euvYdEiWLMGEhIgPv6vZYoVg9q1XYJo2xayZIEKFaBePaheHQoWTHET8QnxXDOmFptiNvHoNY8yuOlgcmU728BtF87PRDAD6CsiHwL1gCNp0j7wwAPgnZmnmTp1YMSIoBaNj4/nm2++oVu3boCrFrriiiv+tkyFChU4duwYf/zxB2vXrmXAgAFpG68xJm3s3u3O7DduhKVLYfbsv8+vWBGio6F8eWjWDEqWhMsug0suOa/NxcTGUChXIaKyRPHstc9SKn8poi+NToMdSVnIEoGITAGaAkVEZBfwNJANQFXfxI092hY3xm4scE+oYgmHEydOUKdOHXbv3k21atVo0aKF3yEZY87H5s0wdKir3lm16q/pBQtCx45QvDg0bQrXXgv586fJJlWVyWsm0/+r/rzQ/AW6X9Gdm6vdnCbrDkYorxq6PZX5CvRJ8w0Heeae1hLbCGJjY2nVqhWjRo2iX79+VK9enfnz5/9t2a1bt5InTx7y5ctHjRo1WLZsGZdddpkvcRsT0U6ccFU7334LEyfCvn1w+PBf8/v3h1tugbJloUQJCMFl3DuP7KTnrJ7M3jybq0teTYPSDdJ8G6lS1Qz1uOKKKzSp9evX/2NauF100UV/Pl++fLmWLl1az5w5o7GxsVquXDmdO3euqqrGxsbq9ddfr6+//rqqqq5atUorVKigmzZtUlXV+Ph4HTNmTLLbSA/7aUyGtn+/6mefqd5zj2qzZqruwk33KFVKtUYN1UGDVL/4QjUhIeThfLD6A837XF7N/WxuHfHTCI2LjwvZtoClepbjaoa4aiijufzyy6lduzZTpkyhS5cufP7559x///306dOH+Ph4unTpQt++fQGoXbs2I0aM4Pbbbyc2NhYRoV27dj7vgTGZREKCa9T97juYNg22bv1rXqFCcP/9rhG3fXt31h9mBXMVpF7JeoxrN45yBf3rPibDjVkcHR2tSQem2bBhA9WqVfMpovCJlP005oKsXAkffgiTJ8OuXW5aVBRUq+Yadm+4AVq0gLx5wx5aXEIcr/70KqfjTzOw8UDA1cqEo+cAEVmmqsm2PFuJwBiT8W3bBv/9r7u6Z8ECN61GDXfgb94cunWDXKG7/DIYq/auotuMbizbs4xba9z6ZwJID93HWCIwxmQ8GzbAxx/DwoXuceiQm37JJfDQQ9CzJ1Sq5G+MnlNxpxg6fygv/PgChXIV4uOOH9OhWod0kQASZZpEEK7ilV8yWhWeMWlu+3Z491348kt38Ad3ll+3rjv779IFGjb0N8ZkbD64mRd/fJE7at3BKy1foXDuwn6H9A+ZIhHkzJmTmJiYTNsVtXrjEeTMmdPvUIwJr9OnYfp0mDQJZs500/LmdY28PXpAzZr+xncWx04f4/ONn9O5dmdqFqvJxr4bKV+wvN9hnVWmSAQlS5Zk165d7N+/3+9QQiZxhDJjMrWEBNfQO3YsrF/veuNMLA3fey907uxu5ErH5v4ylx4ze7D98HbqFq9LtaLV0nUSgEySCLJly2YjdxmTkf36K4wZA6+88lefPZUqQaNGcOut0K4d5Mnjb4ypOHTiEA9//TATVk6gcuHKfN/1e6oVzRhX+WWKRGCMyWASEtwdva++Cj/88Nf1/SLw8svQp4/vV/mci/iEeBpMaMDPMT/zeMPHearJU+TMmnGqci0RGGPC5+RJGDYMnnvOde+Q6I474OGH4fLL/YvtPByIPfBnJ3HPNX+O0vlLU7f4eY+66xtLBMaY0FqwwPXgOX++u+IHXJ/7PXq4K31KlUr5/emQqvL+6vd54KsHeOG6F+hxRQ9uqnqT32GdN0sExpjQmDnTXc+/2xtmpEQJ19jboQPcdFNIOnALh+2Ht3PfzPuY88scril1DY3LNPY7pAtmicAYk3ZOnICpU13d/+rVrsvmfv2gd283+lYGPfgnmrR6Er1m9UJVeaPNG/S+sjdZJIvfYV0wSwTGmAt38CD85z/w9tvudeHCrkv4++6DTHT/S9HcRWlQqgFj242lTIEyfoeTZiwRGGPO39at8OKLkDjOdrt2cNttbgCXHDn8jS0NnIk/w/CfhnMm/gxPNnmSVhVb0bJCy0x346olAmPMuVu/Hho0+GsQl5tvhkcfhauv9jWstLRizwq6zejGir0r6FSzU7rqJC6tWSIwxgQnLs517TxnDkyZ4qZ17uzGCY8O/bi64XIy7iRDvh/CSz++RJHcRfjk1k/4V7V/+R1WSFkiMMak7MgRV/f/xBOu7x9wY/aOHOk6e8tkthzcwrAFw7jrsrsY3nI4BXMV9DukkLNEYIxJ3p49MH68awM4dsyN5NWxo0sIPgzqEkrHTh9j+obpdLmsCzWL1WRT302+jhgWbpYIjDF/iYuD7793d/muXOmm1a8PgwdDy5Z+RhYyc7bMocfMHuw8spPoS6OpVrRaRCUBgIx/Aawx5sJt2OC6di5SBK67zr1u2xZ++gl+/DFTJoGY2Bju/uxuWk9uTe5sufnhnh8yTCdxac1KBMZEsjlz3DCOiXf/VqkCQ4bA3XdD/vz+xhZCiZ3EbTm4hYGNBjKo8aAM1UlcWrNEYEykOXjQdfc8fbq7DBTgiivclUDpZHjHUNl/fD+FcxcmKksUL173ImUKlKHOJXX8Dst3VjVkTKQ4ftx19FapEjz7rLsZ7M47XTXQ0qWZOgmoKhNXTKTyyMq8tewtANpXbW9JwGMlAmMyu4MHXR//L73kxgGoXBlmzcpUN3+lZNvhbfT4ogdzt86lUelGNCvXzO+Q0h1LBMZkZj/8ADfc4O4FKFXKdQXRurXfUYXN+6vep9esXogIo9uO5r7o+zJFJ3FpzT4RYzKjbdvg3/+Gxo0hSxbXHrBjR0QlAYCL81xM4zKNWdd7Hb2u7GVJ4CysRGBMZnL6tLsLuE8f9/qaa1wSKFbM37jC5Ez8GV768SXiNZ6nmjxFywotaVkh8136mtYsPRqTGZw549oBihd3SaBGDVi4EP73v4hJAsv3LOfKt65k0HeD2BSzCVX1O6QMw0oExmR0y5e7u39Pn3aJYNo0aN8eskbGv/eJMyd45vtnGLZgGEUvKsr026Zn6GEj/RDSEoGItBaRTSKyRUQeS2Z+aRH5TkRWiMhqEWkbyniMyVQOHHBXAtWrB9myuRvBdu1yQ0FGSBIA2HpoK6/89Apd63Rlfe/1lgTOQ8h+LSISBYwCWgC7gCUiMkNV1wcsNgj4SFXHiEh1YDZQNlQxGZMpJCRA//6u90+AOnXczWBVq/oaVjj9ceoPPt3wKV3rdKVGsRpsvn9zphoxLNxCWSK4CtiiqltV9TTwIdA+yTIK5POe5wd+C2E8xmRscXHwwQcQFeWSQKNGMG+eqxqKoCQwe/Nsao6uSbcZ3diwfwOAJYELFMpEUALYGfB6lzct0GDgThHZhSsN3J/cikSkh4gsFZGl+/fvD0WsxqRfx4/DI4+46p/OnV07wODBLgk0aZLhB4QP1oHYA3SZ3oXrP7ievDny8uO9P0ZsJ3Fpze+KxNuBd1R1uIjUB94XkZqqmhC4kKqOA8YBREdH26UAJjKouvEAHn0UDh1y9f5PPw2PP+5KBREksZO4rYe28lTjp3ii0RPkyJrxx0ROL0KZCHYDpQJel/SmBeoGtAZQ1Z9EJCdQBNgXwriMSf8+/NANAPPrr67aZ/x4uPHGiEsAvx/7naIXFSUqSxTDWgyjTIEy1L64tt9hZTqhrBpaAlQSkXIikh3oBMxIsswOoDmAiFQDcgJW92Mi1+nTrlvo2293SaB3b1izxg0OH0FJQFUZv3w8VUZWYdyycQDcUOUGSwIhErISgarGiUhfYA4QBUxQ1XUiMgRYqqozgAHAWyLyIK7huKvaXSAmEqnCsGHusW+fq/ufORPy5PE7srDbemgr3b/ozre/fkuTMk24rvx1foeU6YW0jUBVZ+MagQOnPRXwfD3QIJQxGJOuxce7K4Eeewx+8y6ae+stVyqIkEbgQO+ufJfes3sTJVG8ef2bdL+iu/UPFAZ+NxYbE7nWroV77nFjARQpAq+95jqKy53b78h8c2neS7m23LWMuX4MJfOV9DuciGGJwJhwU4VPPnHtAHFx7qqgoUPd5aER5nT8aV743wskaAKDmw6mRYUWtKjQwu+wIo4lAmPCafNm6NULvvkGKlRwPYPWquV3VL5YsnsJ9864l7X71tKldhdUFYnA6rD0wCrfjAmHM2fc8JCVK7sk0K+fGyIyApNA7JlYHv76Ya4efzWHThxiRqcZvHfze5YEfGQlAmNCKSHBDQv5xBOuTaB0aZg8GRo29Dsy3/x66FfeWPwG3et258XrXiR/zvx+hxTxrERgTCgkJLhhIUuVcjeC/f47vPmmuzcgApPAkZNHmLhiIgA1itVgy/1beLPdm5YE0gkrERiTllTdeADPPAPr1rlp//0vPPRQxF4NNOvnWdw38z72HNtD/VL1qVqkKqXyl0r9jSZsrERgTFr55BO49FK49Vb4+Wd44w3XNjBoUEQmgf3H99P50860m9KOgrkK8lO3n6haJHJ6Sc1IrERgzIXavt21AXzwgXvduzcMHw45c/obl4/iE+JpOLEhvx76lWeaPsNjDR8je1R2v8MyZ2GJwJgL8cYb7goggHvvdVcGXXKJvzH5aO+xvRS7qBhRWaIY3nI4ZQuUpWaxmn6HZVIRdNWQiERe2daY5Ki6ht+LL3ZJoHx5+PZb10NohCaBBE1g7NKxVH6jMmOXjgWgXeV2lgQyiFQTgYhcIyLrgY3e68tEZHTIIzMmPdq7190I1quX6xxuwAB3k1izZn5H5pstB7fQ/L3m9JzVkytLXEmriq38Dsmco2BKBK8CrYAYAFVdBTQOZVDGpDuxsW6c4OLF/+oeeudO11tolsi95mLiionUGlOL5XuW89YNb/F/Xf6P8gXL+x2WOUdBtRGo6s4kd/3FhyYcY9KhUaOgb1/3PHdud3lomzb+xpROlM5fmlYVWjGq7ShK5Es6Eq3JKIJJBDtF5BpARSQb0B/YENqwjEkHVq2C1q1ddVCdOm6YyPbtI7J76ESn4k7x/P+eJ0ETGNJsCM3LN6d5+eZ+h2UuUDBl2p5AH9zA87uBOkDvEMZkjL9OnoSHH3YH/4MHoU8fWLwYbropopPAol2LuGLcFTzz/TPsOLIDG0Mq8wimRFBFVTsHThCRBsCPoQnJGJ8cOADdu8Nnn7nX1arB119DycjuF//46eM8+d2TjFg4ghL5SjDz9plcX/l6v8MyaSiYEsEbQU4zJuNatAiKFnVJoGhReO8910VEhCcBgO1HtjN6yWh6RvdkXe91lgQyobOWCESkPnANUFREHgqYlQ83BrExmcPEie5msMTnd98d0VVAAIdPHmba+mn8u+6/qV60Olv6bbERwzKxlKqGsgN5vGXyBkz/A7gllEEZExYHDsC//gU//OBKARMnwvV2tvv5xs/pNasX+47vo2HphlQtUtWSQCZ31kSgqt8D34vIO6q6PYwxGRN6//d/bqjImBjo2hXGjoXskd0Xzr7j++j3ZT+mrptK7YtrM+P2GdZJXIQIprE4VkReBmoAf/aiparXhiwqY0Jp+nRXEihaFJYsgSuu8Dsi38UnxNNgQgN2HNnB0GZDebTBo2SLirwxlCNVMIlgMjAVaIe7lPRuYH8ogzImZCZPhjvvdHcDL1oE5cr5HZGvfjv6G5fkuYSoLFG81vo1yhYoS/Wi1f0Oy4RZMFcNFVbV8cAZVf1eVe8FrDRgMp7nn3dJoHRpN15ABCeBBE1gzJIxVB1ZlTeXvglA20ptLQlEqGBKBGe8v3tE5HrgN6BQ6EIyJo2dOAHdusGUKXDllTB7NhQp4ndUvvk55me6f9Gd+dvnc13562hT0brLiHTBJIKhIpIfGIC7fyAf8EAogzImzWze7PoF+uUXN1zkiy9C1sgdhmP88vH0/bIvObPmZMKNE+hapysS4ZfKmiASgarO9J4eAZrBn3cWG5O+jRwJDz7oxg8YNcr1GBrhyhYoS5uKbRjVdhTF8xb3OxyTTqR0Q1kUcCuuj6GvVHWtiLQDngByAZeHJ0RjzpGquyns/fehaVN46y2oWNHvqHxxKu4U/53/XwCGXjvUOokzyUqpRDAeKAUsBl4Xkd+AaOAxVf0sDLEZc+6OHYO2bd1NYjfdBB9+CDly+B2VLxbsXEC3Gd3YeGAj99a5F1W1aiCTrJQSQTRQW1UTRCQnsBeooKox4QnNmHO0fr07+G/e7LqMfvrpiOwq4tjpYwz8ZiBvLH6DUvlL8VXnr2zUMJOilC4fPa2qCQCqehLYeq5JQERai8gmEdkiIo+dZZlbRWS9iKwTkQ/OZf3GAK6r6EGDoHZtlwSGDYPBgyMyCQDsOLKDscvG0ufKPqzttdaSgElVSiWCqiKy2nsuQAXvtQCqqrVTWrHXxjAKaAHsApaIyAxVXR+wTCXgcaCBqh4SkWIXsC8mEk2Z4i4NPXECmjd3jcJVqvgdVdgdOnGIj9d/TI8relC9aHW29t/KpXkv9Tssk0GklAiqXeC6rwK2qOpWABH5EGgPrA9YpjswSlUPAajqvgvcpokUqvDoo+7sH2DmTNc2EIGlgOkbptN7dm/2H99PkzJNqFKkiiUBc05S6nTuQjuaKwHsDHi9C6iXZJnKACLyI65r68Gq+lXSFYlID6AHQOnSpS8wLJMp3HqrGzu4dm2XBEqV8juisNt7bC/3f3k/09ZPo84ldZh1xyyqFIm80pC5cH7fWZMVqAQ0BUoC80WklqoeDlxIVccB4wCio6NtfLxItmEDPPusSwJlysDy5RAVecNjxCfE02hiI3Ye2clz1z7Hw9c8bJ3EmfMWykSwG3f5aaKS3rRAu4BFqnoG+FVEfsYlhiUhjMtkVDt2QKNGruvo9u1h0qSISwK7/tjFpXkvJSpLFK+3fp1yBctZV9HmggXT6RwikktEzrXMuQSoJCLlRCQ70AmYkWSZz3ClAUSkCK6qaOs5bsdEgvffdyWAw4fdPQKffQZ58vgdVdgkaAJvLHqDqiOrMmbJGADaVGpjScCkiVQTgYjcAKwEvvJe1xGRpAf0f1DVOKAvMAfYAHykqutEZIiI3OgtNgeIEZH1wHfAI3afgvmH++6Du+6CSy+FWbOgYUO/IwqrjQc20nhiY/p91Y+GpRvSrnI7v0MymYyoplzlLiLLcN1Oz1PVy71pa1S1Vhji+4fo6GhdunSpH5s2fvjvf+Gpp6BGDViwAPLl8zuisHp7+dv0nd2X3NlyM6L1CLrU7mJ3B5vzIiLLVDU6uXlBdUOtqkeS/PiswdaE1oEDLgGMGQPNmrmuo3PmTP19mUyFghW4ocoNjGwzkovzXOx3OCaTCiYRrBORO4Ao7wawfsCC0IZlItry5X8NH3nzza6/oAgZT/hk3EmGfD8EgOeaP0ezcs1oVq6Zz1GZzC6YxuL7ceMVnwI+wHVH/UAIYzKRbM6cv5LAk0/Cp59GTBL4cceP1HmzDs//73n2H99PatW2xqSVYEoEVVV1IDAw1MGYCPfuu3DPPVCoEHz0kesyIgIcPXWUJ755glFLRlGmQBnm3DmHlhVa+h2WiSDBJILhInIJMA2YqqprQxyTiTSq0KWLG1i+YkX4/nt3hVCE2PXHLt5e8Tb3X3U/zzZ/ljzZI+eyWJM+pFo1pKrNcCOT7QfGisgaERkU8shMZDh+3DUGT54MV18Ny5ZFRBKIiY35836AakWrsbXfVl5r85olAeOLoG4oU9W9qvo60BN3T8FToQzKRIhffnF9BX3/PTzxRERcHqqqTFs/jeqjq9Pvq35sOrAJwIaNNL4K5oayaiIyWETW4AavX4DrLsKY8/fpp3D55fD776494NlnM33PoXuO7qHDRx3o+HFHSuUrxdLuS62TOJMuBNNGMAGYCrRS1d9CHI+JBN98Ax06QPny8MUXUL263xGFXGIncbuP7ual617iwfoPkjWL330+GuOk+ktU1frhCMREiE8/hTvugIIFIyIJ7DyykxL5ShCVJYpRbUdRrmA5Kheu7HdYxvzNWauGROQj7+8aEVkd8FgTMHKZMcF76SVXEihWDBYvztRJID4hntcXvU7VUX91EteqYitLAiZdSqlE0N/7az1cmQv32GPw4otQv74bSKZQIb8jCpkN+zfQbUY3ftr1E20qtuGGKjf4HZIxKTpriUBV93hPe6vq9sAH0Ds84ZkMTxUef9wlgdKlXftAJk4C45aNo87YOvwc8zPv3/w+s+6YRen8NqqeSd+CuXy0RTLT2qR1ICYTOnkSrr8eXngBiheH1ashVy6/owqpSoUqcXPVm1nfZz131r7Tego1GcJZq4ZEpBfuzL98kjaBvMCPoQ7MZHDHj0ObNm4QmQcfhJdfzpSjiZ04c4LB8wYjIrxw3QvWSZzJkFJqI/gA+BJ4HngsYPpRVT0Y0qhMxrZxI1x3HezeDc8/79oHMqH52+fz7xn/ZvPBzfS8oieqaiUAkyGllAhUVbeJSJ+kM0SkkCUDk6zFi6FlSzhxwg0w36GD3xGluT9O/cFj//cYY5aOoXzB8nxz1zdcW+5av8My5rylViJoByzDDUQTeKqjQPkQxmUyonfege7d4eKLYeFCqJo5x9P97ehvvLPyHR66+iGGNBvCRdkv8jskYy5IqkNVpjc2VGU6dfQolCsHMTGwdat7nokciD3AR+s+oveV7oK534/9biOGmQwlpaEqg+lrqIGIXOQ9v1NEXhERux7O/GXlStdZXEwMfPJJpkoCqsrUtVOpPqo6D3z1AD/H/AxgScBkKsFcPjoGiBWRy4ABwC/A+yGNymQcq1a5AWTy5nVdSf/rX35HlGZ+O/obN029iU6fdKJMgTIs67HM7gw2mVIwvV7FqaqKSHtgpKqOF5FuoQ7MZAA//wzXXguHDsHcuZlqRLH4hHgaT2zM7qO7GdZiGP2v7m+dxJlMK5hf9lEReRzoAjQSkSxAttCGZdK9zz+Hrl3h2DH3PJMkge2Ht1MyX0miskQx+vrRlC9YnoqFKvodljEhFUzV0G24gevvVdW9uLEIXg5pVCb9UoVOneCmm1x10KJFcEPG70snPiGeV356hWqjqjFmqeskrmWFlpYETEQIZqjKvcBkIL+ItANOqup7IY/MpD+xsdC5M0ydCq1bw/r1ULeu31FdsLX71nLNhGsY8PUAmpdvzk1Vb/I7JGPCKpirhm4FFgMdgVuBRSJyS6gDM+nM3r1w2WUwZQrcdRfMng15Mv74um8ufZO6Y+uy9dBWPvjXB8zoNIOS+WwAPhNZgmkjGAhcqar7AESkKPB/wLRQBmbSkSNHoFYtOHAA3nwT7rvP74guWGJ3ENWKVKNjjY6MaDWCohcV9TssY3wRTCLIkpgEPDEEOei9yQT27XNjCx84AMOGZfgkEHsmlqe+e4ooieLFFi/SpGwTmpRt4ndYxvgqmAP6VyIyR0S6ikhXYBYwO7RhGd/FxUHv3q67iN9+gxEjYMAAv6O6IPO2zaP2mNoM/2k4x04fI6PdVW9MqAQzZvEjIvIvoKE3aZyqTg9tWMZXv//uOov78Udo0AD+858MfWXQkZNHeHTuo4xbPo4KBSvw7V3fWlfRxgRIaTyCSsAwoAKwBnhYVXeHKzDjE1V3d/DChfDKK24sgQxuz7E9TFoziYfrP8wzzZ4hd7bcfodkTLqSUtXQBGAm0AHXA+kb57pyEWktIptEZIuInLVTehHpICIqIsl2iGTC5MQJN47AggUwaFCGTgL7j+/njUXuJ1u1SFW29d/Gyy1ftiRgTDJSqhrKq6pvec83icjyc1mxiEQBo3BDXe4ClojIDFVdn2S5vEB/YNG5rN+ksZMnITra3RvQpQs89ZTfEZ0XVWXK2in0+7Iff5z6g1YVW1G5cGW7IsiYFKRUIsgpIpeLSF0RqQvkSvI6NVcBW1R1q6qeBj4E2iez3H+BF4GT5xy9SRtLl7qxA9avhyFD4L33MuSwkjuP7OSGKTfQ+dPOVCxUkRX3rbBO4owJQkolgj3AKwGv9wa8ViC1IZlKADsDXu8C6gUu4CWUUqo6S0QeOduKRKQH0AOgdGnrATtNffutG1tYBMaMgZ49/Y7ovMQlxNH03absPbaXV1u9yv1X3U9UloyXzIzxw1kTgaqG9LIKr/O6V4CuqS2rquOAceAGpgllXBHlww/h9tvdWALLlkHFjNevzrbD2yiVrxRZs2RlbLuxlC9YnvIFbfA8Y85FKG8M2w2UCnhd0puWKC9QE5gnItuAq4EZ1mAcBqp/TwLLl2e4JBCXEMewBcOoNqoao5eMBuC68tdZEjDmPISyg/UlQCURKYdLAJ2AOxJnquoRoEjiaxGZh7tE1cahDLXXXnNXBBUp4kYUq1DB74jOyerfV9NtRjeW/raU9lXa06F6B79DMiZDC1kiUNU4EekLzAGigAmquk5EhgBLVXVGqLZtUjBhgksCWbPCpk1QqJDfEZ2T0UtG0/+r/hTMWZCpt0ylY/WOiIjfYRmToaWaCMT9l3UGyqvqEG+84ktUdXFq71XV2STpjkJVk70uUVWbBhWxOX+zZkG3bpAjhxtiMgMlgcRO4moWq0mnmp14tdWrFMldJPU3GmNSJan1tyIiY4AE4FpVrSYiBYGvVfXKcASYVHR0tC5darVH52zdOjd2QI4cbojJSy7xO6KgHD99nEHfDiJrlqy83NLGQzLmfInIMlVNtg02mMbieqraB+86f1U9BGRPw/hMqH3zDTRr5hqJFy/OMEngm63fUGtMLUYsGsGp+FPWSZwxIRJMG8EZ7y5hhT/HI0gIaVQm7Xz6qetALirKDSZTtarfEaXq8MnDPPz1w4xfMZ5KhSoxv+t8GpVp5HdYxmRawSSC14HpQDEReRa4BRgU0qhM2vjpJze0ZM6crjqoVKnU35MO/H7sdz5c+yH/afAfnm7yNLmy5fI7JGMytWC6oZ4sIsuA5oAAN6nqhpBHZi7M669D//5QrBj873/pPgkkHvz7X92fKkWqsO2BbdYYbEyYBHPVUGkgFvgicJqq7ghlYOY8qbpSwJQpri1gzhyoVMnvqM5KVZm8ZjL9v+rPsdPHaFupLZUKV7IkYEwYBVM1NAvXPiBATqAcsAmoEcK4zPl64gmXBIoXh61bXbVQOrXjyA56zuzJl1u+pH7J+oy/cTyVCqffpGVMZhVM1VCtwNdeR3G9QxaROX9LlsALL7juIjZscDeNpVNxCXE0facp+47v4/XWr9P7yt7WSZwxPjnnI4WqLheReqkvacLq7behe3f3fNKkdJsEth7aSpn8ZciaJStv3fAWFQpVoGyBsn6HZUxEC6aN4KGAl1mAusBvIYvInJuEBOjRA8aPd5eITpoE9dJfno5LiGP4guE8Pe9pXmrxEv3q9aN5+eZ+h2WMIbgSQd6A53G4NoNPQhOOOSeqrhQwYQI0bAgzZ0L+/H5H9Q8r966k24xuLN+znJur3kzH6h39DskYEyDFRODdSJZXVR8OUzwmWJs3uxvF1qyBjh1h6lQ3uEw6M3LxSB6c8yCFcxVmWsdp1lOoMenQWROBiGT1ehBtEM6ATBA2b4bK3hCM/frBq6+muySQ2Elc7Ytr07lWZ15p9QqFcmWcTu6MiSQplQgW49oDVorIDOBj4HjiTFX9NMSxmeT88Qc0buyeT5vmSgXpyLHTxxj4zUCyRWVjWMthNC7TmMZlGvsdljEmBcF0OpcTiMGNUdwOuMH7a8LtzBlo2hT27oUBA9JdEvj6l6+pObombyx+gzPxZ6yTOGMyiJRKBMW8K4bW8tcNZYnsPzzcjh2D1q1hxQp49ll341g6cejEIR76+iHeWfkOVQpXYf4982lYuqHfYRljgpRSIogC8vD3BJDIEkE4bd8O7du7wWSeew4ef9zviP5m3/F9TFs/jccbPs5TTZ4iZ9b0ezezMeafUkoEe1R1SNgiMcnbuBFatIDff4d334W77vI7IgD2HtvLlDVTeLD+g66TuP7bKJy7sN9hGWPOQ0qJIH1dhhKJ/vc/lwROnnTDTLZt63dEqCrvrXqPB+c8SOyZWNpVbkelwpUsCRiTgaXUWGy3ffpp1Spo0+avbqTTQRLYdngbrSe3puvnXaletDore660TuKMyQTOWiJQ1YPhDMQEeOst121Enjzw1VdQrZrfERGXEEezd5txIPYAo9qOomd0T7JIMBedGWPSu/TZM1kk+/xzlwQAvvjC9ySw5eAWyhUoR9YsWZlw4wTKFyxPmQJlfI3JGJO27JQuPZk6FW66CXLnhm3b3D0DPjkTf4bnfniOGqNrMGrJKACalWtmScCYTMhKBOmBKowYAQ895EYVmzsXyvh3wF2+ZzndZnRj5d6VdKzekdtq3OZbLMaY0LNE4DdVuPNO+OADKFwYfvjBDSzjk9cXvc5Dcx6i6EVF+fTWT7m52s2+xWKMCQ+rGvJb//4uCdxzD+zb51sSSOwO4vJLLueuy+5ife/1lgSMiRBWIvDT9OkwciRcf70bWMaHHkSPnjrK4988To6oHAxvNZxGZRrRqEyjsMdhjPGPlQj8MmEC/OtfrgTwwQe+JIGvtnxFzTE1Gb1kNIpaJ3HGRCgrEfhh7Fjo2RMqVYLFiyFfvrBuPiY2hoe+foj3Vr1HtSLV+PHeH6lfqn5YYzDGpB+WCMJtwQKXBEqXhmXLIG/e1N+TxmJOxDB9w3SebPwkAxsNJEfWHGGPwRiTfoS0akhEWovIJhHZIiKPJTP/IRFZLyKrReQbEcncF6nHxUGXLu4+gbffDmsS2HN0D8MWDENVqVy4Mtsf2M6QZkMsCRhjQpcIvPGORwFtgOrA7SJSPcliK4BoVa0NTANeClU8vjtyBJo0ga1b4fnnXWdyYaCqTFgxgWqjqvHkd0+y5eAWAArmKhiW7Rtj0r9QlgiuArao6lZVPQ18CLQPXEBVv1PVWO/lQqBkCOPxz8GDcNVVrlpo+HA3znAY/HroV1pOakm3Gd247JLLWNVzlXUSZ4z5h1C2EZQAdga83gXUS2H5bsCXyc0QkR5AD4DSpUunVXzhsW8flCzphpl85RV48MGwbDYuIY5r37uWmNgYxlw/hh5X9LBO4owxyUoXjcUicicQDTRJbr6qjgPGAURHR2ecaxxV4bLLXBIYOjQsSWBzzGbKFyxP1ixZmdh+IhUKVqBU/lIh364xJuMK5SnibiDwCFTSm/Y3InIdMBC4UVVPhTCe8IqPh6uvdgPNd+0KAweGdHNn4s8wdP5Qao6pycjFIwFoWrapJQFjTKpCWSJYAlQSkXK4BNAJuCNwARG5HBgLtFbVfSGMJbxOnXIDySxeDPnzu/sGQmjpb0vpNqMbq39fTaeanbi91u0h3Z4xJnMJWYlAVeOAvsAcYAPwkaquE5EhInKjt9jLQB7gYxFZKSIzQhVPWHXuDN9+65LB4cOQPXvINvXawteo93Y9DsQe4PNOnzOlwxSKXVQsZNszxmQ+IW0jUNXZwOwk054KeH5dKLfvi++/h08+gWbN3DjDIaKqiAjRl0bT7fJuvNTiJQrkLBCy7RljMq900VicaSxcCK1bQ65c8N57IdnEH6f+4D9z/0POrDl5tfWrNCjdgAalG4RkW8aYyGDXE6aV3buhfn3XPjB3rrtkNI3N3jybGqNrMG75OLJmyWqdxBlj0oSVCNLC8ePQuLF7/u230CBtz9APxB7gga8eYPKaydQoWoNpHadRr2RKt2QYY0zwLBFcKFWoWhV27YLJk0MyzvChE4f44ucveLrJ0zzR6AmyR4Wu8dkYE3ksEVyoBx90SeDKK+GOO1JfPki7/9jN5DWTeeSaR6hUuBLbH9hujcHGmJCwNoILMX06vPYalCsHixalySpVlbeWvUX10dUZPG8wvxz6BcCSgDEmZCwRnK+PP3YjjBUsCEuWpMkIY78c/IXm7zWnx8we1C1el9W9VlOxkH8D2RtjIoNVDZ2Pt9+G7t2hRAmYMQMKF77gVcYlxNH8veYcPHGQse3G8u+6/7ZO4owxYWGJ4FzFx8N//gPFi8P69Rc8zOSmA5uoUKgCWbNk5d2b3qVCoQqUzJc5e+M2xqRPdsp5rnr0cOMLDB58QUngdPxpnpn3DLXG1GLU4lEANCnbxJKAMSbsrERwLjZuhIkToVYtVzV0nhbvXky3Gd1Yu28td9S6g861O6dhkMYYc24sEQTr4EF3iagITJt23o3DIxaOYMDXAyiepzhf3P4F7Sq3S+NAjTHm3FgiCMaZM9ChAxw75i4XrVz5nFeR2EncVSWuonvd7rx43Yvkz5k/BMEaY8y5sUSQGlW49VaYNw+efvqcxxs+cvIIj859lFzZcjGi9QiuKXUN15S6JjSxGmPMebDG4tRMmQKffeZGGRs8+Jze+sWmL6g+ujpvr3ibHFE5rJM4Y0y6ZCWClOzYAX37um6l33wz6LftP76f/l/1Z8raKdQqVovPbvuMK0tcGcJAjTHm/FkiOJuDB6FhQ9ez6EcfQY4cQb/1yKkjzN48m2eaPsNjDR+zTuKMMemaJYLkxMVBp06wc6erFmrfPtW37Dyyk0mrJ/FYw8eoWKgi2x/Ybo3BxpgMwdoIktOtmxtcZujQVJNAgibw5tI3qTG6BkN/GPpnJ3GWBIwxGYUlgqS2bnVVQQ0awMCBKS66OWYz1757Lb1m9eKqElexptca6yTOGJPhWNVQoCNHoEULN9xkKo3DcQlxtHi/BYdPHmb8jeO5p849SBr0QGqMMeFmiSDRwYOuFLB1K0yYADVrJrvYhv0bqFS4ElmzZOX9m9+nQqEKXJr30jAHa4wxaceqhhLdcovrS+iVV+Cee/4x+1TcKZ7+7mlqv1mbkYtHAtCoTCNLAsaYDM9KBOCuDPruO6hb1w09mcTCXQvpNqMb6/evp0vtLnSp3SX8MRpjTIhYIvjkE7j9dihZEr766h+zhy8YziNzH6FkvpLMvmM2bSq18SFIY4wJnchOBHv3uiqhSy+FBQugaNE/ZyVoAlkkC/VL1adndE9euO4F8uW4sEFojDEmPYrsRJDYgdykSVCqFACHTx5mwJwB5M6WmzfavmGdxBljMr3IbSyeMsUNQN+iBTRrBsBnGz+j+qjqvLvqXfLmyGudxBljIkJklgiWL4c77oBq1eCjj9h3fB99Z/fl4/UfU+eSOsy8YyZ1i9f1O0pjjAmLyCwR9O3r/n76KRQowB+n/mDu1rk8e+2zLP73YksCxpiIEnklgkWL4Kef2HFlZd7f9wlPVHmCioUqsuOBHeTNkdfv6IwxJuxCWiIQkdYisklEtojIY8nMzyEiU735i0SkbCjj4eRJEhpcw+grocaNu3juf8/92UmcJQFjTKQKWSIQkShgFNAGqA7cLiLVkyzWDTikqhWBV4EXQxUPwKYbrqHpXQn0uR7ql2nAut7rrJM4Y0zEC2WJ4Cpgi6puVdXTwIdA0j6d2wPves+nAc0lRD23xQ17iVY1V7CmZDYmtp/InDvnULZA2VBsyhhjMpRQthGUAHYGvN4F1DvbMqoaJyJHgMLAgcCFRKQH0AOgdOnS5xVM1pq1mbSjPRX6j6B44bLntQ5jjMmMMkRjsaqOA8YBREdHn9/F/a1b07B167QMyxhjMoVQVg3tBkoFvC7pTUt2GRHJCuQHYkIYkzHGmCRCmQiWAJVEpJyIZAc6ATOSLDMDuNt7fgvwrdrtvMYYE1Yhqxry6vz7AnOAKGCCqq4TkSHAUlWdAYwH3heRLcBBXLIwxhgTRiFtI1DV2cDsJNOeCnh+EugYyhiMMcakLDK7mDDGGPMnSwTGGBPhLBEYY0yEs0RgjDERTjLa1Zoish/Yfp5vL0KSu5YjgO1zZLB9jgwXss9lVLVocjMyXCK4ECKyVFWj/Y4jnGyfI4Ptc2QI1T5b1ZAxxkQ4SwTGGBPhIi0RjPM7AB/YPkcG2+fIEJJ9jqg2AmOMMf8UaSUCY4wxSVgiMMaYCJcpE4GItBaRTSKyRUQeS2Z+DhGZ6s1fJCJlfQgzTQWxzw+JyHoRWS0i34hIGT/iTEup7XPAch1EREUkw19qGMw+i8it3ne9TkQ+CHeMaS2I33ZpEflORFZ4v++2fsSZVkRkgojsE5G1Z5kvIvK693msFpG6F7xRVc1UD1yX178A5YHswCqgepJlegNves87AVP9jjsM+9wMyO097xUJ++wtlxeYDywEov2OOwzfcyVgBVDQe13M77jDsM/jgF7e8+rANr/jvsB9bgzUBdaeZX5b4EtAgKuBRRe6zcxYIrgK2KKqW1X1NPAh0D7JMu2Bd73n04DmIiJhjDGtpbrPqvqdqsZ6LxfiRozLyIL5ngH+C7wInAxncCESzD53B0ap6iEAVd0X5hjTWjD7rEA+73l+4LcwxpfmVHU+bnyWs2kPvKfOQqCAiBS/kG1mxkRQAtgZ8HqXNy3ZZVQ1DjgCFA5LdKERzD4H6oY7o8jIUt1nr8hcSlVnhTOwEArme64MVBaRH0VkoYhk9IG6g9nnwcCdIrILN/7J/eEJzTfn+v+eqgwxeL1JOyJyJxANNPE7llASkSzAK0BXn0MJt6y46qGmuFLffBGppaqH/QwqxG4H3lHV4SJSHzfqYU1VTfA7sIwiM5YIdgOlAl6X9KYlu4yIZMUVJ2PCEl1oBLPPiMh1wEDgRlU9FabYQiW1fc4L1ATmicg2XF3qjAzeYBzM97wLmKGqZ1T1V+BnXGLIqILZ527ARwCq+hOQE9c5W2YV1P/7uciMiWAJUElEyolIdlxj8Iwky8wA7vae3wJ8q14rTAaV6j6LyOXAWFwSyOj1xpDKPqvqEVUtoqplVbUsrl3kRlVd6k+4aSKY3/ZnuNIAIlIEV1W0NYwxprVg9nkH0BxARKrhEsH+sEYZXjOAu7yrh64GjqjqngtZYaarGlLVOBHpC8zBXXEwQVXXicgQYKmqzgDG44qPW3CNMp38i/jCBbnPLwN5gI+9dvEdqnqjb0FfoCD3OVMJcp/nAC1FZD0QDzyiqhm2tBvkPg8A3hKRB3ENx10z8omdiEzBJfMiXrvH00A2AFV9E9cO0hbYAsQC91zwNjPw52WMMSYNZMaqIWOMMefAEoExxkQ4SwTGGBPhLBEYY0yEs0RgjDERzhKBSZdEJF5EVgY8yqaw7LE02N47IvKrt63l3h2q57qOt0Wkuvf8iSTzFlxojN56Ej+XtSLyhYgUSGX5Ohm9N04Tenb5qEmXROSYquZJ62VTWMc7wExVnSYiLYFhqlr7AtZ3wTGltl4ReRf4WVWfTWH5rrheV/umdSwm87ASgckQRCSPN47CchFZIyL/6GlURIqLyPyAM+ZG3vSWIvKT996PRSS1A/R8oKL33oe8da0VkQe8aReJyCwRWeVNv82bPk9EokXkBSCXF8dkb94x7++HInJ9QMzviMgtIhIlIi+LyBKvj/n7gvhYfsLrbExErvL2cYWILBCRKt6duEOA27xYbvNinyAii71lk+ux1UQav/vetoc9knvg7opd6T2m4+6Cz+fNK4K7qzKxRHvM+zsAGOg9j8L1N1QEd2C/yJv+H+CpZLb3DnCL97wjsAi4AlgDXIS7K3sdcDnQAXgr4L35vb/z8MY8SIwpYJnEGG8G3vWeZ8f1IpkL6AEM8qbnAJYC5ZKJ81jA/n0MtPZe5wOyes+vAz7xnncFRga8/zngTu95AVxfRBf5/X3bw99HputiwmQaJ1S1TuILEckGPCcijYEE3JnwxcDegPcsASZ4y36mqitFpAlusJIfva41suPOpJPzsogMwvVT0w3Xf810VT3uxfAp0Aj4ChguIi/iqpN+OIf9+hJ4TURyAK2B+ap6wquOqi0it3jL5cd1FvdrkvfnEpGV3v5vAOYGLP+uiFTCdbOQ7SzbbwncKCIPe69zAqW9dZkIZYnAZBSdgaLAFap6RlyPojkDF1DV+V6iuB54R0ReAQ4Bc1X19iC28YiqTkt8ISLNk1tIVX8WN9ZBW2CoiHyjqkOC2QlVPSki84BWwG24gVbAjTZ1v6rOSWUVJ1S1jojkxvW/0wd4HTcAz3eqerPXsD7vLO8XoIOqbgomXhMZrI3AZBT5gX1eEmgG/GPMZXHjMP+uqm8Bb+OG+1sINBCRxDr/i0SkcpDb/AG4SURyi8hFuGqdH0TkUiBWVSfhOvNLbszYM17JJDlTcR2FJZYuwB3UeyW+R0Qqe9tMlrrR5voBA+SvrtQTuyLuGrDoUVwVWaI5wP3iFY/E9UprIpwlApNRTAaiRWQNcBewMZllmgKrRGQF7mz7NVXdjzswThGR1bhqoarBbFBVl+PaDhbj2gzeVtUVQC1gsVdF8zQwNJm3jwNWJzYWJ/E1bmCg/1M3/CK4xLUeWC5u0PKxpFJi92JZjRuY5SXgeW/fA9/3HVA9sbEYV3LI5sW2znttIpxdPmqMMRHOSgTGGBPhLBEYY0yEs0RgjDERzhKBMcZEOEsExhgT4SwRGGNMhLNEYIwxEe7/AUX2OdSH5T+zAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SML AUROC score: 0.65204821\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, auroc_score = auroc(test_y, ood_scores)\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "print('SML AUROC score:', auroc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAjOUlEQVR4nO3deZgV5Z328e8tIIssRsCorI4QEUHRtBsm4xJXgpjNRKLjmHhpdCTOJL4aM84YX3UyxlzJ5HXEuIxm0aiJMVFMMCRuMRg1NoLKEpWgQpM4IoqILLL83j+eOvbp5kCfbrq66a77c13n6nOq6tT5Fcu5+3meqnoUEZiZWXHt0N4FmJlZ+3IQmJkVnIPAzKzgHARmZgXnIDAzKzgHgZlZwTkIrMOQdJqk31ax3Q2S/r0tamoLkl6RdEz2/HJJt7d3Tda5OAisVWRfVmskrZL0v5J+KKl3a35GRPwkIo6rYrtzI+LK1vzsEkkh6d3sOJdK+q6kLnl8VktI6ivpe5IWZzX+JXs9oL1rs+2Xg8Ba00kR0Rs4EKgB/q3xBpK6tnlVrW//7DiPAD4HfLGd6wFA0o7AQ8C+wAlAX+AwYDlwcAv21xn+rqwKDgJrdRGxFHgAGAPv/xZ9vqSXgJeyZRMlzZG0QtIfJe1Xer+kIZJ+IWmZpOWSrsuWnylpZvZckv5L0uuSVkp6XlLp834o6aqy/Z0taaGkNyVNk7RH2bqQdK6kl7JapkpSlce5EHgcGFe2v5Yc116SHs6WvSHpJ5J2buYfO8AZwFDgkxExPyI2RcTrEXFlREwvO94RZTW9/2cl6UhJdZK+Juk14AeSFkiaWLZ916z+A7PXh2bHuULSs5KObEHd1s4cBNbqJA0BJgCzyxZ/AjgEGC3pAOBW4EtAf+BGYJqk7lk3y6+AV4HhwCDgrgofcxzw98CHgH7AZ0m/+Tau5WjgP7P1u2f7bby/icBBwH7ZdsdXeZyjgI8CC7PXLT0uZTXuAewDDAEur6aGRo4BfhMRq1rw3pLdgF2AYcA5wJ3A5LL1xwNvRMQzkgYBvwauyt7zf4B7JA3chs+3duAgsNZ0r6QVwEzg98A3y9b9Z0S8GRFrSF8wN0bEUxGxMSJ+BKwDDiV1YewBXBQR70bE2oiYWeGz1gN9gFGAImJBRPytwnanAbdGxDMRsQ74OnCYpOFl21wdESsiYjHwCGW/4W/BM5LeBRYAjwLXZ8tbdFwRsTAifhcR6yJiGfBdUrdTc/UHKv0ZNMcm4BtZLWuAO4BJknpl6z9PCgeA04HpETE9a338Dqgl/RJgHYiDwFrTJyJi54gYFhH/lH2RlCwpez4MuDDrTliRhccQ0hflEODViNiwtQ+KiIeB64CpwOuSbpLUt8Kme5B+Cy+9bxWp5TCobJvXyp6vBnoDSJqXDbiukvTRsm0OzLb5HKmVs9O2HJekD0q6Kxt8XgncDrRkcHc5qdWzLZZFxNrSi6z7awFwUhYGk0jhAOl4T2l0vB9phRqsjTkIrK2U3+Z2CfAfWWiUHr0i4s5s3dBqBioj4tqI+DAwmtRFdFGFzf5K+sICQNJOpN+cl1ax/30jonf2+EOjdRERPwOeAC7bxuP6JunPZ2xE9CX9pl3VOEUjDwLHZ8e4JauBXmWvd2u0vtLtiEvdQycD87NwgHRMtzU63p0i4uoW1G7tyEFg7eFm4FxJh2SDvjtJ+rikPsCfSN0bV2fLe0g6vPEOJB2Uvb8b8C6wltSt0didwBckjZPUnfSl+1REvNJKx3I1cLak3bbhuPoAq4C3s373SoFWjdtIX873SBolaQdJ/SX9q6RSd80c4POSukg6geq6oO4ijcmcR31rAFLL5SRJx2f765ENOA9uYf3WThwE1uYiohY4m9S18xZpsPXMbN1G4CRgBLAYqCN1wTTWl/TF+xap62c58O0Kn/Ug8O/APaQv4r2AU1vxWJ4HHiP1/bf0uP4vqbvpbdLg6y9aWMs60oDxn4HfAStJATQAeCrb7J+zOlaQxk/urWK/fyO1fMYDPy1bvoTUSvhXYBkphC7C3ysdjjwxjZlZsTm5zcwKzkFgZlZwDgIzs4JzEJiZFVyHu6nUgAEDYvjw4e1dhplZhzJr1qw3IqLi7T86XBAMHz6c2tra9i7DzKxDkfTqlta5a8jMrOAcBGZmBecgMDMruA43RmBm1lLr16+nrq6OtWvXNr1xB9WjRw8GDx5Mt27dqn6Pg8DMCqOuro4+ffowfPhwqpyIrkOJCJYvX05dXR177rln1e/LrWtI0q1K0wjO3cJ6SbpWaQrB50pT35mZ5WXt2rX079+/U4YAgCT69+/f7BZPnmMEPyRNoL0lJwIjs8c5wPdzrMXMDKDThkBJS44vtyCIiMeAN7eyycnAj7MJPp4EdpaU38xGM2fCZZfBe+/l9hFmZh1Re541NIiG0xfW0XD6wPdJOkdSraTaZcuWtezTnngCrrwS1q9v2fvNzFpBly5dGDduHGPGjOGkk05ixYoV76+bN28eRx99NHvvvTcjR47kyiuvpHyqgAceeICamhpGjx7NAQccwIUXXtgqNXWI00cj4qaIqImImoEDK14hbWbWIfTs2ZM5c+Ywd+5cdtllF6ZOnQrAmjVrmDRpEpdccgkvvPACzz77LH/84x+5/vrrAZg7dy5Tpkzh9ttvZ/78+dTW1jJixIhWqak9g2ApaULvksFUMY+smVlncdhhh7F0afrau+OOOzj88MM57rjjAOjVqxfXXXcdV1+dpoC+5ppruPTSSxk1ahSQWhbnnXdeq9TRnqePTgOmSLoLOAR4O5sSz8wsf//yLzBnTuvuc9w4+N73qtp048aNPPTQQ5x11llA6hb68Ic/3GCbvfbai1WrVrFy5Urmzp3bal1BjeUWBJLuBI4EBkiqA74BdAOIiBuA6cAE0ryuq4Ev5FWLmdn2Ys2aNYwbN46lS5eyzz77cOyxx7Z3SfkFQURMbmJ9AOfn9flmZltV5W/ura00RrB69WqOP/54pk6dygUXXMDo0aN57LHHGmy7aNEievfuTd++fdl3332ZNWsW+++/f6vX1CEGi83MOptevXpx7bXX8p3vfIcNGzZw2mmnMXPmTB588EEgtRwuuOACLr74YgAuuugivvnNb/Liiy8CsGnTJm644YZWqcVBYGbWTg444AD2228/7rzzTnr27Ml9993HVVddxd57783YsWM56KCDmDJlCgD77bcf3/ve95g8eTL77LMPY8aMYdGiRa1Sh+81ZGbWhlatWtXg9f333//+87Fjx/Loo49u8b0TJ05k4sSJrV6TWwRmZgXnIDAzKzgHgZkVSvktGzqjlhyfg8DMCqNHjx4sX76804ZBaT6CHj16NOt9Hiw2s8IYPHgwdXV1tPjmlR1AaYay5nAQmFlhdOvWrVkzdxWFu4bMzArOQWBmVnAOAjOzgnMQmJkVnIPAzKzgHARmZgXnIDAzKzgHgZlZwTkIzMwKzkFgZlZwDgIzs4JzEJiZFZyDwMys4BwEZmYF5yAwMys4B4GZWcE5CMzMCs5BYGZWcA4CM7OCcxCYmRWcg8DMrOAcBGZmBZdrEEg6QdILkhZKuqTC+qGSHpE0W9JzkibkWY+ZmW0utyCQ1AWYCpwIjAYmSxrdaLN/A34WEQcApwLX51WPmZlVlmeL4GBgYUQsioj3gLuAkxttE0Df7Hk/4K851mNmZhXkGQSDgCVlr+uyZeUuB06XVAdMB75caUeSzpFUK6l22bJledRqZlZY7T1YPBn4YUQMBiYAt0narKaIuCkiaiKiZuDAgW1epJlZZ5ZnECwFhpS9HpwtK3cW8DOAiHgC6AEMyLEmMzNrJM8geBoYKWlPSTuSBoOnNdpmMfAxAEn7kILAfT9mZm0otyCIiA3AFGAGsIB0dtA8SVdImpRtdiFwtqRngTuBMyMi8qrJzMw21zXPnUfEdNIgcPmyy8qezwcOz7MGMzPbuvYeLDYzs3bmIDAzKzgHgZlZwRUnCB55JP3s3bt96zAz284UJwgeeKC9KzAz2y4VJwjMzKwiB4GZWcE5CMzMCs5BYGZWcA4CM7OCcxCUrFsHq1e3dxVmZm2umEEgbb6sRw/YaSf4zGfavh4zs3ZUzCBo7Je/rH9+zz31zxctgvHjYd68tq/JzKyNFDcI5s9PP195BT71qYbrrr8+tRr22gueeALGjGnz8szM2kpxg+A3v0k/jz1283Xnn7/5sk2b8q3HzKydFDcIHn4Yfv5zWLgwve7dOy3bki5dUiuh9Fi0qG3qNDPLWXGD4Ne/hlNOqX99991w1FH1r999FyK2fI+ivfaCe++FjRthxIj6gFizJj1awq0OM2sHxQ2CcnvvDSeckJ6vXQvr10OvXun1CSeklkPJJz5R//yTn4SuXeEvf6lf1qtXekiphVHeiviP/4A33oDXXkvb/vnPKVBOPz2tL7U61q3L9XDNzMqpo00RXFNTE7W1tc1/Y6VTRktmzoTDmzFj5pNPwmGHNVx2++3pCz0Pgwal1sLGjXDlldC3L5x6av36iK0fn5kVnqRZEVFTaZ1bBJBaBM1x6KHpy/fll+Gaa1Ir4rTT0rIImDu3ftuhQ9Nv+FubB2HMGHjzTXj77crrly6Fv/0NXn8dvvQlmDy5YUtjhx3Szw98IJ0FNWMGLFiQgsPMrAm5Tl7fITz0EAwY0LL3Dh8OF120+fJ9960PhdJv6u+8k35GwKpV6QK2bt02f2/pfQsWwP33wzHHwAUXpHGIH/946/WsWAF77ll53e67w5IlKaSefx5efTUt++IXqz1aM+ukit019Pjj6YKxzmDxYpgwofUufrvvPpg0qXX2ZWbtzl1DW9JZQgBSF9TcufUtitJj1Sr41rfSNqNGpcFpgCFDtr6/k09u2P0kwa23pm4udzmZdSrFDoIi2GknuPji+u6mhQvT88WLNw+NiDQo/f3vV97XWWelLq2uXRsGxOGHe0zCrAMrXhB89KPp59e+1r51bK8kOPfc+mDYuDENYt92GwweDHvssfl7/vhHGD1684Aof3zlK/Df/93wVFsz2y4Ub4zgqKO2fgWxVS8inTF11VVpIHvjxnR2UzX694fly1Ow/PWv0L176nb69KfT6bIf+hDsthscfTT065fOjCpdcLdD8X5/MdtWWxsjKF4QfOxj8OCDrVuUbdmmTTBtGjz2GPzv/8Idd+TzOV261F/lffrpcOCB6ayuUaNSi2b5chg50iFiheUggPogOPZY+O1vW7coa56I1ArYbbf0BV6ybBn85CfpbK6ePeG999JV3aNHw7Bh6UK6JUvgD39I2++4Y9qmucaPh0MOSS2QceNSl9Y++6TP2GWX1Dox62QcBFAfBMcfX3/nUet85s6FWbNS2L/xRhrcfucdeOsteO656u/nNHhw+jfz2mvpeouxY2H27PTvZ8aMtPyoo1I4DR0KNTXpDK1Ro1J3V+/eKah69Ei3HHFLxNqZgwDqg2DChHTDOSumTZvg2WfTWVMAH/xgumXInDmpdbJgQbqSe82a1EKB+m6nkh490thIS+y0U7qhIaRWye67p26rQYNS+PTvn64QHz06hc/OO6dtfRsR20ZbC4JcryyWdALw/4AuwP9ExNUVtvkscDkQwLMR8fk8a/JvZgW3ww5wwAHpUXLooU2/LyKFQ8+eDb+Q33wTnn46dS+99loaC+ndO92/auHCNNhdur5j6NAUKKWTFRYuhKeeqq7u8vDZY48UHCtXputBBgyA3/0ujX+tWwfXXps+y6xKubUIJHUBXgSOBeqAp4HJETG/bJuRwM+AoyPiLUm7RsTrW9vvNrcIJk1KV82abS82bEitkNdeS/NcdO8OL76Yupr+8AfYb7/0pT99emox7LIL/OlPzbtuo0+f+tucfPzjqfVx8MFpv336pH1u2pS6s/r0Sfff2m03t0I6kfZqERwMLIyIRVkRdwEnA/PLtjkbmBoRbwE0FQKtwi0C29507ZoGqocNS91F1dqwIbU0unVLofH662lsZO7cNFfG0qWp5TN7dn0IQH3X6I03Vv9Zw4alLrKzzkpdVx//eGqNOCg6hTyDYBCwpOx1HdD4X/mHACQ9Tuo+ujwiNhvJlXQOcA7A0G1t8vofrnUWXcv++/bunR7nnpteX3fdlt+3cmU6pXblynT21ptv1ndtDRuWli9ZAvfcAy+8kLrDXn01vffSSxvuqzTmMWQIHHdcujVJTQ0MHNiwPtuuVfU3JelwUj/+sOw9AiIi/q4VPn8kcCQwGHhM0tiIWFG+UUTcBNwEqWtomz7RQWBF17dvekC6U+6WNL76/vXX06D6vfem25136QK/+lVat2QJ3HJLepTr3h0+9al05tSZZ6buqB13bJ3jsFZTbWTfAnwFmAVU2zG5FCi/s9ngbFm5OuCpiFgPvCzpRVIwPF3lZzRfBztLymy7seuu6bf+445ruLx0j6qZM1MrokuXNPve7NnpIsI770zblUKie/c02L3HHumsqfHj03hHv37p9NuxY9NzazPVBsHbEbGFyXu36GlgpKQ9SQFwKtD4jKB7gcnADyQNIHUV5Tsr/IgRue7erHBK06wecUR6NLZ+PTzzTLoVSZ8+UFeXruuYOTOtv/vuzd8zdGgaGJ89OwXDwQenM6cOOihNH/vBD+Z7TAVTbRA8IunbwC+A9yfUjYhntvSGiNggaQowg9T/f2tEzJN0BVAbEdOydcdJmk9qaVwUEctbeCxbN3o0zJ8P//APuezezLagW7c0CH7//Zuve++9dH+qNWvSeMWcOekaj7/8JXU7Sek02+efb/i+XXdNg+UjRqRrLcaPT+8fNiwNkB94YNrGXcFVqer0UUmPVFgcEXF065e0dS0+fXTffVMQPP98mhrSzDqO9evTqbX33ZeuGF+0KLUqqr2BZP/+6XYiS5em27JPmJCuvyi/xUknt82nj0bEUa1bUjvybwhmHU+3bunahosv3nzdpk2pZfHGG+mW6D17pussnngiTd86a1b60n/oobR9aXrW0h1tJ02CiRPTtLBbmuq1k6v2rKF+wDeAv88W/R64IiK2MNu6mVkb2WGHNH4weDB89rNp2UknVd52xYo0096CBfA//5OWTZuWHiW77566nD760TRoXVOTHqUzrTqhascIbgXmAtmfMv8A/AD4VB5FmZnlYued4atfTc9vvjn9XLw4dTEtXpxaFDNmpHGL0l1uGxs/Hj7ykTQGccYZ6ZqJDq7aMYI5ETGuqWVtYZvHCObO3fq502Zm69eni+jefTd1Mb32Whrsnj1781PQpXQm05FHwuc/n1om26HWuMXEGkkfiYiZ2Q4PB9a0VoFtwtcPmFm1unWrP9V8//3Tz8svr1+/cmU6q+mXv4SXX4YHHkiP0kV4RxwBJ54In/kM7LVXm5beEtXeeOc8YKqkVyS9ClwHnJtfWTnyYLGZbau+fdNv/3ffDbW16a6v11+fTl/93Ofg97+HSy5JYSKlU1qvuiqdtbQdatbdRyX1BYiIlblV1IQWdw2NHp0GiObNS8/NzPLyzjvpbrF//nMaiH6m7JKrXXZJd3Y94wz48pfT7TfaQIsnppF0ekTcLumrldZHxHdbqcaqOQjMrMNZty4Fwvz58Oij6VFSUwMXXphaEjn2WGwtCJrqGtop+9lnCw8zM2tK9+5wyinwjW/AI4/A6tXw05+m7qXaWpg8OZ0G268ffP3r9bPYtZHiTFW5zz6pmeYWgZltT95+G267LY03PP54/YRDEybA97/farPNbUuLoLSDayT1ldRN0kOSlkk6vVWqa2seLDaz7Um/fjBlShpgXr8+naZ67LFpjGHYMDj//NxLqPasoeOyAeKJwCvACOCivIoyMyskKd3u4re/TRe5jR2bzkY64ohcT4GvNghK1xt8HLjbt5YwM8vZUUeli9kmToTHHks3zstJtUHwK0l/Bj4MPCRpILA2t6rMzCxNBTptWrow7a234MEHc/mYqoIgIi4BxgM12Wxi75Imou84OtiguJkZkLqLbr4Zhg9PA8s52OotJiQdHREPS/pU2bLyTX6RS1V58mCxmXU0O+8ML70EXau9K1DzNLXXI4CHgUr3dA06YhCYmXVEOYUANBEEEfGN7OcXcqvAzMzaVbXXEXxT0s5lrz8g6arcqjIzszZT7VlDJ0bEitKLiHgLmJBLRWZm1qaqDYIukrqXXkjqCXTfyvbbH581ZGZWUbWjDz8hXT/wg+z1F4Af5VNSznzWkJlZA1UFQUR8S9KzwDHZoisjYkZ+ZZmZWVtpzvlIC4ANEfGgpF6S+kTEO3kVZmZmbaPas4bOBn4O3JgtGgTcm1NNZmbWhqodLD4fOBxYCRARLwG75lVULjxYbGZWUbVBsC4i3iu9kNSVdGVxx+PBYjOzBqoNgt9L+legp6RjgbuB+/Mry8zM2kq1QfA1YBnwPPAlYDrwb3kVZWZmbafJs4YkdQHmRcQo4Ob8SzIzs7bUZIsgIjYCL0hqnRmU24sHi83MKqq2a+gDwLxs4vpppUdTb5J0gqQXJC2UdMlWtvu0pJBUU23hLebBYjOzBqq9oOzfm7vjrEtpKnAsUAc8LWlaRMxvtF0f4J+Bp5r7GWZmtu2amqGsB3AuMII0UHxLRGyoct8HAwsjYlG2r7tI01vOb7TdlcC3gIuaUbeZmbWSprqGfgTUkELgROA7zdj3IGBJ2eu6bNn7JB0IDImIX29tR5LOkVQrqXbZsmXNKMHMzJrSVNfQ6IgYCyDpFuBPrfXBknYAvguc2dS2EXETcBNATU1Ny0Z9PVhsZlZRUy2C9aUnzegSKlkKDCl7PThbVtIHGAM8KukV4FBgWu4Dxh4sNjNroKkWwf6SVmbPRbqyeGX2PCKi71be+zQwUtKepAA4Ffh8aWVEvA0MKL2W9CjwfyKittlHYWZmLdbU5PVdWrrjiNggaQowA+gC3BoR8yRdAdRGRJOnn5qZWf6aMx9Bs0XEdNLtKMqXXbaFbY/MsxYzM6us2gvKOj4PFpuZVVScICjxYLGZWQPFCwIzM2vAQWBmVnAOAjOzgitOEHiw2MysouIEQYkHi83MGiheEJiZWQMOAjOzgnMQmJkVnIPAzKzgihMEPmvIzKyi4gRBic8aMjNroHhBYGZmDTgIzMwKzkFgZlZwxQkCDxabmVVUnCAo8WCxmVkDxQsCMzNrwEFgZlZwDgIzs4IrThB4sNjMrKLiBEGJB4vNzBooXhCYmVkDDgIzs4JzEJiZFVxxgsCDxWZmFRUnCEo8WGxm1kDxgsDMzBpwEJiZFVyuQSDpBEkvSFoo6ZIK678qab6k5yQ9JGlYnvWYmdnmcgsCSV2AqcCJwGhgsqTRjTabDdRExH7Az4Fr8qrHg8VmZpXl2SI4GFgYEYsi4j3gLuDk8g0i4pGIWJ29fBIYnGM9iQeLzcwayDMIBgFLyl7XZcu25CzggUorJJ0jqVZS7bJly1qxRDMz2y4GiyWdDtQA3660PiJuioiaiKgZOHBg2xZnZtbJdc1x30uBIWWvB2fLGpB0DHApcERErMuxHjMzqyDPFsHTwEhJe0raETgVmFa+gaQDgBuBSRHxeo61eLDYzGwLcguCiNgATAFmAAuAn0XEPElXSJqUbfZtoDdwt6Q5kqZtYXetx4PFZmYN5Nk1RERMB6Y3WnZZ2fNj8vx8MzNr2nYxWGxmZu3HQWBmVnDFCQIPFpuZVVScICjxYLGZWQPFCwIzM2vAQWBmVnAOAjOzgnMQmJkVXHGCwGcNmZlVVJwgKPFZQ2ZmDRQvCMzMrAEHgZlZwTkIzMwKrjhB4MFiM7OKihMEJR4sNjNroHhBYGZmDTgIzMwKzkFgZlZwxQkCDxabmVVUnCAo8WCxmVkDxQsCMzNrwEFgZlZwDgIzs4IrThB4sNjMrKLiBEGJB4vNzBooXhCYmVkDDgIzs4JzEJiZFVxxgsCDxWZmFRUnCEo8WGxm1kDxgsDMzBrINQgknSDpBUkLJV1SYX13ST/N1j8laXie9ZiZ2eZyCwJJXYCpwInAaGCypNGNNjsLeCsiRgD/BXwrr3rMzKyyPFsEBwMLI2JRRLwH3AWc3Gibk4EfZc9/DnxMyqkTv1ev9NNjBGZmDXTNcd+DgCVlr+uAQ7a0TURskPQ20B94o3wjSecA5wAMHTq0ZdU8+CDcfTcMHNiy95uZdVIdYrA4Im6KiJqIqBnY0i/yESPg619v3cLMzDqBPINgKTCk7PXgbFnFbSR1BfoBy3OsyczMGskzCJ4GRkraU9KOwKnAtEbbTAP+MXv+GeDhCF/5ZWbWlnIbI8j6/KcAM4AuwK0RMU/SFUBtREwDbgFuk7QQeJMUFmZm1obyHCwmIqYD0xstu6zs+VrglDxrMDOzresQg8VmZpYfB4GZWcE5CMzMCs5BYGZWcOpoZ2tKWga82sK3D6DRVcsF4GMuBh9zMWzLMQ+LiIpX5Ha4INgWkmojoqa962hLPuZi8DEXQ17H7K4hM7OCcxCYmRVc0YLgpvYuoB34mIvBx1wMuRxzocYIzMxsc0VrEZiZWSMOAjOzguuUQSDpBEkvSFoo6ZIK67tL+mm2/ilJw9uhzFZVxTF/VdJ8Sc9JekjSsPaoszU1dcxl231aUkjq8KcaVnPMkj6b/V3Pk3RHW9fY2qr4tz1U0iOSZmf/vie0R52tRdKtkl6XNHcL6yXp2uzP4zlJB27zh0ZEp3qQbnn9F+DvgB2BZ4HRjbb5J+CG7PmpwE/bu+42OOajgF7Z8/OKcMzZdn2Ax4AngZr2rrsN/p5HArOBD2Svd23vutvgmG8CzsuejwZeae+6t/GY/x44EJi7hfUTgAcAAYcCT23rZ3bGFsHBwMKIWBQR7wF3ASc32uZk4EfZ858DH5M69Kz2TR5zRDwSEauzl0+SZozryKr5ewa4EvgWsLYti8tJNcd8NjA1It4CiIjX27jG1lbNMQfQN3veD/hrG9bX6iLiMdL8LFtyMvDjSJ4Edpa0+7Z8ZmcMgkHAkrLXddmyittExAbgbaB/m1SXj2qOudxZpN8oOrImjzlrMg+JiF+3ZWE5qubv+UPAhyQ9LulJSSe0WXX5qOaYLwdOl1RHmv/ky21TWrtp7v/3JuU6MY1tfySdDtQAR7R3LXmStAPwXeDMdi6lrXUldQ8dSWr1PSZpbESsaM+icjYZ+GFEfEfSYaRZD8dExKb2Lqyj6IwtgqXAkLLXg7NlFbeR1JXUnFzeJtXlo5pjRtIxwKXApIhY10a15aWpY+4DjAEelfQKqS91WgcfMK7m77kOmBYR6yPiZeBFUjB0VNUc81nAzwAi4gmgB+nmbJ1VVf/fm6MzBsHTwEhJe0rakTQYPK3RNtOAf8yefwZ4OLJRmA6qyWOWdABwIykEOnq/MTRxzBHxdkQMiIjhETGcNC4yKSJq26fcVlHNv+17Sa0BJA0gdRUtasMaW1s1x7wY+BiApH1IQbCsTatsW9OAM7Kzhw4F3o6Iv23LDjtd11BEbJA0BZhBOuPg1oiYJ+kKoDYipgG3kJqPC0mDMqe2X8Xbrspj/jbQG7g7GxdfHBGT2q3obVTlMXcqVR7zDOA4SfOBjcBFEdFhW7tVHvOFwM2SvkIaOD6zI/9iJ+lOUpgPyMY9vgF0A4iIG0jjIBOAhcBq4Avb/Jkd+M/LzMxaQWfsGjIzs2ZwEJiZFZyDwMys4BwEZmYF5yAwMys4B4FZBZI2Spojaa6k+yXt3Mr7fyU7zx9Jq1pz32bN5SAwq2xNRIyLiDGka03Ob++CzPLiIDBr2hNkN/WStJek30iaJekPkkZlyz8o6ZeSns0e47Pl92bbzpN0Tjseg9kWdbori81ak6QupNsX3JItugk4NyJeknQIcD1wNHAt8PuI+GT2nt7Z9l+MiDcl9QSelnRPR77S1zonB4FZZT0lzSG1BBYAv5PUGxhP/W06ALpnP48GzgCIiI2kW5sDXCDpk9nzIaQbwDkIbLviIDCrbE1EjJPUi3Sfm/OBHwIrImJcNTuQdCRwDHBYRKyW9Cjphmhm2xWPEZhtRTar2wWkG5utBl6WdAq8P3fs/tmmD5GmAEVSF0n9SLc3fysLgVGkW2GbbXccBGZNiIjZwHOkCVBOA86S9Cwwj/ppE/8ZOErS88As0ty5vwG6SloAXE26FbbZdsd3HzUzKzi3CMzMCs5BYGZWcA4CM7OCcxCYmRWcg8DMrOAcBGZmBecgMDMruP8Pzil+z3ONyvIAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SML AUPR score: 0.6173494618913611\n"
     ]
    }
   ],
   "source": [
    "precision, recall, aupr_score = aupr(test_y, ood_scores)\n",
    "\n",
    "plot_precision_recall_curve(precision, recall)\n",
    "print('SML AUPR score:', aupr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SML fpr95 score: 0.8168\n"
     ]
    }
   ],
   "source": [
    "fpr95 = fpr_at_95_tpr(test_y, ood_scores)\n",
    "print('SML fpr95 score:', fpr95)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}