ood_cifar100_vs_cifar10.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Base settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import time\n",
    "from pprint import pprint\n",
    "from pathlib import Path\n",
    "from collections import OrderedDict\n",
    "from tqdm import tqdm\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create ConfigDict from config yaml file\n",
    "import yaml\n",
    "from ml_collections import config_dict\n",
    "\n",
    "config_path = Path('configs') / 'deit_tiny-pretrained-cifar100.yaml'\n",
    "with config_path.open('r') as f:\n",
    "    config = yaml.safe_load(f)\n",
    "    config = config_dict.ConfigDict(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# frequently used variables\n",
    "model_name = config.model.name\n",
    "patch_size = config.model.patch_size\n",
    "summary = config.summary\n",
    "\n",
    "# log directories\n",
    "log_root = Path('./logs') / model_name / summary\n",
    "checkpoint_path = log_root / 'checkpoints'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize ViT model and load pretrained weights\n",
    "from ood_with_vit.models.vit import ViT\n",
    "\n",
    "def initialize_vit_model(config, verbose=0):\n",
    "    n_class = config.dataset.n_class\n",
    "    if config.model.pretrained:\n",
    "        model = torch.hub.load(\n",
    "            repo_or_dir=config.model.repo,\n",
    "            model=config.model.pretrained_model,\n",
    "            pretrained=False,\n",
    "        )\n",
    "        model.head = nn.Linear(model.head.in_features, n_class)\n",
    "    else:\n",
    "        model = ViT(\n",
    "            image_size=config.model.img_size,\n",
    "            patch_size=config.model.patch_size,\n",
    "            num_classes=n_class,\n",
    "            dim=config.model.dim_head,\n",
    "            depth=config.model.depth,\n",
    "            heads=config.model.n_heads,\n",
    "            mlp_dim=config.model.dim_mlp,\n",
    "            dropout=config.model.dropout,\n",
    "            emb_dropout=config.model.emb_dropout,\n",
    "            visualize=True,\n",
    "        )\n",
    "\n",
    "    model = model.to(device=device)\n",
    "    if verbose:\n",
    "        print(model)\n",
    "\n",
    "    checkpoint = torch.load(checkpoint_path / f'{summary}_best.pt')\n",
    "\n",
    "    state_dict = checkpoint['model_state_dict']\n",
    "    trimmed_keys = []\n",
    "    for key in state_dict.keys():\n",
    "        # remove prefix 'module.' for each key (in case of DataParallel)\n",
    "        trimmed_keys.append(key[7:])\n",
    "    trimmed_state_dict = OrderedDict(list(zip(trimmed_keys, state_dict.values())))\n",
    "\n",
    "    model.load_state_dict(trimmed_state_dict)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'apple': 0, 'aquarium_fish': 1, 'baby': 2, 'bear': 3, 'beaver': 4, 'bed': 5, 'bee': 6, 'beetle': 7, 'bicycle': 8, 'bottle': 9, 'bowl': 10, 'boy': 11, 'bridge': 12, 'bus': 13, 'butterfly': 14, 'camel': 15, 'can': 16, 'castle': 17, 'caterpillar': 18, 'cattle': 19, 'chair': 20, 'chimpanzee': 21, 'clock': 22, 'cloud': 23, 'cockroach': 24, 'couch': 25, 'crab': 26, 'crocodile': 27, 'cup': 28, 'dinosaur': 29, 'dolphin': 30, 'elephant': 31, 'flatfish': 32, 'forest': 33, 'fox': 34, 'girl': 35, 'hamster': 36, 'house': 37, 'kangaroo': 38, 'keyboard': 39, 'lamp': 40, 'lawn_mower': 41, 'leopard': 42, 'lion': 43, 'lizard': 44, 'lobster': 45, 'man': 46, 'maple_tree': 47, 'motorcycle': 48, 'mountain': 49, 'mouse': 50, 'mushroom': 51, 'oak_tree': 52, 'orange': 53, 'orchid': 54, 'otter': 55, 'palm_tree': 56, 'pear': 57, 'pickup_truck': 58, 'pine_tree': 59, 'plain': 60, 'plate': 61, 'poppy': 62, 'porcupine': 63, 'possum': 64, 'rabbit': 65, 'raccoon': 66, 'ray': 67, 'road': 68, 'rocket': 69, 'rose': 70, 'sea': 71, 'seal': 72, 'shark': 73, 'shrew': 74, 'skunk': 75, 'skyscraper': 76, 'snail': 77, 'snake': 78, 'spider': 79, 'squirrel': 80, 'streetcar': 81, 'sunflower': 82, 'sweet_pepper': 83, 'table': 84, 'tank': 85, 'telephone': 86, 'television': 87, 'tiger': 88, 'tractor': 89, 'train': 90, 'trout': 91, 'tulip': 92, 'turtle': 93, 'wardrobe': 94, 'whale': 95, 'willow_tree': 96, 'wolf': 97, 'woman': 98, 'worm': 99}\n",
      "{'airplane': 0, 'automobile': 1, 'bird': 2, 'cat': 3, 'deer': 4, 'dog': 5, 'frog': 6, 'horse': 7, 'ship': 8, 'truck': 9}\n"
     ]
    }
   ],
   "source": [
    "from torchvision.datasets import CIFAR10, CIFAR100\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "\n",
    "dataset_mean, dataset_std = config.dataset.mean, config.dataset.std\n",
    "dataset_root = config.dataset.root\n",
    "img_size = config.model.img_size\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.Resize(img_size),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(dataset_mean, dataset_std),\n",
    "])      \n",
    "\n",
    "id_dataset = CIFAR100(\n",
    "    root=dataset_root, \n",
    "    train=False, \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "id_test_dataloader = DataLoader(\n",
    "    dataset=id_dataset, \n",
    "    batch_size=config.eval.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "id_dataset_name = 'cifar100'\n",
    "\n",
    "ood_dataset = CIFAR10(\n",
    "    root=dataset_root, \n",
    "    train=False, \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "ood_test_dataloader = DataLoader(\n",
    "    dataset=ood_dataset, \n",
    "    batch_size=config.eval.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "ood_dataset_name = 'cifar10'\n",
    "\n",
    "print(id_dataset.class_to_idx)\n",
    "print(ood_dataset.class_to_idx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Utility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set seeds\n",
    "import random\n",
    "import numpy as np\n",
    "\n",
    "def set_seed(seed):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "seed = 1234"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.visualizer.feature_extractor import FeatureExtractor\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "# add hooks for feature extraction\n",
    "feature_extractor = FeatureExtractor(\n",
    "    model=model,\n",
    "    layer_name=config.model.layer_name.penultimate,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:00<00:00, 93.00it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.99it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.62it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 107.79it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 108.44it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 108.30it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 108.92it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 108.55it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 108.13it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 108.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:00<00:00, 107.55it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 108.57it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 108.23it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 108.13it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 108.93it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 102.45it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 104.14it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.72it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 104.07it/s]\n",
      "100%|██████████| 20/20 [00:00<00:00, 103.97it/s]\n"
     ]
    }
   ],
   "source": [
    "# gather penultimate features\n",
    "from ood_with_vit.utils import compute_penultimate_features\n",
    "\n",
    "set_seed(seed)\n",
    "num_samples = 20\n",
    "num_class = 10\n",
    "\n",
    "print('processing in-distribution samples...')\n",
    "X_in, y_in = [], []\n",
    "in_img_paths = []\n",
    "cls_indices = random.sample(range(len(id_dataset.classes)), num_class)\n",
    "for cls_idx in cls_indices:\n",
    "    img_indices = np.where(np.array(id_dataset.targets) == cls_idx)[0]\n",
    "    img_indices = random.sample(list(img_indices), num_samples)\n",
    "    for i in tqdm(img_indices):\n",
    "        img, _ = id_dataset[i]\n",
    "        img = img.to(device)\n",
    "        penultimate_features = compute_penultimate_features(\n",
    "            config=config, \n",
    "            model=model, \n",
    "            imgs=img.unsqueeze(0),\n",
    "            feature_extractor=feature_extractor,    \n",
    "        )\n",
    "        X_in.append(penultimate_features.squeeze().numpy())\n",
    "        y_in.append((cls_idx, id_dataset.classes[cls_idx]))\n",
    "        img_path = f'./assets/{id_dataset_name}/test/{i:05d}.jpg'\n",
    "        # img_path = f'/assets/{id_dataset_name}/test/{i:05d}.jpg'\n",
    "        in_img_paths.append(img_path)\n",
    "        \n",
    "print('processing out-of-distribution samples...')\n",
    "X_out, y_out = [], []\n",
    "out_img_paths = []\n",
    "cls_indices = random.sample(range(len(ood_dataset.classes)), num_class)\n",
    "for cls_idx in cls_indices:\n",
    "    img_indices = np.where(np.array(ood_dataset.targets) == cls_idx)[0]\n",
    "    img_indices = random.sample(list(img_indices), num_samples)\n",
    "    for i in tqdm(img_indices):\n",
    "        img, _ = ood_dataset[i]\n",
    "        img = img.to(device)\n",
    "        penultimate_features = compute_penultimate_features(\n",
    "            config=config, \n",
    "            model=model, \n",
    "            imgs=img.unsqueeze(0),\n",
    "            feature_extractor=feature_extractor,    \n",
    "        )\n",
    "        X_out.append(penultimate_features.squeeze().numpy())\n",
    "        y_out.append((cls_idx, ood_dataset.classes[cls_idx]))\n",
    "        img_path = f'./assets/{ood_dataset_name}/test/{i:05d}.jpg'\n",
    "        # img_path = f'/assets/{ood_dataset_name}/test/{i:05d}.jpg'\n",
    "        out_img_paths.append(img_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature visualization by T-SNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ipywidgets import HTML, VBox\n",
    "\n",
    "def interactive_plot(partitioned_info, fig, event=\"click\") :\n",
    "    \"\"\"\n",
    "    Make a plot react on hover or click of a data point and update a HTML preview below it.    \n",
    "    \"\"\"\n",
    "    html = HTML(\"\")\n",
    "    fig = go.FigureWidget(data=fig.data, layout=fig.layout)\n",
    "\n",
    "    if event == \"hover\" :\n",
    "        for data in fig.data:\n",
    "            def update(trace, points, state):\n",
    "                if len(points.point_inds) == 0:\n",
    "                    return\n",
    "                trace_idx, point_idx = points.trace_index, points.point_inds[0]\n",
    "                df_row = partitioned_info[trace_idx].iloc[point_idx]\n",
    "                img_url = df_row['img']\n",
    "                label = df_row['label']\n",
    "                html.value = f'\\\n",
    "                    <body>\\\n",
    "                        <img src=\"{img_url}\" width=\"240\"><br>\\\n",
    "                        <b>Image: {img_url}</b><br>\\\n",
    "                        <b>Label: {label}</b>\\\n",
    "                    </body>'\n",
    "            data.on_hover(update)\n",
    "    else :\n",
    "        for data in fig.data:\n",
    "            def update(trace, points, state):\n",
    "                if len(points.point_inds) == 0:\n",
    "                    return\n",
    "                trace_idx, point_idx = points.trace_index, points.point_inds[0]\n",
    "                df_row = partitioned_info[trace_idx].iloc[point_idx]\n",
    "                img_url = df_row['img']\n",
    "                label = df_row['label']\n",
    "                html.value = f'\\\n",
    "                    <body>\\\n",
    "                        <img src=\"{img_url}\" width=\"240\"><br>\\\n",
    "                        <b>Image: {img_url}</b><br>\\\n",
    "                        <b>Label: {label}</b>\\\n",
    "                    </body>'\n",
    "            data.on_click(update)\n",
    "\n",
    "    return VBox([fig, html])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:780: FutureWarning: The default initialization in TSNE will change from 'random' to 'pca' in 1.2.\n",
      "  warnings.warn(\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:790: FutureWarning: The default learning rate in TSNE will change from 200.0 to 'auto' in 1.2.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b50e12ad13b43b7b82d34c2ffaf782d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/cifar100/test/01191.jpg'],\n",
       "       …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "\n",
    "tsne = TSNE(n_components=2, random_state=seed)\n",
    "X_tsne = tsne.fit_transform(X_in)\n",
    "\n",
    "df_embed = pd.DataFrame(X_tsne)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_in)\n",
    "df_embed = df_embed.assign(img=in_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_in:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "    \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained, CIFAR100 finetuned ViT CIFAR100 T-SNE',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:780: FutureWarning:\n",
      "\n",
      "The default initialization in TSNE will change from 'random' to 'pca' in 1.2.\n",
      "\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:790: FutureWarning:\n",
      "\n",
      "The default learning rate in TSNE will change from 200.0 to 'auto' in 1.2.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "337d272eb034450c9560284fd599dc14",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/cifar10/test/02593.jpg'],\n",
       "        …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tsne = TSNE(n_components=2, random_state=seed)\n",
    "X_tsne = tsne.fit_transform(X_out)\n",
    "\n",
    "df_embed = pd.DataFrame(X_tsne)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_out)\n",
    "df_embed = df_embed.assign(img=out_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_out:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "\n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained, CIFAR100 finetuned ViT CIFAR10 T-SNE',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature visualization by PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "baa66c5eb1a343d4beb94786fdf56547",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/cifar100/test/01191.jpg'],\n",
       "       …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "\n",
    "pca = PCA()\n",
    "components = pca.fit_transform(X_in)\n",
    "\n",
    "df_embed = pd.DataFrame(components)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_in)\n",
    "df_embed = df_embed.assign(img=in_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_in:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "        \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained, CIFAR100 finetuned ViT CIFAR100 PCA',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9b7362b2c95468f9833e193f122dbf2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/cifar10/test/02593.jpg'],\n",
       "        …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pca = PCA()\n",
    "components = pca.fit_transform(X_out)\n",
    "\n",
    "df_embed = pd.DataFrame(components)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_out)\n",
    "df_embed = df_embed.assign(img=out_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_out:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "        \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained, CIFAR100 finetuned ViT CIFAR10 PCA',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OOD detection baselines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ood_with_vit.utils import compute_ood_scores\n",
    "from ood_with_vit.utils.ood_metrics import auroc, aupr, fpr_at_95_tpr\n",
    "from ood_with_vit.utils.visualization import plot_roc_curve, plot_precision_recall_curve"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### OOD detection by MSP (Maximum over Softmax Probability)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.metrics.msp import MSP\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "msp = MSP(config, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 16.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 16.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id mean score: -0.9059253\n",
      "ood mean score: -0.70739657\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "test_y, ood_scores, id_ood_scores, ood_ood_scores = compute_ood_scores(\n",
    "    metric=msp,\n",
    "    in_dist_dataloader=id_test_dataloader,\n",
    "    out_of_dist_dataloader=ood_test_dataloader,    \n",
    ")\n",
    "print('id mean score:', np.mean(id_ood_scores))\n",
    "print('ood mean score:', np.mean(ood_ood_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABCIUlEQVR4nO3dd5gTVffA8e9hl96rhd47gqwgIk2kimIXC4qivHSxvpbXxk9RVKwUQVEsWFEQBUFFETu9I4JIWQSBpbOULef3x53VuG4JsNnZJOfzPHmSzExmzkySOTP3ztwrqooxxpjolc/vAIwxxvjLEoExxkQ5SwTGGBPlLBEYY0yUs0RgjDFRzhKBMcZEOUsEEU5EVolIe7/jyCtE5D4RecWnZU8SkUf9WHZOE5FrReTzE/ys/SbzGEsEuUhENorIYRE5KCLbvR1DsVAuU1UbqurcUC4jjYgUFJHHRWSzt57rROQuEZHcWH4G8bQXkfjAYao6QlVvDtHyRESGishKETkkIvEi8oGINA7F8k6UiDwsIm+dzDxUdbKqdg5iWf9Kfif6mxSRAl7s67ztu1FEXhWRasc7L/NPlghy34WqWgxoCjQD7vU3nOMnIrGZjPoA6Ah0B4oDvYF+wPMhiEFEJK/9fp8HbgWGAmWAOsA04IKcXlAW30HI+bjsKcBFwDVASeAMYBHuN3dc/Nx+eZKq2iOXHsBG4PyA908CMwLenw38AOwFlgHtA8aVAV4D/gD2ANMCxvUAlnqf+wFokn6ZwOnAYaBMwLhmwC4gv/f+JmCNN//ZQNWAaRUYBKwDfs9g3ToCR4DK6Ya3BFKAWt77ucDjwHxgP/Bxupiy2gZzgceA7711qQXc6MV8ANgA/Mebtqg3TSpw0HucDjwMvOVNU81brxuAzd62uD9geYWB173tsQa4G4jP5Lut7a1niyy+/0nAGGCGF+/PQM2A8c8DW7ztsghoEzDuYdyO8C1v/M1AC+BHb1ttA0YDBQI+0xD4AtgN/AncB3QFjgFJ3jZZ5k1bEpjozWcr8CgQ443r423zZ4EEb1wf4DtvvHjjdnixrQAa4Q4CkrzlHQQ+Sf8/AGK8uH7ztski0v2GvOnO977Pf43L4v+V0Xfd1/uu5wGfAYPTzWMZcKn3ul7A9lsLXOn3PiRk+ya/A4imR7o/QCXvD/O8976i9yfrjjtT6+S9L++NnwG8B5QG8gPtvOHNvD9gS+9PdYO3nIIZLPMr4JaAeJ4CXvJe9wTWA/WBWOB/wA8B06r3pygDFM5g3Z4AvslkvTfx9w56rrejaYTbWX8Y8GfNbhvM9f7EDb0Y8+OOtmvidkbtgETgTG/69qTbcWeyc3gZt9M/AzgK1A9cJ2+bVwKWp59fwHz7A5uy+f4neevTwot/MvBuwPjrgLLeuDuA7UChgLiTgIu9bVMYaI5LnLHeuqwBhnnTF8ft1O8ACnnvW6bfBgHLngqM976TCrhEnfad9QGSgSHesgrzz0TQBbcDL+V9D/WB0wLW+dEs/gd34f4Hdb3PngGUPZ7fV0bzzeK7fsNbx8LA9cD3AdM3wCXVgt40W3AHGrH8fdDUwO/9SCgeee3UOhpME5EDuB/ZDuAhb/h1wExVnamqqar6BbAQ6C4ipwHdgP6qukdVk1T1G+9z/YDxqvqzqqao6uu4ndnZGSz7beBqcEUrQC9vGLgd2eOqukZVk4ERQFMRqRrw+cdVdbeqHs5g3uVwO56MbPPGp3lTVVeq6iHgAeBKEYnJahsEfHaSqq5S1WRvO8xQ1d/U+Qb4HGiTSRyZeURVD6vqMtwR4Rne8CuBEd42jwdeyGIeZbNY/0BTVXW+t40n44oIAVDVt1Q1wVu3UbgdUt2Az/6oqtO8bXNYVRep6k/e9BtxO/J23rQ9gO2qOkpVj6jqAVX9OaOAROQU3DYepqqHVHUH7gi/V8Bkf6jqi96y0n//SbhEUw8Q7zcUzLYAd2bzP1Vd632Hy1Q1IYPpgt2+2XnYW8fDuOQX+Bu/FvhIVY/itt9GVX3NW+cluIOWK3IghjzHEkHuu1hVi+OOVuvx9w6yKnCFiOxNewDnAqcBlYHdqrong/lVBe5I97nKuGKQ9D4EWnmJpS2u2OTbgPk8HzCP3bgjtIoBn9+SxXrt8mLNyGne+Izmswl3ZF+OrLdBhjGISDcR+UlEdnvTd+efSScY2wNeJwJpFfinp1teVuufQObrH8yyEJE7RWSNiOzz1qUk/1yX9OteR0Q+9S482I9L3mnTV8YVtwSjKu472Baw3cfjzgwyXHYgVf0KVyw1BtghIhNEpESQyw42zmC3b3b+Wg9VPYA7005LeFfjkjO4bdIy3W/xWuDUHIghz7FE4BPv6HUS8LQ3aAvuSLlUwKOoqj7hjSsjIqUymNUW4LF0nyuiqu9ksMw9uCPmq3AVbu+qqgbM5z/p5lNYVX8InEUWq/Ql7o9TOXCgiLTE/dm/ChgcOE0V3BHlrmy2wb9iEJGCuOT2NHCKqpYCZuISWHbxBmMbrkgoo7jTmwNUEpG4E1mQiLTB1UFcCZT21mUff68L/Ht9xgG/ALVVtQSurD1t+i1AjUwWl34+W3BnkeUCtnsJVW2YxWf+OUPVF1S1Oa54pQ6uyCfbz3nLrpnNNOB+Xy1EpFIW0xwCigS8z2innT6ed4CrRaQVrgjt64C4vkn3WyymqgOCiDXsWCLw13NAJxE5A1cJeKGIdBGRGBEp5F3+WMk7zf4MGCsipUUkv4i09ebxMtBfRFp6V9IUFZELRKR4Jst8G1c2ejl/FwsBvATcKyINAUSkpIgEfRqsql/idoYfikhDbx3O9tZrnKquC5j8OhFpICJFgOHAFFVNyWobZLLYArjik51Asoh0AwIvafwTKCsiJYNdj3Tex22T0iJSERic2YTe+o0F3vFiLuDF30tE7gliWcVx5fA7gVgReRDI7qi6OK5y9qCI1AMCd1KfAqeJyDBxl/UW95IyuO1SLe2qK+/39TkwSkRKiEg+EakpIu0Igoic5f3+8uN2xkdwZ5tpy8osIQG8AvyfiNT2fr9NRKRs+om839cXwFQRaS4isd469ReRm7zJlgK9vP9HHO43np2ZuKP/4cB7qpoW96dAHRHp7c0vv7ee9YOYZ9ixROAjVd2Jq7x6UFW34Cps78PtDLbgjqrSvqPeuCPnX3B1C8O8eSwEbsGdmu/BVfj2yWKx03FXuGz3ysTTYpkKjATe9YoZVuLqJY7HZbgjqlm4q0Tewl2JMiTddG/izoa2447ChnoxZLcN/sE7tR+K22HvwZ3lTA8Y/wvuiG+Dd3qfUXFZVoYD8cDvuCPSKbgj58wM5e8ikr24Io9LgE+CWNZs3Hb7FVdcdoSsi6IA7sSt8wHcAcF7aSO8bdMJuBC3ndcBHbzRH3jPCSKy2Ht9PS6xrsZtyykEXxRTwlv+Hi/2BNyFCOC+/wbe9p+WwWefwX1/n+OS2kRcRW5GLsftuN/DnS2tBOJw3w24+qaaXhyP8M8DnQx59QEf4a5Kejtg+AHcQUUv3JV623H/j4LZzTMcyd8lA8aEnojMxV3J4cvdvSdDRAYAvVQ1qCNlY8KFnREYkwkROU1EWntFJXVxl2JO9TsuY3Ka3V1nTOYK4K6eqY4r6nkXVw9gTESxoiFjjIlyVjRkjDFRLuyKhsqVK6fVqlXzOwxjjAkrixYt2qWq5TMaF3aJoFq1aixcuNDvMIwxJqyIyKbMxlnRkDHGRDlLBMYYE+UsERhjTJQLuzqCjCQlJREfH8+RI0f8DiVkChUqRKVKlcifP7/foRhjIkxEJIL4+HiKFy9OtWrVEH+6xw0pVSUhIYH4+HiqV6/udzjGmAgTsqIhcZ1K7xCRlZmMFxF5QUTWi8hyETnzRJd15MgRypYtG5FJAEBEKFu2bESf8Rhj/BPKOoJJuP5RM9MN1wpmbVwvW+NOZmGRmgTSRPr6GWP8E7KiIVWdJyLVspikJ/CG1zHKTyJSSkROO44u7owxJrRSUyEpCZKT3ePwYTcsJSX756NH//5s2vOxY7B3b8bDDx+GTJr8OcQxdpJItQt7w1ln5fhq+llHUJF/trce7w37VyIQkX64swaqVKmSK8Edr5iYGBo3bkxycjLVq1fnzTffpFSpUgCsWrWKIUOGsHXrVlJTU7n++uv53//+99dR/meffcYDDzxAYmIiBQsW5LzzzmPUqFE+ro0xeZwqJCTAtm2wY4fbkSYmwoED7nHwoHuflOTGJSW5YTt2/L2DTkqCVaugQgW34w7cOR886D6X222xZXDm/1U15ZYLoeRRWLiwHvkiLBEETVUnABMA4uLi8mQreYULF2bp0qUA3HDDDYwZM4b777+fw4cPc9FFFzFu3Dg6d+5MYmIil112GWPHjmXQoEGsXLmSwYMHM2PGDOrVq0dKSgoTJkzwd2WMyQ0pKbB/v3vs2QO7drmd+y+/QP78sHIlFCoEa9dCgQKwc6fbyR86BPv2uZ12VvLlc/PJn999vkABOOUUKFrUDStaFM4+2x2h16nz97SxsVC4MBQp4l7Hxv49fPduqFzZzTsmJuvntGUGfj5/fihdOuPhhQr9IxHsPbKXuz6/i1eWvEKtMrV49sJXyFctNF1h+JkItvLPPmArecPCXqtWrVi+fDkAb7/9Nq1bt6ZzZ9eDYpEiRRg9ejTt27dn0KBBPPnkk9x///3Uq1cPcGcWAwZEZLeoJtIdPQrbt7vHzp1uh370qDsK37cPFi92xSaHD7ud/v79Wc9PxB2R16njdtatW0OJEm4HXqqUe1SrBsWKuZ3vqae61yVKuOdChUK/ziGSkprCORPPYW3CWu4+524ebv8whfNn1nHbyfMzEUwHBovIu0BLYF+O1A8MGwbekXmOadoUnnsuqElTUlKYM2cOffv2BVyxUPPmzf8xTc2aNTl48CD79+9n5cqV3HHHHTkbrzE5LTUVNm1yR+dbt8Kff8KiRW6nvnkzxMe7nX1GSpZ0O+cKFdzR/NlnQ5kyUK6cK4apWNGNK1/e7dxLl3bvC0Zkr5BZSkhMoEzhMsTki+Gx8x6jcsnKxJ0eF/LlhiwRiMg7QHugnIjEAw8B+QFU9SVc36PdcX3sJgI3hiqW3HD48GGaNm3K1q1bqV+/Pp06dfI7JGOOjyqsWQPz57sj9g0bYN06t6P/9dd/T1+ihNuRd+0K7du7I/JTT4XTToOyZd2OvlKlsD4yzy2qyuQVk7l11q080fEJbml+C5fUvyTXlh/Kq4auzma8AoNyfMFBHrnntLQ6gsTERLp06cKYMWMYOnQoDRo0YN68ef+YdsOGDRQrVowSJUrQsGFDFi1axBlnnOFL3CaKJSfDggUwbRp88w2sX+/K6NMULgx160K9enDuuVC/PlSvDs2auR1+kSK+hR5JtuzbQv8Z/Zm5biZnVzqb1lVa534QqhpWj+bNm2t6q1ev/tew3Fa0aNG/Xi9evFirVKmiSUlJmpiYqNWrV9cvvvhCVVUTExP1ggsu0BdeeEFVVZctW6Y1a9bUtWvXqqpqSkqKjhs3LsNl5IX1NGHo6FHVhQtVX3tNdcAA1VatVAsWVHXnAO6RL5/qBReoPvus6qxZqlu2qKam+h15xHt7+dtafERxLfJYEX3ux+c0OSU5ZMsCFmom+9WwuGoo3DRr1owmTZrwzjvv0Lt3bz7++GOGDBnCoEGDSElJoXfv3gwePBiAJk2a8Nxzz3H11VeTmJiIiNCjRw+f18CErf37YdYsd4T/++/uiH/Xrr/HFygALVvCFVe4MvjWrV2xjjVd4ovShUvTslJLJvSYQPXS/n0HYddncVxcnKbvmGbNmjXUr1/fp4hyT7SspwnSkSOuTH/aNPjyS1eB+9tvf48vUQJatHCXTDZtCh06QIMGrsjH+CI5NZlnf3yWYynHuL/t/YArlcmNlgNEZJGqZljzbGcExoSDnTvdVTtz5rjK22++cTdDpcmf311K2a8ftGsHHTu6BGDyjGXbl9F3el8WbVvElQ2v/CsB5IXmYywRGJPXJCW5Hf5337mj/Z073bX4gTp2hEsvdZdbtmzpinhMnnQ0+SiPznuUJ75/gjKFy/DBFR9wWf3L8kQCSBMxiSC3Tq/8Em5FeCZIqu5If8kSd43+p5/Cjz+66/bBXYoZGwu33uqu2qlb1+30rV+KsLFu9zpGfj+SaxpfwzOdn6FskbJ+h/QvEZEIChUqREJCQsQ2Ra1efwSF7HrsyLBtG3z2Gbz1liveCTzar1gRunWDq6+GCy5wR/wm7Bw8dpCPf/mYa5tcS6MKjfhl8C/UKF3D77AyFRGJoFKlSsTHx7Nz506/QwmZtB7KTBhatszdhbtsGUyZAn/84YbHxrpr8u+4A2rUgLZt3R21Jqx98dsX9Pu0H5v2buLM086kfvn6eToJQIQkgvz581vPXSZv2bIFJk6Ed975+67c2Fho3hwuuwx69nRX8eSzbsMjxZ7De7jz8zt5demr1Clbh2/6fEP98uFxlV9EJAJj8oRffoHPP4dvv4UPP3Tl/0WLwm23wSWXuErdAgX8jtKEQEpqCq1fbc2vCb9y77n38mC7BykUGz5FuZYIjDlRCQmwcKEr73/3XXcdf5revWHIEIiLy7CNeRMZdiXu+quRuBEdR1ClZBXOPO2Ee931jSUCY45HairMnevatPrkEzcsJsYV8wwd6lrWbNPGruqJcKrKm8vfZNisYTxx/hP0a96Pi+td7HdYJ8wSgTHBWL0a7rsPPv7472G9e7sEcNFFrrVNExU27d3Efz79D7N/m805lc+hbdW2fod00iwRGJOZ+Hh3iefbb8OKFa6Ip0oVuOkm6N/f7tyNQm8tf4sBMwagqrzY7UUGnjWQfBL+Ff6WCIxJb+NGuOceeO89975ePXjiCejVC6pW9TU046/yRcrTunJrxvcYT9VSkfNbsERgDLi+aB9/HL7/3t3ZC3DllfDf/8KZ4Vf5Z3JGUkoSo34cRVJKEg+0e4AutbrQuWbniLtx1RKBiV67d8Ps2TB5Msyc6S73BBg4EAYPdk06mKi1ZNsS+k7vy5LtS+jVqFeeaiQup1kiMNFn5053pP/aa+59uXJw3XWu3L9VK7vcM8odST7C8G+G8+T3T1KuSDk+vPJDLq1/qd9hhZQlAhMdVF2LnqNGuZu+UlPh8std8c/FF9vlnuYv63ev5+kfnub6M65nVOdRlC5c2u+QQs4SgYlsqvD++9Cnj+vIBdzO/+67XXMPxuAaiZu6Ziq9z+hNowqNWDt4ra89huU2SwQmcs2dCwMGuKYfRFzzzTNmQMmSfkdm8pDZ62fT79N+bNm3hbjT46hfvn5UJQGA8L8A1pj09uyBM85wN3v98otryz8x0XX0YknAeBISE7hh2g10ndyVIvmL8O2N34ZNI3E5zc4ITORISnIdu/Tt65LBqae6BuBq1fI7MpPHpDUSt373eu5vcz//a/u/sGokLqdZIjCRYetWOO+8v5t8njbNNfVsTICdh3ZStkhZYvLFMPL8kVQtVZWmpzb1OyzfWdGQCW+pqfDII1CnjuvUffhwlxQsCZgAqsprS16jzug6vLzoZQB61utpScBjZwQmfCUkQKdOrr/f1q3hlVdccxDGBNi4dyP9PunHFxu+oE2VNnSo3sHvkPIcOyMw4SchAZ5/3nXsvmQJPPqoqwi2JGDSeXPZmzQa24gf439kbPexzO0zlzpl6/gdVp5jZwQmfOzc6foBGDHCvW/UCB57zDUDbUwGTil2Cm2rtuWlHi9RpWQVv8PJsywRmPAwa5a7EezAAWjQAB54wL23Pn9NgKSUJJ78/klSNIUH2z1I55qd6Vyzs99h5XmWCEzelprq2gB67TVITnY3ibVr53dUJg9avG0xN318E8v+XMY1ja/5q5E4kz07nDJ515Il0LYtvPyy6/7xt98sCZh/OZx0mHu+vIcWL7fgz0N/MvWqqUy+dLIlgeMQ0kQgIl1FZK2IrBeRezIYX0VEvhaRJSKyXES6hzIeEyZ++w169HD9APzwA9x2G3z1FdSo4XdkJg/asGcDz/z4DH2a9mH1wNVh3XewX0KWCEQkBhgDdAMaAFeLSIN0k/0PeF9VmwG9gLGhiseEiWefhdq1XZtAgwfD9u3wzDN+R2XymP1H9zNp6SQAGlZoyLoh63jloleioqXQUAhlHUELYL2qbgAQkXeBnsDqgGkUKOG9Lgn8EcJ4TF62YQP06+eaiq5QAb7+2lUKG5POzHUz6f9pf7Ye2ErLii2pX75+RHUb6YdQFg1VBLYEvI/3hgV6GLhOROKBmcCQjGYkIv1EZKGILNy5c2coYjV+UYU333TFQHPmwF13wbp1lgTMv+xK3EXvqb254O0LKF6wON/f9H3UNhKX0/y+auhqYJKqjhKRVsCbItJIVVMDJ1LVCcAEgLi4OPUhThMKv//uOodZvBgqVYIvv4S4OL+jMnlQWiNxG/Zs4MG2D3Jfm/soGFvQ77AiRigTwVagcsD7St6wQH2BrgCq+qOIFALKATtCGJfJC376yTUSd/gw3HILjB4NBQr4HZXJY/48+Cfli5YnJl8MT3d6mqqlqtLklCZ+hxVxQlk0tACoLSLVRaQArjJ4erppNgMdAUSkPlAIsLKfSDd2rGsbqEABmD8fJkywJGD+QVWZuHgidUfXZcKiCQBcWPdCSwIhErIzAlVNFpHBwGwgBnhVVVeJyHBgoapOB+4AXhaR23AVx31U1Yp+Itnbb8OgQe7KoDlzoHLl7D9josqGPRu45ZNb+Or3r2hXtR3n1zjf75AinoTbfjcuLk4XLlzodxjmREyf7pqHjo2F3buheHG/IzJ5zOtLX2fgzIHESAxPdXqKW5rfQj6x+15zgogsUtUMK+H8riw20eKFF1yXkWXKuF7ELAmYDJxe/HTOq34e4y4YR6USlfwOJ2rYGYEJvc8+g0sugfLlYdUqKFEi+8+YqHAs5RhPfPcEqZrKw+0f9juciJbVGYGdc5nQuvNO6N7d7fy/+caSgPnLgq0LaD6hOQ/NfYgNezYQbgelkcQSgQmdW2+FUaNcxfDy5dZWkAEgMSmROz+/k7Mnns2ew3uY3ms6b1zyhjUS5yOrIzChcc89rl6gZUv49lvIn9/viEwe8fue33lx/ovccuYtjDx/JCULlfQ7pKhnicDkLFXo3RsmT4aaNV1xkCWBqLfvyD4+WvMRNza7kYYVGrJ+yHoql7RLh/MKKxoyOWfzZtdfwOTJ7jLRlSuhoDUDEO1m/DqDhmMbcvMnN/PLrl8ALAnkMZYITM6Ij4czznCdyI8cCVOnQqFCfkdlfLTz0E6u/ehaerzTg9KFS/Nj3x+pV66e32GZDFjRkDl5ixfDxRfD3r3ubOCaa/yOyPgsJTWFc187l9/3/M4j7R/hnnPvoUCMNSOSV1kiMCdn9GgYMgRKl4aPPnL3C5iotf3gdioUrUBMvhhGdR5FtVLVaFShkd9hmWwEXTQkIkVCGYgJM9u2uX6Ehwxxl4euWGFJIIqlairjF46nzot1GL9wPAA96vSwJBAmsk0EInKOiKwGfvHenyEi1qVkNNu71/Up/N138H//5+4RqJi+zyETLdbvXk/HNzrSf0Z/zqp4Fl1qdfE7JHOcgikaehbogteEtKouE5G2IY3K5F2rV7t+BP78E554Av77X78jMj56bclrDJw5kAIxBXj5wpfp26yv3RgWhoKqI1DVLem+3JTQhGPytJ07oWFD9/rzz6FTJ3/jMb6rUrIKXWp2YUz3MVQsYWeF4SqYRLBFRM4BVETyA7cCa0IblsmTevRwz48/bkkgSh1NPsrj3z1OqqYyvMNwOtboSMcaHf0Oy5ykYCqL+wODcB3PbwWaAgNDGJPJi0aNcr2JtWjhmo8wUefn+J9pPqE5j3zzCJv3bbZG4iJIMGcEdVX12sABItIa+D40IZk8Z9o014poWgfzJqocOnaIB75+gOd+eo6KJSry6dWfckGdC/wOy+SgYM4IXgxymIlEa9ZAnz5QtKirKLYOZaLOpn2bGLtgLP3j+rNq4CpLAhEo0zMCEWkFnAOUF5HbA0aVwPVBbCLd99/DBd6f3noViyp7j+xlyuop3HzmzTQo34D1Q9dbj2ERLKuioQJAMW+awD3AfuDyUAZl8oBdu6BzZ0hMhAULIC7Djo1MBPr4l48ZMGMAOw7t4Nwq51KvXD1LAhEu00Sgqt8A34jIJFXdlIsxGb+puruEExNhxgxLAlFix6EdDP1sKO+teo8mpzRh+tXTrZG4KBFMZXGiiDwFNAT+ak5SVc8LWVTGXx984O4avusu182kiXgpqSm0frU1m/dt5tEOj3J367vJH2P9SESLYBLBZOA9oAfuUtIbgJ2hDMr46NtvoVcvqFXLNR9hItofB/7g1GKnEpMvhue7Pk+1UtVoUL6B32GZXBbMVUNlVXUikKSq36jqTYCdDUSiOXPcjWKxsTB9unUqE8FSNZVxC8ZRb3Q9Xlr4EgDda3e3JBClgkkESd7zNhG5QESaAWVCGJPxwzvvwPnnu8tEly2D+vX9jsiEyK8Jv9Lh9Q4MnDmQlpVa0q1WN79DMj4LpmjoUREpCdyBu3+gBDAslEGZXJScDP37w8SJ0Lix61msZk2/ozIhMnHxRAZ/NphCsYV49aJX6dO0jzUSZ7JPBKr6qfdyH9AB/rqz2IS71FR3n8Dnn8MNN8C4cVC4sN9RmRCqVqoa3Wp1Y0z3MZxW/DS/wzF5RFY3lMUAV+LaGJqlqitFpAdwH1AYaJY7IZqQef55lwQGD4YX7WbxSHQ0+Sj/N89V+j963qPWSJzJUFZnBBOBysB84AUR+QOIA+5R1Wm5EJsJpUWL4PbboUoVePZZv6MxIfDDlh/oO70vv+z6hZua3oSqWjGQyVBWiSAOaKKqqSJSCNgO1FTVhNwJzYTM4cPQzasgnDbNXSVkIsbBYwe5f879vDj/RSqXrMysa2dZr2EmS1ldNXRMVVMBVPUIsOF4k4CIdBWRtSKyXkQybLtYRK4UkdUiskpE3j6e+ZsToAqXXuo6mRk+HJpZCV+k2bxvM+MXjWfQWYNYOWClJQGTLcmsTXERSQTWp70FanrvBVBVbZLljF0dw69AJyAeWABcraqrA6apDbwPnKeqe0SkgqruyGq+cXFxunDhwmDWzaSXmAg9e7qmpHv2dGcDJiLsObyHD1Z/QL/m/QB3o9jpxU/3OSqTl4jIIlXNsL2YrMoETvZC8hbAelXd4AXxLtATWB0wzS3AGFXdA5BdEjAnYdkyd4XQ1q1w222uoxkTEaaumcrAmQPZeWgn7aq2o265upYEzHHJqtG5k21oriKwJeB9PNAy3TR1AETke1zT1g+r6qz0MxKRfkA/gCpVqpxkWFEorTho61YYPx769fM7IpMDth/czpDPhjBl9RSantqUGdfMoG65un6HZcKQ37WEsUBtoD1QCZgnIo1VdW/gRKo6AZgArmgol2MMf08/DRs2wMiRlgQiREpqCm1ea8OWfVsYcd4I7jznTmskzpywUCaCrbjLT9NU8oYFigd+VtUk4HcR+RWXGBaEMK7o8sEH8N//utd33OFvLOakxe+P5/TipxOTL4YXur5A9dLVraloc9KCaWsIESksIsd7zrkAqC0i1UWkANALmJ5ummm4swFEpByuqGjDcS7HZGbzZrjpJvd64UKIsY7lwlWqpvLizy9Sb3Q9xi0YB0C32t0sCZgckW0iEJELgaXALO99UxFJv0P/F1VNBgYDs4E1wPuqukpEhovIRd5ks4EEEVkNfA3cZfcp5KCbb4ZDh+Cbb6B5c7+jMSfol12/0Pa1tgydNZRzq5xLjzo9/A7JRJhgioYexl0BNBdAVZeKSPVgZq6qM4GZ6YY9GPBagdu9h8kpyclw443wxRfu7uE2bfyOyJygVxa/wuCZgymSvwivX/w6vZv0truDTY4LJhEkqeq+dD8+q7DNqw4ehLPPhlWrXENyI0f6HZE5CTVL1+TCuhcyuttoTil2it/hmAgVTCJYJSLXADHeDWBDgR9CG5Y5IamprmvJVatc09LjxvkdkTlOR5KPMPyb4QCM6DiCDtU70KF6B5+jMpEumMriIbj+io8Cb+Oaox4WwpjMiUhNhR49XFeT99xjSSAMfb/5e5q+1JTHv3ucnYd2ktld/8bktGDOCOqp6v3A/aEOxpyEQYPgs8/cmcCIEX5HY47DgaMHuG/OfYxZMIaqpaoy+7rZdK7Z2e+wTBQJJhGMEpFTgSnAe6q6MsQxmeP17rvw0kuuTmDsWLDKxLASvz+eV5a8wpAWQ3is42MUK1DM75BMlMm00bl/TOQSwZXAVbiuKt9T1UdDHFuGrNG5dDZudP0Ln3oqrFkDhQr5HZEJQkJiAu+vep8BZw0AYNuBbdZjmAmprBqdC+qGMlXdrqovAP1x9xQ8mPUnTK5Yu9ZdIRQTA2+8YUkgDKgqU1ZPocHYBgydNZS1u9YCWBIwvgrmhrL6IvKwiKzAdV7/A665COOn/fvdTWL797vuJu1egTxv24FtXPb+ZVzxwRVULlGZhbcstEbiTJ4QTB3Bq8B7QBdV/SPE8ZhgJCXBuee6u4bfegvOOcfviEw20hqJ23pgK0+e/yS3tbqN2Hx+t/lojJPtL1FVW+VGIOY43H47rFgBzzwD117rdzQmC1v2baFiiYrE5IthTPcxVC9dnTpl6/gdljH/kGnRkIi87z2vEJHlAY8VIrI890I0/zBhAoweDZdc4jqYMXlSSmoKL/z8AvXG/N1IXJdaXSwJmDwpqzOCW71na+Eqr1i/Hv7zH/f6pZf8jcVkas3ONfSd3pcf43+kW61uXFj3Qr9DMiZLmZ4RqOo27+VAVd0U+AAG5k545h8uu8w9f/stVKjgbywmQxMWTaDp+Kb8mvArb17yJjOumUGVktarnsnbgrl8tFMGw7rldCAmG08+CcuXw4ABrqLY5Em1y9TmknqXsHrQaq5rcp21FGrCQqY3lInIANyRfw3gt4BRxYHvVfW60If3b1F5Q9nixe5S0SZN4Oef7X6BPORw0mEenvswIsIT5z/hdzjGZCqrG8qyqiN4G/gMeBy4J2D4AVXdnYPxmazMnw/nnw+FC7tuJy0J5BnzNs3j5uk3s273Ovo374+q2hmACUtZFQ2pqm4EBgEHAh6ISJnQh2ZISXFXB+XPD3PnQh274iQv2H90PwNnDKTdpHakaApzrp/DuB7jLAmYsJXdGUEPYBGuI5rAX7niioxMKJ1zDvzxB0yaBC1a+B2N8fxx4A8mLZ3E7WffzvAOwylaoKjfIRlzUjJNBKraw3sOqltKk8PWrHHFQk2bwjXX+B1N1NuVuIv3V73PwLMGUq9cPX6/9XfrMcxEjGDaGmotIkW919eJyDMiYtfDhdoTXsXjtGmuaMj4QlV5b+V7NBjTgGGzhvFrwq8AlgRMRAnm8tFxQKKInAHcgbuC6M2QRhXtfvvN9THQujVUrep3NFHrjwN/cPF7F9Prw15ULVWVRf0W2Z3BJiIF0+pVsqqqiPQERqvqRBHpG+rAotqQIa7ryTct3/olJTWFtq+1ZeuBrTzd6WluPftWayTORKxgftkHROReoDfQRkTyAVZWESrvv++6nBw6FKpb9Uxu27R3E5VKVCImXwxjLxhLjdI1qFWmlt9hGRNSwRQNXYXruP4mVd2O64vgqZBGFa3Wr3d9DlesCI895nc0USUlNYVnfnyG+mPqM26haySuc83OlgRMVAimGertIjIZOEtEegDzVfWN0IcWZVTd1UEHDsCcOVDM+q3NLSt3rKTv9L7M3zqfHnV6cHG9i/0OyZhcFcxVQ1cC84ErcP0W/ywil4c6sKjz3HOwYAE89BA0a+Z3NFHjpYUvceb4M9mwZwNvX/o203tNp1IJ64DPRJdg6gjuB85S1R0AIlIe+BKYEsrAosqxYy4BNGkC997rdzRRIa05iPrl6nNFwyt4rstzlC9a3u+wjPFFMIkgX1oS8CQQZKf3Jkh33+2KhO6913VEb0ImMSmRB79+kBiJYWSnkbSr1o521dr5HZYxvgpmhz5LRGaLSB8R6QPMAGaGNqwo8uWX8PzzrjmJq67yO5qINnfjXJqMa8KoH0dx8NhBMmt515hoE0xl8V0icimQ1gj+BFWdGtqwosTWrdCpExQsCB99BNZoWUjsO7KPu7+4mwmLJ1CzdE2+uv4rOlTv4HdYxuQZmSYCEakNPA3UBFYAd6rq1twKLCrcead7fuMNOMWaLAiVbQe38daKt7iz1Z080uERiuQv4ndIxuQpWRUNvQp8ClyGa4H0xeOduYh0FZG1IrJeRO7JYrrLRERFJMNOEyLS5MmuGYlOneDKK/2OJuLsPLSTF392P9l65eqx8daNPNX5KUsCxmQgq6Kh4qr6svd6rYgsPp4Zi0gMMAbX1WU8sEBEpqvq6nTTFQduBX4+nvmHtdWr4eaboW5d19mMyTGqyjsr32HoZ0PZf3Q/XWp1oU7ZOnZFkDFZyOqMoJCINBORM0XkTKBwuvfZaQGsV9UNqnoMeBfomcF0/weMBI4cd/ThSBUuvdS1JTR9OpQs6XdEEWPLvi1c+M6FXPvRtdQqU4sl/1lijcQZE4Sszgi2Ac8EvN8e8F6B87KZd0VgS8D7eKBl4AReQqmsqjNE5K7MZiQi/YB+AFWqhHkL2MOGwdq1MHKk9TiWg5JTk2n/enu2H9zOs12eZUiLIcTks0txjQlGVh3ThPSyCq/xumeAPtlNq6oTgAngOq8PZVwhtWgRvPgitGoFd9zhdzQRYePejVQuUZnYfLGM7zGeGqVrUKO0dZ5nzPEI5Y1hW4HKAe8recPSFAcaAXNFZCNwNjA9YiuMk5OhZ08oUwamTrUbx05ScmoyT//wNPXH1GfsgrEAnF/jfEsCxpyAUDawvgCoLSLVcQmgF/BXn4uqug8ol/ZeRObiLlFdGMKY/POf/7j7Bl56yS4VPUnL/1xO3+l9WfjHQnrW7cllDS7zOyRjwlrIEoGqJovIYGA2EAO8qqqrRGQ4sFBVp4dq2XnO2rXw2mvQuLFLCOaEjV0wlltn3UrpQqV57/L3uKLBFYjdiGfMSck2EYj7l10L1FDV4V5/xaeq6vzsPquqM0nXHIWqPpjJtO2Dijjc7NoFXbu6u4c//NDvaMJWWiNxjSo0olejXjzb5VnKFSmX/QeNMdkK5oxgLJCKu0poOHAA+BA4K4RxRY7GjWH7dpg5E2rX9juasHPo2CH+99X/iM0Xy1Odn6Jt1ba0rdrW77CMiSjBVBa3VNVBeNf5q+oeoEBIo4oU48e7JNCwIXTr5nc0YWfOhjk0HteY535+jqMpR62ROGNCJJgzgiTvLmGFv/ojSA1pVJHg229h8GBXJPRz9Nw0nRP2HtnLnZ/fycQlE6ldpjbz+syjTdU2fodlTMQK5ozgBWAqUEFEHgO+A0aENKpwt2sXtG3r+h7+7TcoWtTviMLKnwf/5N2V7/Lf1v9lWf9llgSMCbFgmqGeLCKLgI6AABer6pqQRxbOhg93z08/7ZKByVbazv/Ws2+lbrm6bBy20SqDjcklwVw1VAVIBD4JHKaqm0MZWNhKSHB3D7dtC5db187ZUVUmr5jMrbNu5eCxg3Sv3Z3aZWtbEjAmFwVTRzADVz8gQCGgOrAWaBjCuMLXkCHu+eGHfQ0jHGzet5n+n/bns/Wf0apSKyZeNJHaZe3KKmNyWzBFQ40D33sNxQ0MWUThbOpUeOcdOOMM6GA9YGUlOTWZ9pPas+PQDl7o+gIDzxpojcQZ45PjvrNYVReLSMvsp4wyqvDII5AvH3z9td/R5Fkb9mygasmqxOaL5eULX6ZmmZpUK1XN77CMiWrB1BHcHvA2H3Am8EfIIgpXP/0Ey5bBAw9A6dJ+R5PnJKcmM+qHUTw09yGe7PQkQ1sOpWONjn6HZYwhuDOC4gGvk3F1BtZWQnpvveU6nx861O9I8pyl25fSd3pfFm9bzCX1LuGKBlf4HZIxJkCWicC7kay4qt6ZS/GEp40bYexYaN8eytnVLoFGzx/NbbNvo2zhsky5Yoq1FGpMHpRpIhCRWK8F0da5GVBYGuHdXzdypL9x5CFpjcQ1OaUJ1za+lme6PEOZwmX8DssYkwHJrP0WEVmsqmeKyDhct5MfAIfSxqvqR7kT4j/FxcXpwoV5qMuC33+HGjXclUJLl/odje8OHjvI/XPuJ39Mfp7u/LTf4RhjPCKySFUz7PgrmCYmCgEJuNZHewAXes8mNRWaNXOv0+4mjmKf//Y5jcY24sX5L5KUkmSNxBkTJrKqI6jgXTG0kr9vKEtj/3CAxx6DffugXz+46CK/o/HNnsN7uP3z25m0dBJ1y9Zl3o3zOLfKuX6HZYwJUlaJIAYoxj8TQBpLBLt2wVNPuf4Gxo/3Oxpf7Ti0gymrp3DvuffyYLsHKRRbyO+QjDHHIatEsE1VrbwjM337woED8Morfkfii+0Ht/POine4rdVtrpG4WzdStkhZv8MyxpyArOoIrCPYzCQlwfffQ2wstGjhdzS5SlV5fenrNBjTgHvn3Mu6hHUAlgSMCWNZJQK77TMzV17pWhmNsiKhjXs30nVyV/p83IcG5RuwtP9SayTOmAiQadGQqu7OzUDCxuzZMG2aSwY33eR3NLkmOTWZDq93YFfiLsZ0H0P/uP7kk2AuOjPG5HXH3ehc1Hv0Ufc8YYK/ceSS9bvXU71UdWLzxfLqRa9So3QNqpaq6ndYxpgcZId0x+OLL+C77+DSS6FkSb+jCamklCRGfDuChmMbMmbBGAA6VO9gScCYCGRnBMFKSYGbb4ZixWDcOL+jCanF2xbTd3pflm5fyhUNruCqhlf5HZIxJoQsEQTruutg82aYOBEqVPA7mpB54ecXuH327ZQvWp6PrvyIS+pf4ndIxpgQs0QQjD/+gHffha5d4cYb/Y4mJNIaiWt2ajOuP+N6RnUeRenC1q+CMdHAEkEwbrvNPQ8b5vociCAHjh7g3jn3UjCmIKO6jKJN1Ta0qdrG77CMMbnIKouzs307vP++a1yuSxe/o8lRs9bPotG4RoxdMBZFrZE4Y6KUnRFk55ln3PNzz/kaRk5KSEzg9s9v541lb1C/XH2+v+l7WlVu5XdYxhifWCLIzvTpcOqp0Lat35HkmITDCUxdM5UH2j7A/W3up2BsQb9DMsb4KKRFQyLSVUTWish6Ebkng/G3i8hqEVkuInNEJG9dpL5mDaxdC4MH+x3JSdt2YBtP//A0qkqdsnXYNGwTwzsMtyRgjAldIvD6Ox4DdAMaAFeLSIN0ky0B4lS1CTAFeDJU8ZyQtOKgMK4bUFVeXfIq9cfU54GvH2D97vUAdkWQMeYvoTwjaAGsV9UNqnoMeBfoGTiBqn6tqone25+ASiGM5/isWOGakTjrLIjLsHe3PO/3Pb/T+a3O9J3elzNOPYNl/ZdZI3HGmH8JZR1BRWBLwPt4oGUW0/cFPstohIj0A/oBVKlSJafiy9qwYe45TPsbSE5N5rw3ziMhMYFxF4yjX/N+1kicMSZDeaKyWESuA+KAdhmNV9UJwARwndfnSlBr10Lx4tCkSa4sLqesS1hHjdI1iM0Xy2s9X6Nm6ZpULlnZ77CMMXlYKA8RtwKBe6BK3rB/EJHzgfuBi1T1aAjjCd5PP8HWrXDPv+q386yklCQenfcojcY1YvT80QC0r9bekoAxJluhPCNYANQWkeq4BNALuCZwAhFpBowHuqrqjhDGcnwmTXLP11yT5WR5xcI/FtJ3el+W/7mcXo16cXXjq/0OyRgTRkKWCFQ1WUQGA7OBGOBVVV0lIsOBhao6HXgKKAZ8IK7phs2qelGoYgpKcjK8/LJrV6haNV9DCcbzPz3P7Z/fzqnFTuXjXh9zUV1/N58xJvyEtI5AVWcCM9MNezDg9fmhXP4J+fFHSE2Fq/J208tpjcTFnR5H32Z9ebLTk5QqVMrvsIwxYShPVBbnKU8/DfnyQefOfkeSof1H9/PfL/5LodhCPNv1WVpXaU3rKq39DssYE8bsesJASUnwySfQsiWcfrrf0fzLzHUzaTi2IRMWTyA2X6w1EmeMyRF2RhBo5kxQhT59/I7kH3Yl7mLYrGFMXjGZhuUbMuWKKbSslNUtGcYYEzxLBIHGuL556d7d3zjS2XN4D5/8+gkPtXuI+9rcR4GYAn6HZIyJIJYI0mzc6Dqnb9sWKvnf0sXW/VuZvGIyd51zF7XL1mbTsE1WGWyMCQmrI0iTdvPY//2fr2GoKi8vepkGYxvw8NyH+W3PbwCWBIwxIWOJAFy9wDffQMOGvvY78Nvu3+j4Rkf6fdqPM087k+UDllOrTC3f4jHGRAcrGgLXpMT27fDAA76FkJyaTMc3OrL78G7G9xjPzWfebI3EGWNyhSUC+LuS+Morc33Ra3etpWaZmsTmi+X1i1+nZpmaVCrhfx2FMSZ62CHnnj0weTLUrw/lyuXaYo+lHOORuY/QeFxjxsx3iahdtXaWBIwxuc7OCCZPds8jRuTaIudvnU/f6X1ZuWMl1zS+hmubXJtryzbGmPQsEbz3HuTPDxdemCuLe+6n57jj8zs4rdhpfHL1J/So0yNXlmuMMZmJ7kSg6iqKzzgDYmJCvCjXSFyLii245cxbGHn+SEoWKhnSZRpjTDCiOxGsWeOanb4odE037zuyj7u/uJvC+QvzXNfnOKfyOZxT+ZyQLc8YY45XdFcWjx/vnnv1CsnsP1n7CQ3GNuCVJa9QMKagNRJnjMmToveMIDnZVRSfdhrUqZOjs955aCe3zrqVd1a+Q+MKjZl21TTOqnhWji7DGGNySvQmgmnTICHh77OCHLTv6D5mrpvJI+0f4Z5z77FG4owxeVr0JoL333fPl16aI7Pbsm8Lby1/i3vOvYdaZWqxadgmqww2xoSF6KwjSEmBDz6Ayy476ZvIUjWVlxa+RMOxDXn020f/aiTOkoAxJlxEZyL46Sf3XL/+Sc1mXcI6znv9PAbMGECLii1YMWCFNRJnjAk70Vk0tG6dez7//BOeRXJqMp3e7MTeI3uZeNFEbmx6IyKSQwEaY0zuic5EMHkyFCgA5xz/9fxrdq6hdtnaxOaL5c1L3qRmmZqcXjzv9W9sjDHBir6iIVVYtgwaNHBNSwTpaPJRHvr6IZq81ITR80cD0KZqG0sCxpiwF31nBEuXws6df/dIFoSf4n+i7/S+rN65mt5NetO7Se/QxWeMMbks+hLBRx+55/btg5p81A+juOuLu6hUohIzr5lJt9rdQhebMcb4IPoSwYIF7rlBgywnS9VU8kk+WlVuRf+4/jxx/hOUKFgiFwI0xpjcFV2J4OhRmD3b9UtcqFCGk+w9spc7Zt9BkfxFeLH7i9ZInDEm4kVXZfGkSe75mmsyHD3tl2k0GNOA15e9TvGCxa2ROGNMVIiuM4Ivv3TP11//j8E7Du1g8MzBfLD6A5qe2pRPr/mUM08704cAjTEm90VXIli61PU9ULjwPwbvP7qfLzZ8wWPnPcZd59xF/pjgLys1xphwFz1FQ3v2wPr1UK8eAJv3beaxeY+hqtQqU4vNwzZzX5v7LAkYY6JOSBOBiHQVkbUisl5E/nXhvogUFJH3vPE/i0i1kAUTHw9Aas0ajF0wloZjGzLiuxF/NRJXvGDxkC3aGGPyspAlAhGJAcYA3YAGwNUikv6azb7AHlWtBTwLjAxVPCxezNqy0P7giwyaOYhWlVqxauAqayTOGBP1QnlG0AJYr6obVPUY8C7QM900PYHXvddTgI4SopbbkvftoUtvWHE0ntd6vsbs62ZTrVS1UCzKGGPCSigriysCWwLexwMtM5tGVZNFZB9QFtgVOJGI9AP6AVSpUuWEgomtWp235reh5kNvc1qpSic0D2OMiURhUVmsqhNUNU5V48qXL39iM+nZk3PfmmdJwBhj0gllItgKVA54X8kbluE0IhILlAQSQhiTMcaYdEKZCBYAtUWkuogUAHoB09NNMx24wXt9OfCV2u28xhiTq0JWR+CV+Q8GZgMxwKuqukpEhgMLVXU6MBF4U0TWA7txycIYY0wuCumdxao6E5iZbtiDAa+PAFeEMgZjjDFZC4vKYmOMMaFjicAYY6KcJQJjjIlylgiMMSbKSbhdrSkiO4FNJ/jxcqS7azkK2DpHB1vn6HAy61xVVTO8IzfsEsHJEJGFqhrndxy5ydY5Otg6R4dQrbMVDRljTJSzRGCMMVEu2hLBBL8D8IGtc3SwdY4OIVnnqKojMMYY82/RdkZgjDEmHUsExhgT5SIyEYhIVxFZKyLrReSeDMYXFJH3vPE/i0g1H8LMUUGs8+0islpElovIHBGp6kecOSm7dQ6Y7jIRUREJ+0sNg1lnEbnS+65XicjbuR1jTgvit11FRL4WkSXe77u7H3HmFBF5VUR2iMjKTMaLiLzgbY/lInLmSS9UVSPqgWvy+jegBlAAWAY0SDfNQOAl73Uv4D2/486Fde4AFPFeD4iGdfamKw7MA34C4vyOOxe+59rAEqC0976C33HnwjpPAAZ4rxsAG/2O+yTXuS1wJrAyk/Hdgc8AAc4Gfj7ZZUbiGUELYL2qblDVY8C7QM900/QEXvdeTwE6iojkYow5Ldt1VtWvVTXRe/sTrse4cBbM9wzwf8BI4EhuBhciwazzLcAYVd0DoKo7cjnGnBbMOitQwntdEvgjF+PLcao6D9c/S2Z6Am+o8xNQSkROO5llRmIiqAhsCXgf7w3LcBpVTQb2AWVzJbrQCGadA/XFHVGEs2zX2TtlrqyqM3IzsBAK5nuuA9QRke9F5CcR6Zpr0YVGMOv8MHCdiMTj+j8Zkjuh+eZ4/+/ZCmnHNCbvEZHrgDignd+xhJKI5AOeAfr4HEpui8UVD7XHnfXNE5HGqrrXz6BC7GpgkqqOEpFWuF4PG6lqqt+BhYtIPCPYClQOeF/JG5bhNCISizudTMiV6EIjmHVGRM4H7gcuUtWjuRRbqGS3zsWBRsBcEdmIK0udHuYVxsF8z/HAdFVNUtXfgV9xiSFcBbPOfYH3AVT1R6AQrnG2SBXU//14RGIiWADUFpHqIlIAVxk8Pd0004EbvNeXA1+pVwsTprJdZxFpBozHJYFwLzeGbNZZVfepajlVraaq1XD1Ihep6kJ/ws0Rwfy2p+HOBhCRcriiog25GGNOC2adNwMdAUSkPi4R7MzVKHPXdOB67+qhs4F9qrrtZGYYcUVDqposIoOB2bgrDl5V1VUiMhxYqKrTgYm408f1uEqZXv5FfPKCXOengGLAB169+GZVvci3oE9SkOscUYJc59lAZxFZDaQAd6lq2J7tBrnOdwAvi8htuIrjPuF8YCci7+CSeTmv3uMhID+Aqr6EqwfpDqwHEoEbT3qZYby9jDHG5IBILBoyxhhzHCwRGGNMlLNEYIwxUc4SgTHGRDlLBMYYE+UsEZg8SURSRGRpwKNaFtMezIHlTRKR371lLfbuUD3eebwiIg281/elG/fDycbozSdtu6wUkU9EpFQ20zcN99Y4TejZ5aMmTxKRg6paLKenzWIek4BPVXWKiHQGnlbVJicxv5OOKbv5isjrwK+q+lgW0/fBtbo6OKdjMZHDzghMWBCRYl4/CotFZIWI/KulURE5TUTmBRwxt/GGdxaRH73PfiAi2e2g5wG1vM/e7s1rpYgM84YVFZEZIrLMG36VN3yuiMSJyBNAYS+Oyd64g97zuyJyQUDMk0TkchGJEZGnRGSB18b8f4LYLD/iNTYmIi28dVwiIj+ISF3vTtzhwFVeLFd5sb8qIvO9aTNqsdVEG7/b3raHPTJ64O6KXeo9puLugi/hjSuHu6sy7Yz2oPd8B3C/9zoG195QOdyOvag3/L/AgxksbxJwuff6CuBnoDmwAiiKuyt7FdAMuAx4OeCzJb3nuXh9HqTFFDBNWoyXAK97rwvgWpEsDPQD/ucNLwgsBKpnEOfBgPX7AOjqvS8BxHqvzwc+9F73AUYHfH4EcJ33uhSuLaKifn/f9vD3EXFNTJiIcVhVm6a9EZH8wAgRaQuk4o6ETwG2B3xmAfCqN+00VV0qIu1wnZV87zWtUQB3JJ2Rp0Tkf7h2avri2q+ZqqqHvBg+AtoAs4BRIjISV5z07XGs12fA8yJSEOgKzFPVw15xVBMRudybriSusbjf032+sIgs9dZ/DfBFwPSvi0htXDML+TNZfmfgIhG503tfCKjizctEKUsEJlxcC5QHmqtqkrgWRQsFTqCq87xEcQEwSUSeAfYAX6jq1UEs4y5VnZL2RkQ6ZjSRqv4qrq+D7sCjIjJHVYcHsxKqekRE5gJdgKtwHa2A621qiKrOzmYWh1W1qYgUwbW/Mwh4AdcBz9eqeolXsT43k88LcJmqrg0mXhMdrI7AhIuSwA4vCXQA/tXnsrh+mP9U1ZeBV3Dd/f0EtBaRtDL/oiJSJ8hlfgtcLCJFRKQorljnWxE5HUhU1bdwjfll1GdskndmkpH3cA2FpZ1dgNupD0j7jIjU8ZaZIXW9zQ0F7pC/m1JPa4q4T8CkB3BFZGlmA0PEOz0S1yqtiXKWCEy4mAzEicgK4HrglwymaQ8sE5EluKPt51V1J27H+I6ILMcVC9ULZoGquhhXdzAfV2fwiqouARoD870imoeARzP4+ARgeVplcTqf4zoG+lJd94vgEtdqYLG4TsvHk80ZuxfLclzHLE8Cj3vrHvi5r4EGaZXFuDOH/F5sq7z3JsrZ5aPGGBPl7IzAGGOinCUCY4yJcpYIjDEmylkiMMaYKGeJwBhjopwlAmOMiXKWCIwxJsr9P/gyL9BP5wsdAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSP AUROC score: 0.79430659\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, auroc_score = auroc(test_y, ood_scores)\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "print('MSP AUROC score:', auroc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAnL0lEQVR4nO3deZyVdd3/8dd7BhQBAQW6FUFRRBFlUUdRtDRX3LC0zO1335S5FWlpentnZbllmWYmpeSapaRZiltq5L4PuSGkIrmAG6KIIKLA5/fH94xnZhiYA5zrnDNz3s/HYx7X9j3X9blYzmeu67spIjAzs+pVU+4AzMysvJwIzMyqnBOBmVmVcyIwM6tyTgRmZlXOicDMrMo5EVibIekISXcXUO5SST8qRUylIOkVSXvk1n8i6Y/ljsnaFycCK4rcl9VCSfMlvS3pakldi3mNiPhTROxVQLnjIuKsYl67gaSQtCB3n7MkXSipNotrrQpJ3SRdJOm1XIwv57Z7lTs2q1xOBFZMB0REV2AboA74YfMCkjqUPKriG5a7z12ArwHfKHM8AEhaA5gEbAmMAroBOwJzgO1X4Xzt4e/KCuBEYEUXEbOAO4Gt4LPfor8t6SXgpdy+/SU9LWmupEckDW34vKR+kv4qabakOZIuye0fI+mh3Lok/UrSO5LmSXpOUsP1rpZ0dqPzHS1puqT3JE2U1KfRsZB0nKSXcrGMk6QC73M68DAwvNH5VuW+Bkj6Z27fu5L+JKnHSv6xA/w3sCHw5YiYGhFLI+KdiDgrIu5odL+bNorpsz8rSbtKminpfyW9BVwlaZqk/RuV75CLf5vc9g65+5wr6RlJu65C3FZmTgRWdJL6AfsCTzXa/SVgBDBY0tbAlcCxQE/gMmCipDVzr1luA14F+gMbABNauMxewBeAzYDuwCGk33ybx7Ib8LPc8fVz521+vv2B7YChuXJ7F3ifg4DPA9Nz26t6X8rF2AfYAugH/KSQGJrZA/h7RMxfhc82WA9YF9gIOAa4Hjis0fG9gXcj4l+SNgBuB87Ofeb7wE2Seq/G9a0MnAismG6WNBd4CLgfOLfRsZ9FxHsRsZD0BXNZRDweEUsi4hpgEbAD6RVGH+CUiFgQER9HxEMtXOtTYG1gEKCImBYRb7ZQ7gjgyoj4V0QsAv4P2FFS/0ZlzouIuRHxGnAvjX7DX45/SVoATAPuA36b279K9xUR0yPinohYFBGzgQtJr51WVk+gpT+DlbEUOCMXy0LgOmC0pM6544eTkgPAkcAdEXFH7unjHqCe9EuAtSFOBFZMX4qIHhGxUUR8K/dF0uD1RusbASfnXifMzSWPfqQvyn7AqxGxeEUXioh/ApcA44B3JI2X1K2Fon1Iv4U3fG4+6clhg0Zl3mq0/hHQFUDS87kK1/mSPt+ozDa5Ml8jPeV0WZ37kvRfkibkKp/nAX8EVqVydw7pqWd1zI6Ijxs2cq+/pgEH5JLBaFJygHS/X212vzsXIQYrMScCK5XGw9y+DpyTSxoNP50j4vrcsQ0LqaiMiIsjYltgMOkV0SktFHuD9IUFgKQupN+cZxVw/i0jomvu58FmxyIibgAeBX68mvd1LunPZ0hEdCP9pl1QPUUz/wD2zt3j8nwEdG60vV6z4y0NR9zweuhAYGouOUC6p2ub3W+XiDhvFWK3MnIisHL4PXCcpBG5St8ukvaTtDbwBOn1xnm5/Z0k7dT8BJK2y32+I7AA+Jj0WqO564GvSxouaU3Sl+7jEfFKke7lPOBoSeutxn2tDcwHPsi9d28poRXiWtKX802SBkmqkdRT0g8kNbyueRo4XFKtpFEU9gpqAqlO5njyTwOQnlwOkLR37nydchXOfVcxfisTJwIruYioB44mvdp5n1TZOiZ3bAlwALAp8Bowk/QKprlupC/e90mvfuYA57dwrX8APwJuIn0RDwAOLeK9PAc8QHr3v6r39VPS66YPSJWvf13FWBaRKoz/DdwDzCMloF7A47liJ+bimEuqP7m5gPO+SXryGQn8udH+10lPCT8AZpOS0Cn4e6XNkSemMTOrbs7cZmZVzonAzKzKORGYmVU5JwIzsyrX5gaV6tWrV/Tv37/cYZiZtSmTJ09+NyJaHP6jzSWC/v37U19fX+4wzMzaFEmvLu+YXw2ZmVU5JwIzsyrnRGBmVuXaXB2Bmdmq+vTTT5k5cyYff/xx64XbqE6dOtG3b186duxY8GecCMysasycOZO1116b/v37U+BEdG1KRDBnzhxmzpzJxhtvXPDnMns1JOlKpWkEpyznuCRdrDSF4LMNU9+ZmWXl448/pmfPnu0yCQBIomfPniv9xJNlHcHVpAm0l2cfYGDu5xjgdxnGYmYG0G6TQINVub/MEkFEPAC8t4IiBwJ/yE3w8RjQQ1J2Mxs99BCcey588EFmlzAza4vK2WpoA5pOXziTptMHfkbSMZLqJdXPnj171a52001w+ulw112r9nkzsyKora1l+PDhbLXVVhxwwAHMnTv3s2PPP/88u+22G5tvvjkDBw7krLPOovFUAXfeeSd1dXUMHjyYrbfempNPPrkoMbWJ5qMRMT4i6iKirnfvFntIt+6449JyyZLiBWZmtpLWWmstnn76aaZMmcK6667LuHHjAFi4cCGjR4/mtNNO44UXXuCZZ57hkUce4be//S0AU6ZMYezYsfzxj39k6tSp1NfXs+mmmxYlpnImglmkCb0b9KWAeWTNzNqLHXfckVmz0tfeddddx0477cRee+0FQOfOnbnkkks477w0BfQvfvELTj/9dAYNGgSkJ4vjjz++KHGUs/noRGCspAnACOCD3JR4ZmbZ++534emni3vO4cPhoosKKrpkyRImTZrEUUcdBaTXQttuu22TMgMGDGD+/PnMmzePKVOmFO1VUHOZJQJJ1wO7Ar0kzQTOADoCRMSlwB3AvqR5XT8Cvp5VLGZmlWLhwoUMHz6cWbNmscUWW7DnnnuWO6TsEkFEHNbK8QC+ndX1zcxWqMDf3IutoY7go48+Yu+992bcuHGccMIJDB48mAceeKBJ2RkzZtC1a1e6devGlltuyeTJkxk2bFjRY2oTlcVmZu1N586dufjii7ngggtYvHgxRxxxBA899BD/+Mc/gPTkcMIJJ3DqqacCcMopp3Duuefy4osvArB06VIuvfTSosTiRGBmViZbb701Q4cO5frrr2ettdbilltu4eyzz2bzzTdnyJAhbLfddowdOxaAoUOHctFFF3HYYYexxRZbsNVWWzFjxoyixOGxhszMSmj+/PlNtm+99dbP1ocMGcJ999233M/uv//+7L///kWPyU8EZmZVzonAzKzKORGYWVVpPGRDe7Qq9+dEYGZVo1OnTsyZM6fdJoOG+Qg6deq0Up9zZbGZVY2+ffsyc+ZMVnnwyjagYYayleFEYGZVo2PHjis1c1e18KshM7MqV32JIDekq5mZJdWXCB56qNwRmJlVlOpLBGZm1kR1JoJ22nTMzGxVVGciuOOOtHzvPXj88fLGYmZWZtXZfHTBgrQcMgTeeMNPCGZW1arziaChwviNN8obh5lZBajORPCb38CiReWOwsysIlRnIgB49tlyR2BmVhGqNxFMmZJf99OBmVWx6kkES5Y03X7hhfx6keb9NDNri6onEXzySdPtRx/Nr3fpUtpYzMwqSPUkgk8/bbr9wAP5BLDOOqWPx8ysQlRvIgAYMSItG78mMjOrMtWTCJrXEQBsvnlavvdeaWMxM6sg1Z0Ihg5NywsugHY8Y5GZ2YpUdyIYMCC//rnPwZgxJQvHzKxSVHciGDSo6fY118Dtt5cmHjOzClE9iWDp0mX3bbDBsvv23z/7WMzMKkj1JIKWnghqauCll5ruO/jg0sRjZlYhqicRdO+elnvv3XT/ppumJDF/ftru27e0cZmZlVmmiUDSKEkvSJou6bQWjm8kaZKkZyXdJym7b+Gdd4abboIzz0zbw4blj9XU5DuX/frXmYVgZlaJMksEkmqBccA+wGDgMEmDmxX7JfCHiBgKnAn8LKt4ADjoINh2Wzj2WPjrXzO9lJlZW5HlDGXbA9MjYgaApAnAgcDURmUGAyfl1u8Fbs4wnqS2dvmDzHXv7tnKzKzqZPlqaAPg9UbbM3P7GnsGOCi3/mVgbUk9m59I0jGS6iXVz86y49cHH8C8ee5pbGZVpdyVxd8HdpH0FLALMAtYpnlPRIyPiLqIqOvdu3f2UfVcJheZmbVbWSaCWUC/Rtt9c/s+ExFvRMRBEbE1cHpu39wMY1qxHXcs26XNzMoly0TwJDBQ0saS1gAOBSY2LiCpl6SGGP4PuDLDeFp355359csvL18cZmYllFkiiIjFwFjgLmAacENEPC/pTEmjc8V2BV6Q9CLwX8A5WcVTkIa+BgBHH12+OMzMSkjRxlrJ1NXVRX19fXYX2G47aDh/G/uzMTNbHkmTI6KupWPlriyuPE8+We4IzMxKyolgRXbYodwRmJllzolgRR5/PD/J/bRp8J//+HWRmbU7TgQtee21/PrIkXDKKTB4MGyyCUycuPzPmZm1QU4ELenXD844I7/9y1/m1y+4AD79tGn5pUv9pGBmbZYTwfIcfnjL+x98ENZYAyR46qm0rK1NI5hKsPnmpY3TzGw1OREsz2abNf0tf8SIZctss82y+158MbuYzMwy4ETQmt13T8tHH4VTT225zDrrNN2W4O23s43LzKxInAhac/fdsGhR+nL/+c/TU0IEnH8+fOMbaf2999KyYdIbgPXWg3XXhU8+KV/sZmYFcM/iYlqyBDo0m+Jh7Fj4zW/KE4+ZWY57FpdKbe2yrYcuuSQtm+9vYwnYzNovJ4IsLF0KCxbkt6V8q6KGn4btP/+5fHGameFEkA0JOneGPfdsveyhh/rpwMzKyokgS3ffDT/9KWy/PeyyS37/8cfDhhvmt2tq4KqrSh+fmRmuLC6vm26Cr3yl5WPrr5/GNlpzzdLGZGbtkiuLK9XBB8OYMS0fe/NN6NQJFi4saUhmVn2cCMrtqqtSHcGSJTB//rLHjzqq9DGZWVVxIqgUNTXQpUu+w9qbb6b911+fb2m0/vqw665w443wxhv5zz7+OLz7blnCNrO2r0PrRaws1ltv2X1vvZV+7r+/5c8sWZISipnZSvC3RiWLSH0SJkworHxtbdO+Cg0/r7+ebZxm1qY5EVQ6Cb72tfwro4afiy5Kx8eMaf2LfsMN4eOPs47UzNooJ4K26sQTU0K46iro2xfmzEktjF55BerrYepUeP/9fPm11mr6lNCtWxo4b+nSst2CmVUGJ4L2Yt11U3PTjTaCbbeFLbaAHj3SXMst+fDDNKx2w+ukxx93D2ezKuVE0N4NGpS+4H/zm1QBfd118P/+X5plrbEddkgVzSNHwi9+UZ5Yzaws3LO42j30EOy227LzMN9yC4weXZ6YzKzo3LPYlm/nndPkORFw7LGwzz5p/4EHph8za/fcj8DyLr00LaW0nDgx3/y0b9/yxWVmmfITgS1ryRL43e/y2/365VsbbbNN+eIys0w4EdiyamrguONg8eJljz31VP6JwczaBScCW76GqTeXLoWPPmpaeSzBjjumzmrjx5cvRjNbbU4E1jopdUi75ZbU/LTBY4+l+oNjj01lunZ1XwSzNijTRCBplKQXJE2XdFoLxzeUdK+kpyQ9K2nfLOOxIjjsMHjuOfjWt2D4cDjkkPyxBQvSa6W//a1s4ZnZysusH4GkWuBFYE9gJvAkcFhETG1UZjzwVET8TtJg4I6I6L+i87ofQYWaMCEliQZDhsAzz8Czz8KwYeWLy8yA8vUj2B6YHhEzIuITYALQvGF6AN1y692BN7C26dBDU6e0rbZK2889l54Ohg9veURUCa64wmMdmVWAghKBpJ0k3SPpRUkzJP1H0oxWPrYB0HhYzJm5fY39BDhS0kzgDuA7y7n+MZLqJdXPnj27kJCtHDp0SAlg5szCyn/zm6lCevLk/KiqZlZyhT4RXAFcCOwMbAfU5Zar6zDg6ojoC+wLXCtpmZgiYnxE1EVEXe/evYtwWcvUBhvkv9iXLoW//x2mT09Tcb77Lrz9Npx8cr58XV16eqipyT8tNMzQZmaZKzQRfBARd0bEOxExp+Gnlc/MAvo12u6b29fYUcANABHxKNAJ6FVgTNYWSLD33jBgQJqKs2dP+Nzn4Je/TEnizDNb/lyfPmlqTjPLXKGJ4F5J50vaUdI2DT+tfOZJYKCkjSWtARwKTGxW5jVgdwBJW5ASgd/9VAsJfvSjphPuLFwI3bun42+9lco88UR54zRr5woda2hEbtm4xjmA3Zb3gYhYLGkscBdQC1wZEc9LOhOoj4iJwMnA7yV9L3e+MdHWhkO14urUCebOTa+SBg5M+0bk/vl94QswaVKqizCzovEw1Fa5IuCHP4Rzz226//bbYV93OTFbGavdfFRSd0kXNrTckXSBpO7FDdOsGQnOOSclhHffze/fb798pfI664BbkpmtlkLrCK4EPgQOyf3MA67KKiizZfTsmSqXf/7zpvvnzk2Vz1KaYMetjcxWWqGJYEBEnJHrHDYjIn4KbJJlYGbLkNI8yw0Vy9OmpYl1Gtx7b2ptJMH3v++kYFagQhPBQkmf/Y+TtBOwMJuQzAo0aBA8+GDLndEuuCCfFJwQzFao0ERwPDBO0iuSXgUuAY7LLiyzVdCQEB54oOn+hoQgwWWXpak5zewzK9VqSFI3gIiYl1lErXCrISvYkiVpGIsnnoCpU5c9vvbaaQrONdeEW29NCaPGI7Nb+7SiVkMrbJAt6ciI+KOkk5rtByAiLixalGbFVlsLVzVq0/D003DCCel1EsCHH6Z6BkjTcTY45hg4//yUKDwbm1WB1n796ZJbrr2cH7O2Y/jw9NooIr0eeu211BLp0kublhs/PvVubhj7aJttUlLp0CH/iukPf4D6evj447LcilkxuUOZWYOlS+HGG9OQ2ivj2GOXTSZmFaYYHcp+IambpI6SJkmaLenI4oZpVmY1NfC1rzUd+ygCFi9OSaJh+y9/gaFD85+77LL8k8KXvgS//W3qANfGfsmy6lVozdheuQri/YFXgE2BU7IKyqyi1NY2rSs4+OA0+1pEmoGtsVtugW9/G3r3zr9aOvfcfF2EWQUqNBE0VCrvB9wYER9kFI9Z2zJkSP5JYc6c9DTwP//TtMzpp8PgwfmnhhmtzelkVlqFJoLbJP0b2BaYJKk34Foys8bWXReOPx6uvjolhkWL0pPA737XtNyAASkhDB7s10dWEQpKBBFxGjASqIuIT4EFLDv/sJk1tsYaqffzccelL/xPP4UTT8wfnzat6axshxwCL71Uvnitaq0wEUjaLbc8CNgVODC3PoqUGMysUB06wEUX5ZuvbrFF0+M33gibbZaSwhFHFD73s9lqau2JYJfc8oAWfvbPMC6z9q1jx9TbuaF+YckS+Na38sevuy51clt33TRTm1mG3I/ArNK8+GLq3Xz//fl9w4enTm677w6jRsF225UtPGubitGP4FxJPRptryPp7CLFZ2aNbbYZ3Hdf6rtw/PFp39NPp8Tw4x/D9tun10fDh+frF3784zIGbG1doa2G9omIuQ0bEfE+4LkCzbIkpeaoS5bABx/A66+n+RgaPPNMfv2ss1L5889P5c1WQqGJoFbSmg0bktYC1lxBeTMrlpoa6NYtjZT685/nezs31C/MmpUve+qpqVK6c2eoq0tNVx96CD76KD1hmLWg0ETwJ1L/gaMkHQXcA1yTXVhmtkK1tfn1Pn3y8zAMGJD2LVwIkyenCujPfx66dMn3kD7ooPLEbBWr4MpiSaOAPXKb90TEXZlFtQKuLDYrwJIl8Ne/pmRw223w/PPLlrn22tR3YY01Sh+fldyKKotXJhFsBAyMiH9I6gzURsSHRYyzIE4EZqvhrbdg001hwYL8vi5d4LvfTUnhk09SJXSHFU5VYm1QMVoNHQ38Bbgst2sD4OaiRGdmpbPeejB/fqpo/upX074FC+Ccc2DYsNQstWPHfGukPn3g8ss9FEY7V2gdwbeBnYB5ABHxEvC5rIIys4wNHQo33JC+4J95Br7wBdh55/zxrl3T8s034eijmw6FIaXK6K22gjFj4OWXy3ILVjyFJoJFEfHZjN+SOgD+FcGsPRg6NPVRePDBfEukDz9My2nTYN9907SdjS1cmOodrrkmvWpq6Mvw5JPluQdbLYUmgvsl/QBYS9KewI3ArdmFZWYVYdAguP12mDcvPxTGhx+mRPDww6n/QoOzzkqd3XbcMT8vtLUJhSaC/wVmA88BxwJ3AD/MKigzq1A1Nem1UadOMHIk/PCHKUG88QaccUYq89hj6VWTlOZ/fust1zFUuFZbDUmqBZ6PiEGlCWnF3GrIrIItXQq//jWcdNKyxw4/HMaNgx49Sh6WrWaroYhYArwgacOiR2Zm7UtNDXzve+kJ4KWX0uuiPXLdj667DtZZJ5UZORLOPjtVRlvZFdSPQNIDwNbAE6RJaQCIiNHZhdYyPxGYtUGvvQbf+Q5MnLjssdra1DJp0aLUl2HUqNLHVwVWu0OZpF1a2h8R97e0v9HnRgG/BmqByyPivGbHfwV8MbfZGfhcRPRY0TmdCMzagZdfhksvhV/+suXjffqkCulDDklPDr16lTa+dmiVE4GkTsBxwKakiuIrImJxgRetBV4E9gRmAk8Ch0XE1OWU/w6wdUR8Y0XndSIwa2eWLoVHHoFXX4VTTln+66JvfhN+//vSxtaOrE4dwTVAHSkJ7ANcsBLX3R6YHhEzcn0QJrDieY4PA65fifObWXtQU5M6sx1xRGp91NCXYe7c9MqoweWXp5ZIY8Y0HXHVVltriWBwRBwZEZcBXwE+vxLn3gB4vdH2zNy+ZeTGMdoY+OdKnN/M2rPu3VPz04g0jHbDqKnXXJOG5O7Xzx3YiqS1RPBpw0qhr4RW0aHAX3ItlJYh6RhJ9ZLqZ8+enWEYZlaR1loLbropjZO0zz5p38yZ+dnaHn64vPG1ca0lgmGS5uV+PgSGNqxLmtfKZ2cB/Rpt983ta8mhrOC1UESMj4i6iKjr3bt3K5c1s3arSxe4447Uw7nxnM477wwbb5zmZHDntZW2wkQQEbUR0S33s3ZEdGi03q2Vcz8JDJS0saQ1SF/2y7QdkzQIWAd4dFVvwsyqTE1N6r3cMGgewCuvwC67pGP77AN33lnWENuSQoeYWGm5V0ljgbuAacANEfG8pDMlNe5/cCgwIQqdGMHMrLGhQ9M8CvfdB3vtlfb9/e9psLxjjy1raG1FwRPTVAo3HzWzVo0f3zQJ3HQT7LcfrFm9U62v9sQ0ZmZtyjHHpAl3NtkkbR98cBooT4J//tP1CM04EZhZ+9S5c+rB/PTTTYfL3n33VI/ws5+lV0rmRGBm7dywYfnhsl9+Od9J7Qc/SK+KJPjXv8obY5k5EZhZ9dhkk1R/8PDD+f4IANtuC717w+uvL/+z7ZgTgZlVn5EjU3+ECLj5ZhgwAN59FzbcEE48MY1/VEWcCMysuh14IEyfDuefn7Yvvji9MrrttvLGVUJOBGZmAN//Pnz6aRryevFiOOAAGDu23FGVhBOBmVmDDh1g9my49da0PW5c6rDWzpubOhGYmTW3//5pyAqA555LzU3bcTJwIjAza8lGG8GcOfntmpo0jEU75ERgZrY8666bRjrdc8+0/cUvwk47tbtWRU4EZmYrUlMDd98Nr72Whql45JG0fO+9ckdWNE4EZmaF6NcvTYwDqXVRz54pKbQDTgRmZoWqrU2Vxj/7WdreaafUzLSNcyIwM1tZp50Gkyal9dtug1/9qrzxrCYnAjOzVbHbbjB5clo/6aR8YmiDnAjMzFbVNtvkm5TusQf86U9lDWdVORGYma2OXXaBv/0N1l4bjjwyjWS6cGG5o1opTgRmZqvrS19KzUshzW3QpUtZw1lZTgRmZsXQo0d+GIoIGDWqrOGsDCcCM7NieuedtLzrLhg0qLyxFMiJwMysmHr3TpPcALzwAtx/f3njKYATgZlZsfXsCVOmpPVdd03jFVUwJwIzsyxsuSV873tpfZ99YMGC8sazAk4EZmZZufBC+MEP4J570pSYFcqJwMwsS+eck0YrnTQJHnus3NG0yInAzCxrM2ak5Y47ppFLK4wTgZlZ1tZfH77xjbQ+cGB5Y2mBE4GZWSlccQX06gWvvgoHHVTuaJpwIjAzK5WGiWz+9jd44onyxtKIE4GZWakMHJgfrXTECHj77bKG08CJwMyslHbZBf73f9P6SSeVN5acTBOBpFGSXpA0XdJpyylziKSpkp6XdF2W8ZiZVYTzzoORI+G66+Dqq8sdTXaJQFItMA7YBxgMHCZpcLMyA4H/A3aKiC2B72YVj5lZRWlIAF//OixaVNZQsnwi2B6YHhEzIuITYALQvGvd0cC4iHgfICLeyTAeM7PKMXBgfq7jjTYqayhZJoINgNcbbc/M7WtsM2AzSQ9LekxSiwN4SzpGUr2k+tmzZ2cUrplZiZ1wQlq+/TaceGLZwih3ZXEHYCCwK3AY8HtJPZoXiojxEVEXEXW9e/cubYRmZlmpqYH589P6xRfDuHHlCSPDc88C+jXa7pvb19hMYGJEfBoR/wFeJCUGM7Pq0KULfPhhWj/hhPwsZyWUZSJ4EhgoaWNJawCHAhOblbmZ9DSApF6kV0UzMozJzKzydO2aWhItXZpGKy2xzBJBRCwGxgJ3AdOAGyLieUlnShqdK3YXMEfSVOBe4JSImJNVTGZmFev449Py2mtL/lSgKMNjyOqoq6uL+vr6codhZlZ8558Pp54Kd9yRJrMpIkmTI6KupWPlriw2M7MGX/96Wp58ckkv60RgZlYpevWC0aNh2jR47rmSXdaJwMyskjT0LWiY77gEnAjMzCrJbrul5aRJJbukE4GZWSWRYM890/o7pRl1x4nAzKzSHHVUWv7hDyW5nBOBmVmlOfjgtPz3v0tyOScCM7NK06ED7LRTmue4BJwIzMwq0frrp+WM7EfdcSIwM6tEP/pRWl56aeaXciIwM6tEQ4ak5YQJmV/KicDMrBJJMGIEvP56GpU0Q04EZmaV6vDD0/LuuzO9jBOBmVmlOuKItLz++kwv40RgZlapevZMrYfeey/TyzgRmJlVsi98AZ56KtNLOBGYmVWyPn3g3XdhyZLMLuFEYGZWyYYNg0WLYPLkzC7hRGBmVsm23jotMxx3yInAzKySrbdeWr74YmaXcCIwM6tkPXtmfgknAjOzSlZbmyqMZ83K7BJOBGZmla5PnzTUREacCMzMKl3PnvDhh5md3onAzKzSde0K8+ZldnonAjOzSte9u58IzMyqWufO8NFHmZ3eicDMrNKtsw68/35m8xI4EZiZVboePdLyrbcyOb0TgZlZpWtIBNOmZXJ6JwIzs0o3YkRatsUnAkmjJL0gabqk01o4PkbSbElP536+mWU8ZmZt0mabwX77Qa9emZy+QyZnBSTVAuOAPYGZwJOSJkbE1GZF/xwRY7OKw8yszevYEW67LbPTZ/lEsD0wPSJmRMQnwATgwAyvZ2ZmqyDLRLAB0HhwjJm5fc0dLOlZSX+R1K+lE0k6RlK9pPrZs2dnEauZWdUqd2XxrUD/iBgK3ANc01KhiBgfEXURUde7d++SBmhm1t5lmQhmAY1/w++b2/eZiJgTEYtym5cD22YYj5mZtSDLRPAkMFDSxpLWAA4FJjYuIGn9RpujgWwayZqZ2XJl1mooIhZLGgvcBdQCV0bE85LOBOojYiJwgqTRwGLgPWBMVvGYmVnLFBHljmGl1NXVRX19fbnDMDNrUyRNjoi6lo6Vu7LYzMzKrM09EUiaDby6ih/vBbxbxHDaAt9zdfA9V4fVueeNIqLFZpdtLhGsDkn1y3s0aq98z9XB91wdsrpnvxoyM6tyTgRmZlWu2hLB+HIHUAa+5+rge64OmdxzVdURmJnZsqrticDMzJpxIjAzq3LtMhEUMDPampL+nDv+uKT+ZQizqAq455MkTc0N+T1J0kbliLOYWrvnRuUOlhSS2nxTw0LuWdIhub/r5yVdV+oYi62Af9sbSrpX0lO5f9/7liPOYpF0paR3JE1ZznFJujj35/GspG1W+6IR0a5+SOMavQxsAqwBPAMMblbmW8ClufVDSbOklT32jO/5i0Dn3Prx1XDPuXJrAw8AjwF15Y67BH/PA4GngHVy258rd9wluOfxwPG59cHAK+WOezXv+QvANsCU5RzfF7gTELAD8PjqXrM9PhEUMjPageTnPvgLsLsklTDGYmv1niPi3oj4KLf5GGlY8Las0BnwzgJ+DnxcyuAyUsg9Hw2Mi4j3ASLinRLHWGyF3HMA3XLr3YE3Shhf0UXEA6RBOJfnQOAPkTwG9Gg2kvNKa4+JoJCZ0T4rExGLgQ+AniWJLhuFzgbX4CjSbxRtWav3nHtk7hcRt5cysAwV8ve8GbCZpIclPSZpVMmiy0Yh9/wT4EhJM4E7gO+UJrSyWdn/763KbBhqq0ySjgTqgF3KHUuWJNUAF1J9Q5t3IL0e2pX01PeApCERMbecQWXsMODqiLhA0o7AtZK2ioil5Q6srWiPTwStzozWuIykDqTHyTkliS4bhdwzkvYATgdGR35muLaqtXteG9gKuE/SK6R3qRPbeIVxIX/PM4GJEfFpRPwHeJGUGNqqQu75KOAGgIh4FOhEGpytvSro//vKaI+JoNWZ0XLb/5Nb/wrwz8jVwrRRhcwGtzVwGSkJtPX3xtDKPUfEBxHRKyL6R0R/Ur3I6Ihoy5NZFPJv+2bS0wCSepFeFc0oYYzFVsg9vwbsDiBpC1IimF3SKEtrIvDfudZDOwAfRMSbq3PCdvdqKAqbGe0K0uPjdFKlzKHli3j1FXjP5wNdgRtz9eKvRcTosgW9mgq853alwHu+C9hL0lRgCXBKRLTZp90C7/lk4PeSvkeqOB7Tln+xk3Q9KZn3ytV7nAF0BIiIS0n1IPsC04GPgK+v9jXb8J+XmZkVQXt8NWRmZivBicDMrMo5EZiZVTknAjOzKudEYGZW5ZwIzFogaYmkpyVNkXSrpB5FPv8ruXb+SJpfzHObrSwnArOWLYyI4RGxFamvybfLHZBZVpwIzFr3KLlBvSQNkPR3SZMlPShpUG7/f0n6m6Rncj8jc/tvzpV9XtIxZbwHs+Vqdz2LzYpJUi1p+IIrcrvGA8dFxEuSRgC/BXYDLgbuj4gv5z7TNVf+GxHxnqS1gCcl3dSWe/pa++REYNaytSQ9TXoSmAbcI6krMJL8MB0Aa+aWuwH/DRARS0hDmwOcIOnLufV+pAHgnAisojgRmLVsYUQMl9SZNM7Nt4GrgbkRMbyQE0jaFdgD2DEiPpJ0H2lANLOK4joCsxXIzep2Amlgs4+A/0j6Knw2d+ywXNFJpClAkVQrqTtpePP3c0lgEGkobLOK40Rg1oqIeAp4ljQByhHAUZKeAZ4nP23iicAXJT0HTCbNnft3oIOkacB5pKGwzSqORx81M6tyfiIwM6tyTgRmZlXOicDMrMo5EZiZVTknAjOzKudEYGZW5ZwIzMyq3P8Hb17ZWpNWFHAAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSP AUPR score: 0.7596229322434146\n"
     ]
    }
   ],
   "source": [
    "precision, recall, aupr_score = aupr(test_y, ood_scores)\n",
    "\n",
    "plot_precision_recall_curve(precision, recall)\n",
    "print('MSP AUPR score:', aupr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSP fpr95 score: 0.5866\n"
     ]
    }
   ],
   "source": [
    "fpr95 = fpr_at_95_tpr(test_y, ood_scores)\n",
    "print('MSP fpr95 score:', fpr95)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OOD detection by Mahalanobis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n",
      "100%|██████████| 98/98 [00:24<00:00,  3.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "covariance norm: 14.235700347525052\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.metrics.mahalanobis import Mahalanobis\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.Resize(img_size),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(dataset_mean, dataset_std),\n",
    "])      \n",
    "cifar100 = CIFAR100(\n",
    "    root=dataset_root, \n",
    "    train=True, \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "id_train_dataloader = DataLoader(\n",
    "    dataset=cifar100, \n",
    "    batch_size=config.train.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "feature_extractor = FeatureExtractor(\n",
    "    model=model,\n",
    "    layer_name=config.model.layer_name.penultimate,\n",
    ")\n",
    "mahalanobis = Mahalanobis(config, model, id_train_dataloader, feature_extractor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:07<00:00, 12.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:07<00:00, 12.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id mean score: 214.82138 id variance: 8392.44\n",
      "ood mean score: 331.5742 ood variance: 8342.834\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "test_y, ood_scores, id_ood_scores, ood_ood_scores = compute_ood_scores(\n",
    "    metric=mahalanobis,\n",
    "    in_dist_dataloader=id_test_dataloader,\n",
    "    out_of_dist_dataloader=ood_test_dataloader,\n",
    ")\n",
    "print('id mean score:', np.mean(id_ood_scores), 'id variance:', np.var(id_ood_scores))\n",
    "print('ood mean score:', np.mean(ood_ood_scores), 'ood variance:', np.var(ood_ood_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABCC0lEQVR4nO3dd5gUxdbA4d9hyUoOBnLOiIAgIgKCJAkmFEWUK4IIKFyzoqh8igkMKKAoCCIqZhEQjIgXFclZBIlLEFyCZDac74/q1WHdMAs70zs7532eeSZ0TffpCX26q7qrRFUxxhgTvXL5HYAxxhh/WSIwxpgoZ4nAGGOinCUCY4yJcpYIjDEmylkiMMaYKGeJIIcTkdUi0srvOLILEXlIRN7wadmTROQJP5ad1USkp4h8eYrvtd9kNmOJIIxEZLOIHBWRQyKyy9swnBnKZapqHVWdG8plJBORfCLylIhs9dZzvYjcKyISjuWnEk8rEYkNfE1VR6jqrSFanojInSKySkQOi0isiHwgIvVCsbxTJSKPicjbpzMPVZ2qqu2CWNa/kt+p/iZFJK8X+3rv890sIhNFpGJm52VOZokg/Lqo6plAA+B84EF/w8k8EcmdxqQPgDZAJ6AQ0AvoB7wUghhERLLb7/clYDBwJ1AcqA58Clye1QtK5zsIOR+X/SHQFbgBKAKcByzG/eYyxc/PL1tSVbuF6QZsBtoGPH8WmBnw/ELgR2A/sBxoFTCtOPAmsAPYB3waMK0zsMx7349A/ZTLBM4FjgLFA6adD/wJ5PGe3wKs9eY/B6gQUFaBgcB6YFMq69YGOAaUS/F6UyARqOo9nws8BfwC/AV8liKm9D6DucCTwHxvXaoC//FiPghsBG7zyp7hlUkCDnm3c4HHgLe9MhW99boZ2Op9FkMDllcAmOx9HmuB+4DYNL7bat56Nknn+58EjAFmevEuAKoETH8J2OZ9LouBFgHTHsNtCN/2pt8KNAF+8j6rncArQN6A99QBvgL2An8ADwEdgBNAvPeZLPfKFgEmePPZDjwBxHjTenuf+QtAnDetN/A/b7p403Z7sa0E6uJ2AuK95R0CPk/5PwBivLh+9z6TxaT4DXnl2nrf57+mpfP/Su277uN91/OAL4BBKeaxHLjKe1wz4PNbB1zr9zYkZNsmvwOIpluKP0BZ7w/zkve8jPcn64Q7UrvMe17Kmz4TmAYUA/IALb3Xz/f+gE29P9XN3nLypbLMb4G+AfE8B7zqPe4GbABqAbmBh4EfA8qq96coDhRIZd2eBr5PY7238M8Geq63oamL21h/FPBnzegzmOv9iet4MebB7W1XwW2MWgJHgIZe+Vak2HCnsXF4HbfRPw84DtQKXCfvMy8LrEg5v4D59ge2ZPD9T/LWp4kX/1TgvYDpNwIlvGl3A7uA/AFxxwNXeJ9NAaARLnHm9tZlLTDEK18It1G/G8jvPW+a8jMIWPYnwGved1Ial6iTv7PeQAJwh7esApycCNrjNuBFve+hFnBOwDo/kc7/4F7c/6CG997zgBKZ+X2lNt90vuu3vHUsANwEzA8oXxuXVPN5ZbbhdjRy889OU22/tyOhuGW3Q+to8KmIHMT9yHYDj3qv3wjMUtVZqpqkql8Bi4BOInIO0BHor6r7VDVeVb/33tcPeE1VF6hqoqpOxm3MLkxl2e8A14OrWgF6eK+B25A9paprVTUBGAE0EJEKAe9/SlX3qurRVOZdErfhSc1Ob3qyKaq6SlUPA48A14pITHqfQcB7J6nqalVN8D6Hmar6uzrfA18CLdKIIy2Pq+pRVV2O2yM8z3v9WmCE95nHAqPTmUeJdNY/0Ceq+ov3GU/FVRECoKpvq2qct26jcBukGgHv/UlVP/U+m6OqulhVf/bKb8ZtyFt6ZTsDu1R1lKoeU9WDqrogtYBE5CzcZzxEVQ+r6m7cHn6PgGI7VPVlb1kpv/94XKKpCYj3GwrmswB3ZPOwqq7zvsPlqhqXSrlgP9+MPOat41Fc8gv8jfcEPlbV47jPb7Oqvumt81LcTkv3LIgh27FEEH5XqGoh3N5qTf7ZQFYAuovI/uQbcDFwDlAO2Kuq+1KZXwXg7hTvK4erBknpI6CZl1guwVWb/BAwn5cC5rEXt4dWJuD929JZrz+9WFNzjjc9tflswe3ZlyT9zyDVGESko4j8LCJ7vfKdODnpBGNXwOMjQHID/rkplpfe+seR9voHsyxE5B4RWSsiB7x1KcLJ65Jy3auLyAzvxIO/cMk7uXw5XHVLMCrgvoOdAZ/7a7gjg1SXHUhVv8VVS40BdovIeBEpHOSyg40z2M83I3+vh6oexB1pJye863HJGdxn0jTFb7EncHYWxJDtWCLwibf3OgkY6b20DbenXDTgdoaqPu1NKy4iRVOZ1TbgyRTvK6iq76ayzH24PebrcA1u76mqBsznthTzKaCqPwbOIp1V+hr3xykX+KKINMX92b8NeDmwTHncHuWfGXwG/4pBRPLhkttI4CxVLQrMwiWwjOINxk5clVBqcaf0DVBWRBqfyoJEpAWuDeJaoJi3Lgf4Z13g3+szDvgVqKaqhXF17cnltwGV01hcyvlswx1Flgz43Aurap103nPyDFVHq2ojXPVKdVyVT4bv85ZdJYMy4H5fTUSkbDplDgMFA56nttFOGc+7wPUi0gxXhfZdQFzfp/gtnqmqtwcRa8SxROCvF4HLROQ8XCNgFxFpLyIxIpLfO/2xrHeY/QUwVkSKiUgeEbnEm8frQH8RaeqdSXOGiFwuIoXSWOY7uLrRa/inWgjgVeBBEakDICJFRCTow2BV/Rq3MfxIROp463Cht17jVHV9QPEbRaS2iBQEhgMfqmpiep9BGovNi6s+2QMkiEhHIPCUxj+AEiJSJNj1SOF93GdSTETKAIPSKuit31jgXS/mvF78PUTkgSCWVQhXD78HyC0iw4CM9qoL4RpnD4lITSBwIzUDOEdEhog7rbeQl5TBfS4Vk8+68n5fXwKjRKSwiOQSkSoi0pIgiMgF3u8vD25jfAx3tJm8rLQSEsAbwP+JSDXv91tfREqkLOT9vr4CPhGRRiKS21un/iJyi1dsGdDD+380xv3GMzILt/c/HJimqslxzwCqi0gvb355vPWsFcQ8I44lAh+p6h5c49UwVd2Ga7B9CLcx2Ibbq0r+jnrh9px/xbUtDPHmsQjoizs034dr8O2dzmKn485w2eXViSfH8gnwDPCeV82wCtcukRlX4/aoZuPOEnkbdybKHSnKTcEdDe3C7YXd6cWQ0WdwEu/Q/k7cBnsf7ihnesD0X3F7fBu9w/vUqsvSMxyIBTbh9kg/xO05p+VO/qki2Y+r8rgS+DyIZc3BfW6/4arLjpF+VRTAPbh1PojbIZiWPMH7bC4DuuA+5/VAa2/yB959nIgs8R7fhEusa3Cf5YcEXxVT2Fv+Pi/2ONyJCOC+/9re5/9pKu99Hvf9fYlLahNwDbmpuQa34Z6GO1paBTTGfTfg2puqeHE8zsk7Oqny2gM+xp2V9E7A6wdxOxU9cGfq7cL9P/JlNM9IJP/UDBgTeiIyF3cmhy9X954OEbkd6KGqQe0pGxMp7IjAmDSIyDki0tyrKqmBOxXzE7/jMiar2dV1xqQtL+7smUq4qp73cO0AxuQoVjVkjDFRzqqGjDEmykVc1VDJkiW1YsWKfodhjDERZfHixX+qaqnUpkVcIqhYsSKLFi3yOwxjjIkoIrIlrWlWNWSMMVHOEoExxkQ5SwTGGBPlIq6NIDXx8fHExsZy7Ngxv0MJmfz581O2bFny5MnjdyjGmBwmRySC2NhYChUqRMWKFRF/hscNKVUlLi6O2NhYKlWq5Hc4xpgcJmRVQ+IGld4tIqvSmC4iMlpENojIChFpeKrLOnbsGCVKlMiRSQBARChRokSOPuIxxvgnlG0Ek3Djo6alI64XzGq4UbbGnc7CcmoSSJbT188Y45+QVQ2p6jwRqZhOkW7AW97AKD+LSFEROScTQ9wZY0xkOXECDh+GY8fgzz/d/R9/QFKSm3b8OGzbBoUKQXw8JCRAfDyH44+wJ+EAFbv0ggsuyPKw/GwjKMPJ/a3Heq/9KxGISD/cUQPly5cPS3CZFRMTQ7169UhISKBSpUpMmTKFokWLArB69WruuOMOtm/fTlJSEjfddBMPP/zw33v5X3zxBY888ghHjhwhX758XHrppYwaNcrHtTEmiiUkwJEjcPQoHDzoHp844W579oCI24jv2gVbt0LevBAX526rVkHx4q7skSNuo37mmW6ep1i1+20l6NsFihyHRYtqkiuHJYKgqep4YDxA48aNs2UveQUKFGDZsmUA3HzzzYwZM4ahQ4dy9OhRunbtyrhx42jXrh1Hjhzh6quvZuzYsQwcOJBVq1YxaNAgZs6cSc2aNUlMTGT8+PH+rowxOUF8vNs479rlbps2gSrs3g0rV8L27ZCY6DbqBw78s7d+KipWhBIloIw3xHft2i5BJCa6xHHuue75iRNQtKgrm5AAZctC4cKQK5dLIPnyQZ48ULAg+/Uo9/74GG+snEzVYlV5ocvr5KrUKms+mxT8TATbOXkM2LLeaxGvWbNmrFixAoB33nmH5s2b066dG0GxYMGCvPLKK7Rq1YqBAwfy7LPPMnToUGrWrAm4I4vbb8+Rw6Iac/pU4a+/3MZ80yZYutRVq/zxByxf7qbv2wfe/y9NhQu7RHHOOXDhhVCsGBQo4DbE8fFuL75UKShY0JVN3kDnzes22qVKudfPOgtyZ/1mNDEpkYvG1WNd3Druu+g+Hmv1GAXypDVw2+nzMxFMBwaJyHtAU+BAlrQPDBkC3p55lmnQAF58MaiiiYmJfPPNN/Tp0wdw1UKNGjU6qUyVKlU4dOgQf/31F6tWreLuu+/O2niNiWSqbg9+3TrYvBnWrIFFi2D+fLdHnZozznB71IcOQbNm7qYKdeq4jXidOm5PvFgxtxHPmzecaxS0uCNxFC9QnJhcMTx56ZOUK1KOxuc2DvlyQ5YIRORdoBVQUkRigUeBPACq+ipu7NFOuDF2jwD/CVUs4XD06FEaNGjA9u3bqVWrFpdddpnfIRmTPf35J6xf726rV7t69B073J78r7/+e2Mv4vbcr77abdzLlIGaNd2Gv0EDN61A6PaWw0FVmbpyKoNnD+bpNk/Tt1Ffrqx1ZdiWH8qzhq7PYLoCA7N8wUHuuWe15DaCI0eO0L59e8aMGcOdd95J7dq1mTdv3kllN27cyJlnnknhwoWpU6cOixcv5rzzzvMlbmOy3NGjbiO/bp2rutm3z238581zVTqpyZMHSpeGxo3dhr5ePShXDpo2hUqVIH/+8K5DGG07sI3+M/sza/0sLix7Ic3LNw97DBHRWBxJChYsyOjRo7niiisYMGAAPXv2ZMSIEXz99de0bduWo0ePcuedd3LfffcBcO+993LVVVdx8cUXU716dZKSkhg/fjz9+/f3eU2MSUHVnUWzfTts2eJOdVyzxtXP//WXu1+1yp1Jk1KlSm4vvl49qFIFOnaEatXc3nyxYm6vPwq9u/JdbptxG4mayIvtX2RQk0HE5IoJexyWCELg/PPPp379+rz77rv06tWLzz77jDvuuIOBAweSmJhIr169GDRoEAD169fnxRdf5Prrr+fIkSOICJ07d/Z5DUzUO34c5s6FH35wG/b16121zf79qZfPndvVw9erB127Qq1aULkyXHSROyc+Sjf0GSlWoBhNyzZlfOfxVCrmX/cxETdmcePGjTXlwDRr166lVq1aPkUUPtGynibM9u+HSZPgl1/cnv369fDbbyeXuegitwdfo4arj69a1e3Jly7tqnBycNVNVkpISuCFn17gROIJhl4yFHDtA+HoOUBEFqtqqi3PdkRgTLRQdWfhbNrkqnCWL4cFC9yefmKiK5M/P3TqBN26uXPjO3Z01TrmtC3ftZw+0/uweOdirq1z7d8JIDt0H2OJwJic5sQJ+OYbt5Ffv96depmQABs3/rts0aJuo3/bbdC8uTsN02Sp4wnHeWLeEzw9/2mKFyjOB90/4OpaV2eLBJAsxySCcB1e+SXSqvBMmJw4AWvXws8/w+zZrnpnx46Ty5Qr586dHzgQSpb858ycunXd2TompNbvXc8z85/hhno38Hy75ylRsITfIf1LjkgE+fPnJy4uLsd2RZ08HkF+q4eNXnv2wOefu7r7uDjXj81777mrapOddRa0aOHq7ytUgM6d3Vk5MeE/CyXaHTpxiM9+/Yye9XtSt3Rdfh30K5WLVfY7rDTliERQtmxZYmNj2bNnj9+hhEzyCGUmCiQluS4Svv8ePvvMXXC1YcPJZUqXdlU5hQvDVVe58+1r17azc7KBr37/in4z+rFl/xYantOQWqVqZeskADkkEeTJk8dG7jKRSdVV6Uyb5hpsd+xw1TxHjvxTpkwZGDrUnbnTrJk7W8dkO/uO7uOeL+9h4rKJVC9Rne97f0+tUpFxll+OSATGRARVdyHWd9+58/MXLHBX3yafsQPQqJE7U+eCC1xdfosW2bZfHPOPxKREmk9szm9xv/HgxQ8yrOUw8ueOnKpcSwTGhELyVbfLlsHOnTB1qjuDJz7+nzIlS8Kll8I118C117ozeExE+fPIn393EjeizQjKFylPw3NOedRd31giMCYrJCbCp5/CjBnu4qyUSpd2F2Tdeqs7i+eyy6BIkXBHabKIqjJlxRSGzB7C022fpl+jflxR8wq/wzpllgiMOVXx8a4Rd+pUePLJf15v08bt3Xfo4Pq6r1jR9W9vcoQt+7dw24zbmPP7HC4qdxGXVLjE75BOmyUCY4K1bx98/bU7o2f0aNcdQ7KWLaFLF7j9djeYicmR3l7xNrfPvB1V5eWOLzPgggHkklx+h3XaLBEYk5Z16+DHH91G/48/XF1/sjp14OyzXcNu69bQMPLqhU3mlSpYiublmvNa59eoULSC3+FkGUsExiRLSoIlS+Ddd90pnWvWuNfz53djzN57L5x/PrRv7wZFMTlefGI8o34aRXxiPI+0fIT2VdvTrkq7HHfhqiUCE9127nT98kyc6E7rBNelcvPmMHw4tG3rLtbKFfmH/yZzlu5cSp/pfVi6ayk96vbIVp3EZTVLBCa6JCW5K3Y/+QRmzjy5I7Z69dxwiLfd5qp9TFQ6lnCM4d8P59n5z1KyYEk+uvYjrqp1ld9hhZQlAhMd3n8fBg92Db7Hj7vXLr4Yevd2V+s2beoGUDFRb8PeDYz8cSQ3nXcTo9qNoliBnH8ltyUCk3Opuit4777bdcUMrpF38GC44grXI6cxuE7iPln7Cb3O60Xd0nVZN2idryOGhZslApPzHDsGs2bBww+7LpoB+veHkSOtv33zL3M2zKHfjH5sO7CNxuc2plapWlGVBACsBczkHPv3uz73zzzT1fXv3g333ecahMeNsyRgThJ3JI6bP72ZDlM7UDBPQX74zw8R00lcVrMjAhP5EhLc3v7//Z/rtbNCBXjqKdc9c758fkdnsqHkTuI27N3A0BZDefiShyOqk7isZonARKbERFf//9prboAWcGPrTpniTv00JhV7Du+hRMESxOSK4Zm2z1ChaAUanN3A77B8Z1VDJrLMmwc9ekCBAu6K3vfeg65dYcIEN3qXJQGTClXlzaVvUv2V6ry++HUAutXsZknAY0cEJvs7dgyefx7eeQdWr3bj7FavDn37wk032UAtJl2b92+m3+f9+GrjV7Qo34LWlVr7HVK2Y4nAZF8nTsD998OLL7rnRYrAyy/DzTfbOf8mKFOWT+H2mbcjIoztNJbbGt+WIzqJy2qWCEz2s38/vPWWO98f3ODrb74JN9xgA7GbTDnrzLO4pMIlvNr5VcoXKe93ONmWJQKTPai6Lp6ffdbdA9SoAXfdBbfc4vr/MSYD8YnxPDv/WRI1kWEth9GuSjvaVWnnd1jZnv27jP+2bYPy3t5awYKuwffRR90AL9bZmwnSkp1LuOWzW1j+x3JuqHfD353EmYzZv8z4JykJXnoJqlZ1z5s1cxeB/e9/bihHSwImCEfjj/LA1w/Q5PUm/HH4Dz657hOmXjXVkkAmhPSfJiIdRGSdiGwQkQdSmV5eRL4TkaUiskJEOoUyHpNNqMIzz7i9/yFDXKPwt9+6QWDs6l+TSRv3beT5n56nd4PerBmwJqLHDvZLyBKBiMQAY4COQG3gehGpnaLYw8D7qno+0AMYG6p4TDYxcaI7AnjgAdcL6IgRcPSouybAmCD9dfwvJi2bBECd0nVYf8d63uj6RlT0FBoKoTwiaAJsUNWNqnoCeA/olqKMAoW9x0WAHSGMx/gpKQkGDYI+fWDrVhg2zB0JPPigGwHMmCDNWj+LumPr0md6H9bucZ0K5qRhI/0QysbiMsC2gOexQNMUZR4DvhSRO4AzgLapzUhE+gH9AMqXt1PAIsr+/TB1Kjz9NMTGus7g3nrLBng3mfbnkT/575z/8vaKt6ldqjbzu8+P2k7isprfZw1dD0xS1VEi0gyYIiJ1VTUpsJCqjgfGAzRu3Fh9iNNk1v798NBD8PrrrlM4gF69YPJksEY8k0nJncRt3LeRYZcM46EWD5Evt3UomFVCmQi2A+UCnpf1XgvUB+gAoKo/iUh+oCSwO4RxmVBKTHQb+wcegD174KyzYPRodyRgF4OZTPrj0B+UOqMUMbliGHnZSCoUrUD9s+r7HVaOE8o2goVANRGpJCJ5cY3B01OU2Qq0ARCRWkB+YE8IYzKhtHMn1Krl2gEKFoQPPoBdu+Daay0JmExRVSYsmUCNV2owfvF4ALrU6GJJIERClghUNQEYBMwB1uLODlotIsNFpKtX7G6gr4gsB94FequqVf1Eop9/hvr1Yf16ePxx+P13uOYav6MyEWjjvo20ndKWWz+/lQZnN6Bt5VSbDk0WCmkbgarOAmaleG1YwOM1gPUbHMlUXadw997ruoGYNs0dARhzCiYvm8yAWQOIkRhevfxV+jbqa53EhYHfjcUmkq1eDZ07w+bNcP75LglUq+Z3VCaCnVvoXC6tdCnjLh9H2cJl/Q4nalgiMJmn+k/voAcPuovC7r/fuoQwmXYi8QRP/+9pkjSJx1o9xmVVLuOyKpf5HVbUsURgMic+3o0FPGOGez53LrRs6WtIJjIt3L6QW6bfwqrdq+hVv5d1Eucj24UzwYuLg6ZNXRLo1QsOH7YkYDLtSPwR7vnyHi6ccCH7ju5jeo/pvHXlW5YEfGRHBCY4v/8Ol1wCO3bAffe5TuOMOQWb9m3i5V9epm/DvjzT9hmK5C/id0hRzxKBydj06dC7N+zb564NsNNCTSYdOHaAj9d+zH/O/w91Stdhwx0bKFekXMZvNGFhVUMmbXv3usFhunl9BX72mSUBk2kzf5tJnbF1uPXzW/n1z18BLAlkM5YITOq+/hpKlHDjBFxyCWzZAl27Zvw+Yzx7Du+h58c96fxuZ4oVKMZPfX6iZsmafodlUmFVQ+ZkiYmua+jnnnPPZ8+G9u39jclEnMSkRC5+82I27dvE460e54GLHyBvTF6/wzJpsERg/rFkCdx4I6xdCxdc4LqPtgvETCbsOrSL0meUJiZXDKPajaJi0YrULV3X77BMBoKuGhIR60A+pzpxAm67DRo1cklg1ChYsMCSgAlakibx2qLXqP5ydV5b9BoAnat3tiQQITI8IhCRi4A3gDOB8iJyHnCbqg4IdXAmDD74AK67zl0tfNZZriqoQQO/ozIRZMPeDfT9vC9zN8/l0kqX0r6qVSVGmmCOCF4A2gNxAKq6HLgklEGZMFB14wbcfLN7PG6c60bakoDJhDeXvkm9cfVYsnMJr3d5na97fU3lYpX9DstkUlBtBKq6LcVVf4mhCceETd++MGECVKkCn34Kde0Q3mRe+SLlaV+lPWM6jaFM4TJ+h2NOUTCJYJtXPaQikgcYjBtfwESi48fdWUDffw+lS8Ovv7ruo40JwvGE4zz1v6dI0iSGtx5Om8ptaFO5jd9hmdMUTNVQf2AgbjD67UADwNoHIlFCgusw7vvvXZXQli2WBEzQFsQuoNH4Rjz+/eNsPbAVG0Mq5whmK1BDVXsGviAizYH5oQnJhMSuXe6U0NhYGDAAxozxOyITIQ6fOMwj3z3Ciz+/SJnCZZhx/Qwur36532GZLBTMEcHLQb5msqtly1wjcGwsDBsGr7zid0Qmgmw5sIWxC8fSv3F/Vg9YbUkgB0rziEBEmgEXAaVE5K6ASYUBG4k8UiQkuK6jT5yAmTOhUye/IzIRYP+x/Xy45kNubXgrtUvVZsOdG2zEsBwsvaqhvLhrB3IDhQJe/wuwnscixeDBLgk88oglAROUz379jNtn3s7uw7u5uPzF1CxZ05JADpdmIlDV74HvRWSSqm4JY0wmq4waBWPHQp068Nhjfkdjsrndh3dz5xd3Mm31NOqfVZ/p10+3TuKiRDCNxUdE5DmgDpA/+UVVvTRkUZnTd9NNMGUK1KvnehC18YRNOhKTEmk+sTlbD2zlidZPcF/z+8gTk8fvsEyYBJMIpgLTgM64U0lvBvaEMihzGg4ehFtvhfffd11GLF4MeewPbVK34+AOzj7zbGJyxfBSh5eoWLQitUvV9jssE2bB7CaWUNUJQLyqfq+qtwB2NJAdxcZCw4YuCfTpA2vWWBIwqUrSJMYtHEfNV2ry6qJXAehUrZMlgSgVzBFBvHe/U0QuB3YAxUMXkjklCxdCkyYQE+O6j77hBr8jMtnUb3G/0ffzvszbMo+2ldvSsWpHv0MyPgsmETwhIkWAu3HXDxQGhoQyKJNJ27a5JADw8cc2kphJ04QlExj0xSDy587PxK4T6d2gNyn6ETNRKMNEoKozvIcHgNbw95XFJjtQhf793eMJEywJmHRVLFqRjlU7MqbTGM4pdI7f4ZhsIr0LymKAa3F9DM1W1VUi0hl4CCgAnB+eEE26rroKZs2C++6DW27xOxqTzRxPOM7/zfs/AJ649AnrJM6kKr0jgglAOeAXYLSI7AAaAw+o6qdhiM1k5N57XRfSrVrB00/7HY3JZn7c9iN9pvfh1z9/5ZYGt6CqVg1kUpVeImgM1FfVJBHJD+wCqqhqXHhCM+n66CMYOdKNJzBzJtgf3HgOnTjE0G+G8vIvL1OuSDlm95xto4aZdKV3+ugJVU0CUNVjwMbMJgER6SAi60Rkg4g8kEaZa0VkjYisFpF3MjP/qLVrlxtkvlQpN+B8QRtO2vxj64GtvLb4NQZeMJBVt6+yJGAylN4RQU0RWeE9FqCK91wAVdX66c3Ya2MYA1wGxAILRWS6qq4JKFMNeBBorqr7RKT0aaxLdDh61FUFHTsGX30FhQv7HZHJBvYd3ccHaz6gX6N+1C5Vm42DN3JuoXP9DstEiPQSQa3TnHcTYIOqbgQQkfeAbsCagDJ9gTGqug9AVXef5jJztsRE6NYN1q2Dd96Biy/2OyKTDXyy9hMGzBrAnsN7aFmhJTVK1rAkYDIlvU7nTrejuTLAtoDnsUDTFGWqA4jIfFzX1o+p6uyUMxKRfkA/gPLly59mWBFsxAh3FDBsGFx/vd/RGJ/tOrSLO764gw/XfEiDsxsw84aZ1ChZw++wTATye5zC3EA1oBVQFpgnIvVUdX9gIVUdD4wHaNy4cXSOj/fDDy4BtG1rPYkaEpMSafFmC7Yd2MaIS0dwz0X3WCdx5pSFMhFsx51+mqys91qgWGCBqsYDm0TkN1xiWBjCuCLP1q1wySVQtChMnGhnCEWx2L9iObfQucTkimF0h9FUKlbJuoo2py2ovolFpICIZPaYcyFQTUQqiUheoAcwPUWZT3FHA4hISVxV0cZMLifna9XK3b/5JpQrl25RkzMlaRIvL3iZmq/UZNzCcQB0rNbRkoDJEhkmAhHpAiwDZnvPG4hIyg36v6hqAjAImAOsBd5X1dUiMlxEkvtBmAPEicga4DvgXrtOIYUFC2DTJteJ3BVX+B2N8cGvf/7KJW9ewp2z7+Ti8hfTuXpnv0MyOYyopl/lLiKLcd1Oz1XV873XVqpqvTDE9y+NGzfWRYsW+bFofzRsCEuXwvbtcK6dCRJt3ljyBoNmDaJgnoK82OFFetXvZVcHm1MiIotVtXFq04LqhlpVD6T48UVng224jRjhksAdd1gSiFJVilWhS40uvNLxFc468yy/wzE5VDCJYLWI3ADEeBeA3Qn8GNqwDOPHw9Ch0KgRPPus39GYMDmWcIzh3w8HYESbEbSu1JrWlVr7HJXJ6YJpLL4DN17xceAdXHfUQ0IYk1m9GgYPdo/nz4f8+dMvb3KE+Vvn0+DVBjz1v6fYc3gPGVXbGpNVgjkiqKmqQ4GhoQ7G4PoRatnSjTPw5ZeQL5/fEZkQO3j8IA998xBjFo6hQtEKzLlxDu2qtPM7LBNFgkkEo0TkbOBDYJqqrgpxTNGte3c3AP3338OFF/odjQmD2L9ieWPpG9zR5A6ebPMkZ+Y90++QTJTJsGpIVVvjRibbA7wmIitF5OGQRxaNPv4Y/vc/6NnTkkAOF3ck7u/rAWqVqsXGOzfyUseXLAkYX2R4+uhJhUXqAfcB16lq3pBFlY4ce/ro3r1QvjwUKuQ6lbNeRXMkVeWjtR8xcNZA9h7dy6rbV1n/QCYs0jt9NJgLymqJyGMishI3eP2PuO4iTFZRhfbt4fBhmDzZkkAOtfPgTq5+/2q6f9CdcoXLsajvIksCJlsIpo1gIjANaK+qO0IcT3Tq3h0WLXLXC7SzRsKcKLmTuO0Ht/Ns22f5b7P/kjuX330+GuNk+EtU1WbhCCRqLVnihp1s1AheesnvaEwW23ZgG2UKlyEmVwxjOo2hUrFKVC9R3e+wjDlJmlVDIvK+d79SRFYE3FYGjFxmTtewYZAnj407nMMkJiUyesFoao75p5O49lXbWxIw2VJ6RwTeFU1YD1ehsmCBSwB9+8JZ1n1ATrF2z1r6TO/DT7E/0bFqR7rU6OJ3SMakK80jAlXd6T0coKpbAm/AgPCEl4MlJUGPHpArlw00k4OMXzyeBq814Le435hy5RRm3jCT8kWieFQ9ExGC6WLislRe65jVgUSdyZNh82brUC6HqVa8GlfWvJI1A9dwY/0bradQExHSvI5ARG7H7flXBn4PmFQImK+qN4Y+vH/LEdcRHDwIlSq5PoQ2bXJtBCYiHY0/ymNzH0NEeLrt036HY0yaTrUb6neAL4CngAcCXj+oqnuzML7o06sXxMXBF19YEohg87bM49bpt7J+73r6N+qPqtoRgIlI6VUNqapuBgYCBwNuiEjx0IeWQw0fDp99BtdcAx06+B2NOQV/Hf+LATMH0HJSSxI1kW9u+oZxncdZEjARK6Mjgs7AYtxANIG/csVVGZnMUIUxY9zjN9/0NxZzynYc3MGkZZO468K7GN56OGfkPcPvkIw5LWkmAlXt7N1XCl84Odyzz8Lu3W7ksTOtc7FI8ueRP3l/9fsMuGAANUvWZNPgTTZimMkxgulrqLmInOE9vlFEnhcROx8us1asgIcecp3K3Xef39GYIKkq01ZNo/aY2gyZPYTf4n4DsCRgcpRgTh8dBxwRkfOAu3FnEE0JaVQ5TWKiaxNISoKFCyEmxu+ITBB2HNzBFdOuoMdHPahQtAKL+y22K4NNjhRMr1cJqqoi0g14RVUniEifUAeWozz/PKxf74afrGG9TUaCxKRELnnzErYf3M7Iy0Yy+MLB1kmcybGC+WUfFJEHgV5ACxHJBdg5j8Hatg0eeQQqV4YXXvA7GpOBLfu3ULZwWWJyxTD28rFULlaZqsWr+h2WMSEVTNXQdbiB629R1V24sQieC2lUOcl117kqobfesk7lsrHEpESe/+l5ao2pxbhFrpO4dlXaWRIwUSGYoSp3AVOBIiLSGTimqm+FPLKc4NNP4aef4LbboHlzv6MxaVi1exUXTbyIu7+8mzaV23BFzSv8DsmYsArmrKFrgV+A7sC1wAIRuSbUgUW8xES48kr3+Kmn/I3FpOnVRa/S8LWGbNy3kXeueofpPaZTtrANwGeiSzBtBEOBC1R1N4CIlAK+Bj4MZWAR7/773X2vXnbNQDaU3B1ErZK16F6nOy+2f5FSZ5TyOyxjfBFMIsiVnAQ8cQTXthC9/vc/GDUKunVzbQMm2zgSf4Rh3w0jRmJ45rJnaFmxJS0rtvQ7LGN8FcwGfbaIzBGR3iLSG5gJzAptWBHsxAno6PXS/fLL/sZiTjJ381zqj6vPqJ9GcejEIdLqedeYaBPMmMX3ishVwMXeS+NV9ZPQhhXBXn0VDh1yVUPlyvkdjQEOHDvAfV/dx/gl46lSrArf3vQtrSu19jssY7KNNBOBiFQDRgJVgJXAPaq6PVyBRaxRo+Ccc+Bp65s+u9h5aCdvr3ybe5rdw+OtH6dgnoJ+h2RMtpJe1dBEYAZwNa4H0kzXc4hIBxFZJyIbROSBdMpdLSIqIqkOmhAxFi2CrVuhVSu/I4l6ew7v4eUF7idbs2RNNg/ezHPtnrMkYEwq0qsaKqSqr3uP14nIkszMWERigDG4oS5jgYUiMl1V16QoVwgYDCzIzPyzpXHuQiSef97fOKKYqvLuqne584s7+ev4X7Sv2p7qJarbGUHGpCO9I4L8InK+iDQUkYZAgRTPM9IE2KCqG1X1BPAe0C2Vcv8HPAMcy3T02cmBAzBxIhQvDmef7Xc0UWnbgW10ebcLPT/uSdXiVVl621LrJM6YIKR3RLATCNy13RXwXIFLM5h3GWBbwPNYoGlgAS+hlFPVmSJyb1ozEpF+QD+A8uWzaQ/YQ4e6+4ce8jeOKJWQlECrya3YdWgXL7R/gTua3EFMLuvl1ZhgpDcwTUhPq/A6r3se6J1RWVUdD4wHN3h9KOM6Jaowdap7fNdd/sYSZTbv30y5wuXInSs3r3V+jcrFKlO5mA2eZ0xmhPLCsO1A4PmTZb3XkhUC6gJzRWQzcCEwPSIbjF94AfbvdyOPWcdyYZGQlMDIH0dSa0wtxi4cC0Dbym0tCRhzCkLZwfpCoJqIVMIlgB7ADckTVfUAUDL5uYjMxZ2iuiiEMYXG11+7+wfSPDHKZKEVf6ygz/Q+LNqxiG41unF17av9DsmYiBayRKCqCSIyCJgDxAATVXW1iAwHFqnq9FAtO6w2boQvvoDu3e1oIAzGLhzL4NmDKZa/GNOumUb32t0R+9yNOS0ZJgJx/7KeQGVVHe6NV3y2qv6S0XtVdRYpuqNQ1WFplG0VVMTZTfKpogMH+htHDpfcSVzd0nXpUbcHL7R/gZIFS2b8RmNMhiSj/lZEZByQBFyqqrVEpBjwpapeEI4AU2rcuLEuWpRNao+OH4dzz4XzzoNvv/U7mhzp8InDPPztw+TOlZvn2tl4SMacKhFZrKqptsEG01jcVFUH4p3nr6r7gLxZGF/k+uAD2LsXevb0O5Ic6ZuN31BvXD1eXPAixxOPWydxxoRIMG0E8d5Vwgp/j0eQFNKoIsWUKRATY4kgi+0/tp97vryHCUsnUK14Neb1nkeLCi38DsuYHCuYI4LRwCdAaRF5EvgfMCKkUUWCvXvhyy/h5pshf36/o8lR/jj0B++teo/7m9/P8v7LLQkYE2LBdEM9VUQWA20AAa5Q1bUhjyy7G+vOXefaa/2NI4dI3vgPvnAwNUrWYPOQzdYYbEyYBHPWUHngCPB54GuqujWUgWVrS5bAI4/AhRdCu3Z+RxPRVJWpK6cyePZgDp04RKdqnahWopolAWPCKJg2gpm49gEB8gOVgHVAnRDGlb2NHu3uJ060awdOw9YDW+k/oz9fbPiCZmWbMaHrBKqVqOZ3WMZEnWCqhuoFPvc6ihsQsoiyu4MH3TjE7dtDrVp+RxOxEpISaDWpFbsP72Z0h9EMuGCAdRJnjE8yfWWxqi4RkaYZl8yhhg93nczdf7/fkUSkjfs2UqFIBXLnys3rXV6nSvEqVCxa0e+wjIlqwbQRBHanmQtoCOwIWUTZ2d69MHIktG7tbiZoCUkJjPpxFI/OfZRnL3uWO5veSZvKbfwOyxhDcEcEhQIeJ+DaDD4KTTjZ3M03u/u77/Y3jgizbNcy+kzvw5KdS7iy5pV0r93d75CMMQHSTQTehWSFVPWeMMWTfX33HcyYAS1awOWX+x1NxHjll1f475z/UqJACT7s/qH1FGpMNpRmIhCR3F4Pos3DGVC29bo3fPP77/sbR4RI7iSu/ln16VmvJ8+3f57iBYr7HZYxJhVpdjonIktUtaHX6VwZ4APgcPJ0Vf04PCGezLdO5woVcuMRb9kS/mVHkEMnDjH0m6HkicnDyHYj/Q7HGOM53U7n8gNxuDGKOwNdvPvo8f77cOgQXHON35Fka1/+/iV1x9bl5V9eJj4x3jqJMyZCpNdGUNo7Y2gV/1xQliy6/uFDhrj7J57wNYzsat/Rfdz15V1MWjaJGiVqMO8/87i4/MV+h2WMCVJ6iSAGOJOTE0Cy6EkEhw/Dzp3QpAkUKOB3NNnS7sO7+XDNhzx48YMMazmM/LmtEz5jIkl6iWCnqg4PWyTZ1YQJ7r53b1/DyG52HdrFuyvf5b/N/us6iRu8mRIFS/gdljHmFKTXRmCd6AC88IK779vX3ziyCVVl8rLJ1B5Tmwe/eZD1cesBLAkYE8HSSwR22eeuXbB5MzRoALkz3RtHjrN5/2Y6TO1A7896U7tUbZb1X2adxBmTA6S5dVPVveEMJFt67DF3P9xqyBKSEmg9uTV/HvmTMZ3G0L9xf3JJMCedGWOyO9vNTc/EiVCpEnTp4nckvtmwdwOVilYid67cTOw6kcrFKlOhaAW/wzLGZCHbpUvLypUQHw9XXul3JL6IT4xnxA8jqDO2DmMWjgGgdaXWlgSMyYHsiCAtH37o7m+91d84fLBk5xL6TO/Dsl3L6F67O9fVuc7vkIwxIWSJIC3PPANFikTd4DOjF4zmrjl3UeqMUnx87cdcWSs6j4iMiSaWCFKzaxccPw516/odSdgkdxJ3/tnnc9N5NzGq3SiKFSjmd1jGmDCwRJCaN9909yNzfqdpB48f5MFvHiRfTD5GtR9FiwotaFGhhd9hGWPCyBqLUzpyBEaNgoIF3dgDOdjsDbOpO64uYxeORVHrJM6YKGVHBClNmQJxca7H0ZicOZh63JE47vryLt5a/ha1StZi/i3zaVaumd9hGWN8YokgpSefdPcdOvgbRwjFHY3jk7Wf8MgljzC0xVDy5c7nd0jGGB+FtGpIRDqIyDoR2SAiD6Qy/S4RWSMiK0TkGxHx9yT1FStg2za44QY3EE0OsvPgTkb+OBJVpXqJ6mwZsoXhrYdbEjDGhC4ReOMdjwE6ArWB60WkdopiS4HGqlof+BB4NlTxBCV5vIHHH/c1jKykqkxcOpFaY2rxyHePsGHvBgA7I8gY87dQHhE0ATao6kZVPQG8B3QLLKCq36nqEe/pz0DZEMaTvs2b4YMPoHt3qFrVtzCy0qZ9m2j3djv6TO/DeWefx/L+y62TOGPMv4SyjaAMsC3geSzQNJ3yfYAvUpsgIv2AfgDly5fPqvhOltw2MGBAaOYfZglJCVz61qXEHYlj3OXj6Neon3USZ4xJVbZoLBaRG4HGQMvUpqvqeGA8uMHrszyAxER44w2oVw9atcry2YfT+rj1VC5Wmdy5cvNmtzepUqwK5YqU8zssY0w2FspdxO1A4BaorPfaSUSkLTAU6Kqqx0MYT9q+/97d33CDL4vPCvGJ8Twx7wnqjqvLK7+8AkCriq0sCRhjMhTKI4KFQDURqYRLAD2Ak7a0InI+8BrQQVV3hzCW9L3iNpxcF5mdqy3asYg+0/uw4o8V9Kjbg+vrXe93SMaYCBKyRKCqCSIyCJgDxAATVXW1iAwHFqnqdOA54EzgAxEB2KqqXUMVUxqBwiefQJ06buyBCPPSzy9x15d3cfaZZ/NZj8/oWiO8H58xJvKFtI1AVWcBs1K8NizgcdtQLj8osbHuvkEDX8PIrORO4hqf25g+5/fh2cuepWj+on6HZYyJQNmisdhXM2e6++7d/Y0jSH8d/4v7v7qf/Lnz80KHF2hevjnNyzf3OyxjTASz8wk/+8zdd+zobxxBmLV+FnXG1mH8kvHkzpXbOokzxmSJ6D4i2L0bZs+Grl0hb16/o0nTn0f+ZMjsIUxdOZU6perwYfcPaVo2vUsyjDEmeNGdCJJPG732Wn/jyMC+o/v4/LfPebTlozzU4iHyxmTfpGWMiTyWCCBbDlC//a/tTF05lXsvupdqJaqxZcgWaww2xoREdLcRHDjg7gsW9DeOAKrK64tfp/bY2jw29zF+3/c7gCUBY0zIRHcimDEDzjnH7yj+9vve32nzVhv6zehHw3MasuL2FVQtnjM6wDPGZF/RWzW0bx/s3w+NG/sdCeA6iWvzVhv2Ht3La51f49aGt1onccaYsIjeRPD11+7+6qt9DWPdn+uoUrwKuXPlZvIVk6lSvAplC/vXG7cxJvpE7y7ntGnu3qeG4hOJJ3h87uPUG1ePMb+MAaBlxZaWBIwxYRe9RwQ//wx168JZZ4V90b9s/4U+0/uwavcqbqh3Az3r9wx7DMYYkyw6jwj27IHt26FLl7Av+sWfX6TZhGbu2oDrP2fqVVMpWbBk2OMwxphk0XlEkFwtFMbeRpM7iWtSpgl9G/blmbbPUCR/kbAt3xhj0hKdiSD5QrIePUK+qAPHDnDfV/dRIE8BXuzwIheVu4iLyl0U8uUaY0yworNqaLc3Bk6hQiFdzOfrPqf22Nq8sfQN8sXks07ijDHZUnQeESxZ4hqKQ2TP4T0Mnj2Yd1e9S73S9fj0uk+5oMwFIVueMcacjuhLBKpw6BBUqxayRRw4foBZ62fxeKvHeeDiB6yTOGNMthZ9iWDTJndfNWu7bth2YBtvr3ibBy5+gKrFq7JlyBZrDDbGRIToayP49lt336ZNlswuSZN4ddGr1Blbhyd+eOLvTuIsCRhjIkX0JYJdu9x9w4anPav1ceu5dPKl3D7zdpqUacLK21daJ3HGmIgTfVVDP/4IJUtCqVKnNZuEpAQum3IZ+4/tZ0LXCfynwX8QkSwK0hhjwif6EkFcHBQocMpvX7tnLdVKVCN3rtxMuXIKVYpX4dxC52ZhgMYYE17RVTWUlARr18Kll2b6rccTjvPod49S/9X6vPLLKwC0qNDCkoAxJuJF1xHB7t1w8GCmryH4OfZn+kzvw5o9a+hVvxe96vcKUYDGGBN+0ZUIVq9298WKBf2WUT+O4t6v7qVs4bLMumEWHat1DFFwxhjjj+hKBCtWuPs6dTIsmqRJ5JJcNCvXjP6N+/N026cpnK9wiAM0xpjwi65EsHSpu2/QIM0i+4/t5+45d1MwT0Fe7vSydRJnjMnxoquxeP58aNcO8udPdfKnv35K7TG1mbx8MoXyFbJO4owxUSF6jgiOHoWNG+Gaa/41affh3QyaNYgP1nxAg7MbMOOGGTQ85/QvODPGmEgQPYkguY+h0qX/Nemv43/x1cavePLSJ7n3onvJE5MnzMEZY4x/oicRrFvn7qtUAWDrga1MWT6Fh1o8RNXiVdk6ZCuF8oV2fAJjjMmOQtpGICIdRGSdiGwQkQdSmZ5PRKZ50xeISMWQBRMXB0BS1SqMXTiWOmPrMOJ/I/7uJM6SgDEmWoUsEYhIDDAG6AjUBq4XkdopivUB9qlqVeAF4JlQxUN8POtKQKv5fRk4ayDNyjZj9YDV1kmcMSbqhfKIoAmwQVU3quoJ4D2gW4oy3YDJ3uMPgTYSop7bEo4con0vWLnvV97s9iZzbpxDxaIVQ7EoY4yJKKFsIygDbAt4Hgs0TauMqiaIyAGgBPBnYCER6Qf0AyhfvvwpBZO7anXeXt6SKsPe4pxipzYPY4zJiSLiOgJVHa+qjVW1calT7T66WzcufmuuJQFjjEkhlIlgO1Au4HlZ77VUy4hIbqAIEBfCmIwxxqQQykSwEKgmIpVEJC/QA5ieosx04Gbv8TXAt2qX8xpjTFiFrI3Aq/MfBMwBYoCJqrpaRIYDi1R1OjABmCIiG4C9uGRhjDEmjEJ6QZmqzgJmpXhtWMDjY0D3UMZgjDEmfRHRWGyMMSZ0LBEYY0yUs0RgjDFRzhKBMcZEOYm0szVFZA+w5RTfXpIUVy1HAVvn6GDrHB1OZ50rqGqqV+RGXCI4HSKySFUb+x1HONk6Rwdb5+gQqnW2qiFjjIlylgiMMSbKRVsiGO93AD6wdY4Ots7RISTrHFVtBMYYY/4t2o4IjDHGpGCJwBhjolyOTAQi0kFE1onIBhF5IJXp+URkmjd9gYhU9CHMLBXEOt8lImtEZIWIfCMiFfyIMytltM4B5a4WERWRiD/VMJh1FpFrve96tYi8E+4Ys1oQv+3yIvKdiCz1ft+d/Igzq4jIRBHZLSKr0pguIjLa+zxWiEjD016oquaoG67L69+BykBeYDlQO0WZAcCr3uMewDS/4w7DOrcGCnqPb4+GdfbKFQLmAT8Djf2OOwzfczVgKVDMe17a77jDsM7jgdu9x7WBzX7HfZrrfAnQEFiVxvROwBeAABcCC053mTnxiKAJsEFVN6rqCeA9oFuKMt2Ayd7jD4E2IiJhjDGrZbjOqvqdqh7xnv6MGzEukgXzPQP8H/AMcCycwYVIMOvcFxijqvsAVHV3mGPMasGsswKFvcdFgB1hjC/Lqeo83PgsaekGvKXOz0BRETnndJaZExNBGWBbwPNY77VUy6hqAnAAKBGW6EIjmHUO1Ae3RxHJMlxn75C5nKrODGdgIRTM91wdqC4i80XkZxHpELboQiOYdX4MuFFEYnHjn9wRntB8k9n/e4ZCOjCNyX5E5EagMdDS71hCSURyAc8DvX0OJdxy46qHWuGO+uaJSD1V3e9nUCF2PTBJVUeJSDPcqId1VTXJ78AiRU48ItgOlAt4XtZ7LdUyIpIbdzgZF5boQiOYdUZE2gJDga6qejxMsYVKRutcCKgLzBWRzbi61OkR3mAczPccC0xX1XhV3QT8hksMkSqYde4DvA+gqj8B+XGds+VUQf3fMyMnJoKFQDURqSQieXGNwdNTlJkO3Ow9vgb4Vr1WmAiV4TqLyPnAa7gkEOn1xpDBOqvqAVUtqaoVVbUirl2kq6ou8ifcLBHMb/tT3NEAIlISV1W0MYwxZrVg1nkr0AZARGrhEsGesEYZXtOBm7yzhy4EDqjqztOZYY6rGlLVBBEZBMzBnXEwUVVXi8hwYJGqTgcm4A4fN+AaZXr4F/HpC3KdnwPOBD7w2sW3qmpX34I+TUGuc44S5DrPAdqJyBogEbhXVSP2aDfIdb4beF1E/otrOO4dyTt2IvIuLpmX9No9HgXyAKjqq7h2kE7ABuAI8J/TXmYEf17GGGOyQE6sGjLGGJMJlgiMMSbKWSIwxpgoZ4nAGGOinCUCY4yJcpYITLYkIokisizgVjGdsoeyYHmTRGSTt6wl3hWqmZ3HGyJS23v8UIppP55ujN58kj+XVSLyuYgUzaB8g0jvjdOEnp0+arIlETmkqmdmddl05jEJmKGqH4pIO2CkqtY/jfmddkwZzVdEJgO/qeqT6ZTvjet1dVBWx2JyDjsiMBFBRM70xlFYIiIrReRfPY2KyDkiMi9gj7mF93o7EfnJe+8HIpLRBnoeUNV7713evFaJyBDvtTNEZKaILPdev857fa6INBaRp4ECXhxTvWmHvPv3ROTygJgnicg1IhIjIs+JyEKvj/nbgvhYfsLrbExEmnjruFREfhSRGt6VuMOB67xYrvNinygiv3hlU+ux1UQbv/vetpvdUrvhropd5t0+wV0FX9ibVhJ3VWXyEe0h7/5uYKj3OAbX31BJ3Ib9DO/1+4FhqSxvEnCN97g7sABoBKwEzsBdlb0aOB+4Gng94L1FvPu5eGMeJMcUUCY5xiuByd7jvLheJAsA/YCHvdfzAYuASqnEeShg/T4AOnjPCwO5vcdtgY+8x72BVwLePwK40XtcFNcX0Rl+f9928/eW47qYMDnGUVVtkPxERPIAI0TkEiAJtyd8FrAr4D0LgYle2U9VdZmItMQNVjLf61ojL25POjXPicjDuH5q+uD6r/lEVQ97MXwMtABmA6NE5BlcddIPmVivL4CXRCQf0AGYp6pHveqo+iJyjVeuCK6zuE0p3l9ARJZ5678W+Cqg/GQRqYbrZiFPGstvB3QVkXu85/mB8t68TJSyRGAiRU+gFNBIVePF9SiaP7CAqs7zEsXlwCQReR7YB3ylqtcHsYx7VfXD5Cci0ia1Qqr6m7ixDjoBT4jIN6o6PJiVUNVjIjIXaA9chxtoBdxoU3eo6pwMZnFUVRuISEFc/zsDgdG4AXi+U9UrvYb1uWm8X4CrVXVdMPGa6GBtBCZSFAF2e0mgNfCvMZfFjcP8h6q+DryBG+7vZ6C5iCTX+Z8hItWDXOYPwBUiUlBEzsBV6/wgIucCR1T1bVxnfqmNGRvvHZmkZhquo7DkowtwG/Xbk98jItW9ZaZK3WhzdwJ3yz9dqSd3Rdw7oOhBXBVZsjnAHeIdHonrldZEOUsEJlJMBRqLyErgJuDXVMq0ApaLyFLc3vZLqroHt2F8V0RW4KqFagazQFVdgms7+AXXZvCGqi4F6gG/eFU0jwJPpPL28cCK5MbiFL7EDQz0tbrhF8ElrjXAEnGDlr9GBkfsXiwrcAOzPAs85a174Pu+A2onNxbjjhzyeLGt9p6bKGenjxpjTJSzIwJjjIlylgiMMSbKWSIwxpgoZ4nAGGOinCUCY4yJcpYIjDEmylkiMMaYKPf/V/erh0mj80YAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis AUROC score: 0.8120999550000001\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, auroc_score = auroc(test_y, ood_scores)\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "print('Mahalanobis AUROC score:', auroc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAoFUlEQVR4nO3de7yUVdn/8c+XDQgIqAGaAQopCHhCw1NokYqgKWZHTX/mETNNfVTM7GTqU6Zp5iPmoczzoaOiqYR4zjQ3igkaingCsxBFVEAFr98fa8Y9bDfsYTP33Hvv+b5fr3ndx5n7ujc616y17rWWIgIzM6tdHfIOwMzM8uVEYGZW45wIzMxqnBOBmVmNcyIwM6txTgRmZjXOicDaDEkHSvprGeddIukH1YipGiS9IGn3wvrpkq7NOyZrX5wIrCIKX1ZLJL0t6T+SrpTUvZLXiIjrImKPMs77ZkScWclrF0kKSe8U7nOepPMl1WVxrZaQ1FPSBZJeKsT4XGG7d96xWevlRGCVtE9EdAe2BUYA3298gqSOVY+q8rYu3Odnga8Bh+UcDwCSOgNTgc2BsUBPYCdgAbB9Cz6vPfxbWRmcCKziImIecAewBXz4K/oYSc8Czxb27S1puqSFkh6StFXx/ZL6S/qTpPmSFki6qLD/EEkPFtYl6ReS/itpkaQnJRWvd6Wks0o+70hJsyW9LmmSpE+UHAtJ35T0bCGWiZJU5n3OBv4GDC/5vJbc1yaS7i7se03SdZLWXc0/O8DBwEbAfhHxVER8EBH/jYgzI+L2kvvdtCSmD/9WkkZJmivpO5JeBX4r6WlJe5ec37EQ/7aF7R0L97lQ0hOSRrUgbsuZE4FVnKT+wF7A4yW7vwDsAAyTtA1wBXAU0Au4FJgkaa1CNcttwIvAAKAvcGMTl9kD+AwwGFgH+Crpl2/jWHYFflo4vmHhcxt/3t7AdsBWhfPGlHmfQ4BdgNmF7ZbelwoxfgIYCvQHTi8nhkZ2B+6MiLdb8N6ijwMfAzYGxgM3AAeUHB8DvBYRj0nqC/wFOKvwnpOBP0rqswbXtxw4EVgl3SxpIfAgcB/wk5JjP42I1yNiCekL5tKIeCQilkfEVcC7wI6kKoxPABMi4p2IWBoRDzZxrfeBHsAQQBHxdET8u4nzDgSuiIjHIuJd4LvATpIGlJxzdkQsjIiXgHso+YW/Eo9Jegd4GrgXuLiwv0X3FRGzI2JKRLwbEfOB80nVTqurF9DU32B1fAD8qBDLEuB6YJykboXjXyclB4CDgNsj4vZC6WMKUE/6EWBtiBOBVdIXImLdiNg4Ir5V+CIperlkfWPgpEJ1wsJC8uhP+qLsD7wYEctWdaGIuBu4CJgI/FfSZZJ6NnHqJ0i/wovve5tUcuhbcs6rJeuLge4AkmYWGlzflrRLyTnbFs75GqmUs/aa3JekDSTdWGh8XgRcC7SkcXcBqdSzJuZHxNLiRqH662lgn0IyGEdKDpDu9yuN7nfnCsRgVeZEYNVSOszty8D/FpJG8dUtIm4oHNuonIbKiLgwIj4FDCNVEU1o4rRXSF9YAEham/TLeV4Zn795RHQvvB5odCwi4nfA34EfruF9/YT099kyInqSfmmX1U7RyF3AmMI9rsxioFvJ9scbHW9qOOJi9dC+wFOF5ADpnq5pdL9rR8TZLYjdcuREYHm4HPimpB0Kjb5rS/q8pB7AP0jVG2cX9neRNLLxB0jarvD+TsA7wFJStUZjNwCHShouaS3Sl+4jEfFChe7lbOBISR9fg/vqAbwNvFmod28qoZXjGtKX8x8lDZHUQVIvSadJKlbXTAe+LqlO0ljKq4K6kdQmczQNpQFIJZd9JI0pfF6XQoNzvxbGbzlxIrCqi4h64EhS1c4bpMbWQwrHlgP7AJsCLwFzSVUwjfUkffG+Qar6WQCc28S17gJ+APyR9EW8CbB/Be/lSeB+Ut1/S+/rx6TqpjdJja9/amEs75IajP8FTAEWkRJQb+CRwmnHF+JYSGo/ubmMz/03qeTzaeCmkv0vk0oJpwHzSUloAv5eaXPkiWnMzGqbM7eZWY1zIjAzq3FOBGZmNc6JwMysxrW5QaV69+4dAwYMyDsMM7M2Zdq0aa9FRJPDf7S5RDBgwADq6+vzDsPMrE2R9OLKjrlqyMysxjkRmJnVOCcCM7Ma1+baCMzMWur9999n7ty5LF26tPmT26guXbrQr18/OnXqVPZ7nAjMrGbMnTuXHj16MGDAAMqciK5NiQgWLFjA3LlzGThwYNnvy6xqSNIVStMIzljJcUm6UGkKwX8Wp74zM8vK0qVL6dWrV7tMAgCS6NWr12qXeLJsI7iSNIH2yuwJDCq8xgO/yjAWMzOAdpsEilpyf5klgoi4H3h9FafsC1xdmODjYWBdSdnNbPTgg/DDH8Idd2R2CTOztijPp4b6suL0hXNZcfrAD0kaL6leUv38+fNbdrW//x3OPBO+/e2Wvd/MrALq6uoYPnw4W2yxBfvssw8LFy788NjMmTPZdddd2WyzzRg0aBBnnnkmpVMF3HHHHYwYMYJhw4axzTbbcNJJJ1Ukpjbx+GhEXBYRIyJiRJ8+TfaQbt6ECXDQQeD5F8wsR127dmX69OnMmDGDj33sY0ycOBGAJUuWMG7cOE499VRmzZrFE088wUMPPcTFF18MwIwZMzj22GO59tpreeqpp6ivr2fTTTetSEx5JoJ5pAm9i/pRxjyyFfPaa7BoUdUuZ2bW2E477cS8eelr7/rrr2fkyJHsscceAHTr1o2LLrqIs89OU0Cfc845fO9732PIkCFAKlkcffTRFYkjz8dHJwHHSroR2AF4szAlXvaeew423RS++lW46abmzzez9ueEE2D69Mp+5vDhcMEFZZ26fPlypk6dyuGHHw6kaqFPfepTK5yzySab8Pbbb7No0SJmzJhRsaqgxjJLBJJuAEYBvSXNBX4EdAKIiEuA24G9SPO6LgYOzSqWFUTAYYel9SefrMolzcyKlixZwvDhw5k3bx5Dhw5l9OjReYeUXSKIiAOaOR7AMVldf6Wefz69AGbPrvrlzayVKPOXe6UV2wgWL17MmDFjmDhxIscddxzDhg3j/vvvX+HcOXPm0L17d3r27Mnmm2/OtGnT2HrrrSseU5toLK64bt3S8v33843DzGpWt27duPDCCznvvPNYtmwZBx54IA8++CB33XUXkEoOxx13HKeccgoAEyZM4Cc/+QnPPPMMAB988AGXXHJJRWKpzUTwmc/ALrvkHYWZ1bhtttmGrbbaihtuuIGuXbtyyy23cNZZZ7HZZpux5ZZbst1223HssccCsNVWW3HBBRdwwAEHMHToULbYYgvmzJlTkThqa6yhxYvTcocdYPlyeOCB1MFs7Fho570Nzax1ePvtt1fYvvXWWz9c33LLLbn33ntX+t69996bvffeu+Ix1VaJoNg2MGQI1NWl9b32gmnT8ovJzCxntZUIisWogQNh2bKG/d//fj7xmJm1ArWVCN58My032gieeKJh/+TJ+cRjZlUX7Xx0gZbcX20lgqLeveGMMxrGHdp333zjMbOq6NKlCwsWLGi3yaA4H0GXLl1W631qa3+QESNGRH19fcveXGwQLr3nbt3S9pIlax6cmbVqtTxDmaRpETGiqffU1lNDu+ySnhQq5QRgVjM6deq0WjN31Yraqhq6+25o/Etg9OiGDmZmZjWothJBx46w1lor7uvbN/UveP55aPR8r5lZLaitqqGmdCjkwk9+Mi2XL2/YZ2ZWA/yNN3PmituF7txmZrXCieCtt1bc/tWvYMGCfGIxM8uBE8HNN8Ohh0L/ksnSSjubmZm1c04EgwbBFVfASy/B73+f9hV7IJuZ1QAnglLFx0g99pCZ1RAnglI775yWTz21Yu9jM7N2zImgVM+eDet//Wt+cZiZVZETwcqMHZt3BGZmVeFE0Ni//92w/uc/5xeHmVmVOBE09vGPpxfAL36RbyxmZlXgRNAU9yMwsxriRNCU9ddPywcegHnz0hNEb7wBl12WxiIyM2tHnAia068f3H8/fOxjcNRRaQRTM7N2xImgHKNGrbgtwW9/m9ZPPBEOPhhefhkWLap6aGZma8qJYGXOPnvVxw87DLbaKjUoX3MNbLQRrLMOfOMbqSppyhR49FG4996qhGtm1lJOBCvzne/Ae+81DDfxwAPw8MMrnvPkkx9939VXp/kM9tgDtt8ePvc5+MpXUimiTx/4y1+yj93MbDXU1uT1lSSl5fTpDTObFYeoaM63vgU//jH07p1JaGZmja1q8nqXCFoqAj74ALbeGkaOTK+5c2GvvVKv5MWL4bnnGs7fc8+G9YsvTqUDCZ59tvqxm5mV8CMwa6JYKijq23fFqp9PfvKjg9ettVaqcioaPBjuugt22y27OM3MViHTEoGksZJmSZot6dQmjm8saaqkf0q6V1K/LONpFZYsSX0S7rmnYd/uu8M//5n6KPjJIzOrsswSgaQ6YCKwJzAMOEDSsEan/Ry4OiK2As4AfppVPK1Ghw6w7rrpkdRLLmnYv/XWqY/COuvA0qVp32uvQadOqeRx1VXuzGZmmciyRLA9MDsi5kTEe8CNwL6NzhkG3F1Yv6eJ4+3bUUeldobGunaFX/4ytSMsW5b2HXJIShTvvFPVEM2s/csyEfQFXi7ZnlvYV+oJ4IuF9f2AHpJ6Nf4gSeMl1Uuqnz9/fibB5kZKv/RPPx1uvLFh/wknpOVhh614fvfu0N7+BmaWq7yfGjoZ+Kykx4HPAvOAj9R/RMRlETEiIkb06dOn2jFmr0MH+NGP4Gtfg1mz0r6xY1Oj8m9+kxqcn3++4fz1129oqH7pJXj99erHbGbtRpZPDc0D+pds9yvs+1BEvEKhRCCpO/CliFiYYUyt3+DBTU+TOWAAzJmTnkQqavzU0sknw7nnZhqembU/WZYIHgUGSRooqTOwPzCp9ARJvSUVY/gucEWG8bR9AwemJPHKK00f//nP09hHr75a3bjMrE3LLBFExDLgWGAy8DTwu4iYKekMSeMKp40CZkl6BtgA+N+s4mlXNtwwJYTi64MP0rhHkMY+2nDDVFqYOjXfOM2sTfAQE+3JoYfClVeuuG/qVNh111zCMbPWw0NM1Irf/rahlLD11mnfbrt9tC3BzKyEE0F7NX36itsSLFyYRyRm1so5EbRnEal/QtF66604EJ6ZGU4E7d+PftTQOxlg001T6WDBgvxiMrNWxYmgFtTVfbRvQu/eaRiLlT2KamY1w4mglkSkeRKKTjghDZ293Xa5hWRm+XMiqDVdu8ILL8AppzTsq6+HP/6x6QHwzKzdcyKoRRtvDD/7WSohFEsDX/5yqkKS0mvXXdMjqBdfnJ5AeuqpXEM2s+w4EdS6Rx5Jg901ds89abKcY46BbbaBzTdPCWL8+OrHaGaZciKodVIa/rp0yIoHHkhDVZxamFRurbUazr/88oZSw957w6WXpiolVyuZtVkeYsLK9+STDWMarcp118HXv559PGZWNg8xYZWx5ZapxPD++6l/wgYbNH3egQemEkO/fmk5YAC89VZVQzWz8jkR2Orr2DH1WH711RWrlJYtg+OPbzhvXmH6iRdfhJ49nQzMWiknAqucujq44IKUFJYvT30W7rmn4XjPnvCJT6THV82s1XAisGx06JD6LIwalRqSu3ZN+//97zTBjpSm5WxjbVRm7ZETgWVPSqWDOXNW3D9kSEoYxaeQ3nknn/jMapwTgVVPcarNlZUCund3xzWzHDgRWD5KG5kXLWrYX+y4Vvrq3h123x2mTXN/BbMMOBFY/nr0SF/wAwc2ffydd9KUmyNGNAyDMWpU6vV80EFprCQzazF3KLPW67334Pnn4bXX4Ljj4LHHmn9Pnz7w+ONpVNWlS6Fz59QOYVbj3KHM2qbOnWGzzWDkyFQtFJFKDu+9B2efDVOmwLXXrvie+fMbOrJ17dpQghg5Mi1LZ2wzM8AlAmtvvvnNNP4RpI5vpbOzldptN7j99pRszGqASwRWOy65pKER+v330/K11+DNN+HhhxvOmzo1DaYnuS+D1TwnAmv/evVKvZp32CF96T///IrHi30ZzjzTTyVZTXIisNozYEBKCO+9t+L+H/6woU1hxoxcQjPLgxOB1a5OnRqqkebOhf/3/xqObbllSgj9+6dqJbN2zInADNLjpldfnZLCddc17J87F9ZdNyWF447LLTyzLDkRmDX29a83NDaXTs35f/+XEsIee8DQoXDeefnFaFZBTgRmK9OxY3oUNSLN31w0ZQr8619w8skpMWy7Ldx5Z35xmq0hJwKzchRnZyu+3noLunVLxx5/HPbcMyWFYcPghBPSpD2NG6PNWqlME4GksZJmSZot6dQmjm8k6R5Jj0v6p6S9sozHrGK6d09jIC1fDn/6U8P+p5+GX/4SNtywoZ9Cr15p6IujjkrDXpi1MpklAkl1wERgT2AYcICkYY1O+z7wu4jYBtgfuDireMwy0aED7LdfQ0nh9dfh859PX/pFr7+eOrVddlka9uLcc/OL16wJWZYItgdmR8SciHgPuBHYt9E5AfQsrK8DvJJhPGbZW289uO22FXs4R8B//tNwzimnpJLChhumhunJk1c+FIZZFWSZCPoCL5dszy3sK3U6cJCkucDtwLczjMcsP+uvnxLCG2+kXs6Q2hFuuAHGjk19Grp0gZkz843TalJZiUDSSElTJD0jaY6k5yXNaf6dzToAuDIi+gF7AddI+khMksZLqpdUP3/+/Apc1iwn666bOqgVG5xPPx0++cl07N13YYstYPRoOOQQWLIkx0CtlpQ1+qikfwH/A0wDlhf3R8SCVbxnJ+D0iBhT2P5u4T0/LTlnJjA2Il4ubM8BdoyI/67scz36qLVbU6akPgqljjkmdWTr1y+VGIqztpmtpkqMPvpmRNwREf+NiAXFVzPveRQYJGmgpM6kxuBJjc55CditEORQoAvgn/xWm0aPTqWA225reDR14sQ0J8Paa6dxkIoD5J12WurwZlYB5SaCeySdK2knSdsWX6t6Q0QsA44FJgNPk54OminpDEnjCqedBBwp6QngBuCQaGsTJJhVUpcu6amjd96BhQvhpJNS+8KnP51eRT/9aZpLQUpPLc2alVvI1vaVWzV0TxO7IyJ2rXxIq+aqIat5b7wBW28NL7/80WNHHpmGvujRo/pxWau2xlVDEfG5Jl5VTwJmRnpE9aWXUoPz8uWpf0LR5Zenp5KkVLJ48klPvGPNKvepoXUknV98ckfSeZLWyTo4M2tGhw6pFBCRqpN2373h2O23w1ZbpXN22MHDadtKldtGcAXwFvDVwmsR8NusgjKzFujWLT15VOzEdsstDcf+8Y/06OrWW8PIkfD73+cWprU+Hcs8b5OI+FLJ9o8lTc8gHjOrlHHjUkJYtix1WNtkk4ZRVB96KM3B8OCDacY2q2nllgiWSNq5uCFpJODeLmZtQceOKSHMnp2Wt92W9s+bBwMHwk47pWolq1nlJoKjgYmSXpD0InAR8M3swjKzzHz+8ykh/PrXafvhh9NoqieckAbIs5pT1uOjH54s9QSIiEWZRdQMPz5qVmGNeyp//vNwzjlpbgVrN1r8+KikgwrLEyWdCBwBHFGybWZtXbHaaKON0vZf/gKbb94wnMWOO6ZqJGu3mqsaWruw7LGSl5m1B5tsAi++mKqGTjttxWOPPJLGOpLSQHnW7qxW1VBr4KohsyqbPBkOOwxeKZku5MQTUw9mazPWuGexpHMk9ZTUSdJUSfOL1UZm1s6NGZOqhiZMaNh3/vmphHDkkfDBB/nFZhVR7lNDexQaiPcGXgA2BSas8h1m1r6cc05qT1i0CD7xibTv179Oo6KecIIfQW3Dyk0ExY5nnwd+HxHuq25Wq3r0SCWE5cth/Pi075e/TI+gLmhudHprjcpNBLcVJqf5FDBVUh9gaXZhmVmr16EDXHppmo+5OP1m795pvmZrU8odffRU4NPAiIh4H3iHj05Eb2a1aP3104B2xxyTto8+OrUfLFyYa1hWvub6EexaWH4RGAXsW1gfS0oMZmbJRRfBtGkN2+utB1/4QhrryFq15koEny0s92nitXeGcZlZW7TttqlB+aij0vYtt6QB70aNyjUsWzX3IzCzbCxcCDvvDDNnpu26OnjhhdQ5zaquEv0IfiJp3ZLt9SSdVaH4zKw9WnddmDGjYT7l5cuhf//UfjBmDLz/fq7hWYNynxraMyIWFjci4g1gr0wiMrP2ZfDgVF00cWLDvr/+FTp3TklhypT8YjOg/ERQJ2mt4oakrsBaqzjfzGxF3/pWSghLl6Yqo6I99kizq/3pT+6lnJNyE8F1pP4Dh0s6HJgCXJVdWGbWbq21FjzwQEoKr76a9i1ZAl/6EmywAfzrX/nGV4PK7UfwM+AsYGjhdWZEnJNlYGZWAzbYICWE++5L26+9BkOHpiqjc89N7QqWuXJLBABPA3dGxMnAA5I8DLWZVcZnPpMSwuWXpwZlgFNOSdNsfuc76Zhlptynho4E/gBcWtjVF7g5o5jMrFYdcQS89FJ60miPPdK+c85Jw1k89VS+sbVj5ZYIjgFGAosAIuJZYP2sgjKzGjd4cJoHobRX8uabw1l+aj0L5SaCdyPiveKGpI6Ay2pmlq26ulQtdMUVafsHP4CHHso3pnao3ERwn6TTgK6SRgO/B27NLiwzsxKHHgp33ZXWR45MndUOPNCd0iqk3ETwHWA+8CRwFHA78P2sgjIz+4jddoMbb4S9906jnV5/feqUNnVq3pG1ec2ONSSpDpgZEUOqE9KqeawhM+ODD+BjH0sJAeDkk9PjprZSazTWUEQsB2ZJ2qjikZmZtUSHDmlQu+KX/89/nvoeHHwwvP56rqG1ReVWDa0HzCxMXD+p+GruTZLGSpolabakU5s4/gtJ0wuvZyQtXM34zayWnXzyihPgXHMN9OrlWdJWU8fmTwHgB6v7wYUqpYnAaGAu8KikSRHx4cPAEfE/Jed/G9hmda9jZjVunXXSk0XLl8P228Njj6VZ0rbfPs2PYM1qboayLpJOAL4CDAH+FhH3FV/NfPb2wOyImFN49PRGVj295QHADeWHbmZWoq4uzZD285+n7U99Ck47Ld+Y2ojmqoauAkaQnhbaEzhvNT67L/Byyfbcwr6PkLQxMBC4eyXHx0uql1Q/f/781QjBzGrOSSfBvfem9Z/+NLUdPPtsriG1ds0lgmERcVBEXAp8Gdglozj2B/5QaJj+iIi4LCJGRMSIPn36ZBSCmbUbn/0szJnTsD14MPzmN/nF08o1lwg+7K0REas7A/U8oH/Jdr/Cvqbsj6uFzKySBg5MbQcTJqTtI45Ig9rZRzSXCLaWtKjwegvYqrguaVEz730UGCRpoKTOpC/7jzxpJGkI6amkv7fkBszMVumcc+CZZ9L6+PENw1XYh1aZCCKiLiJ6Fl49IqJjyXrPZt67DDgWmEwawvp3ETFT0hmSxpWcuj9wYzTXs83MrKUGDYJHH03rhx8O112XbzytTLM9i1sb9yw2sxZ78EHYpdDUOXx4mhCn5yp/07Yba9Sz2Mys3dh5Z7jjjrQ+fXrqg/CLX+QaUmvgRGBmtWXs2DRW0Xe/m7ZPPBH22Qfee2/V72vHnAjMrPZI8JOfwKJFqf3gttuga9eabTtwIjCz2tWjBzz9NJx6aiolHHRQShILFuQdWVU5EZhZbaurSz2Q58xJ65CmxWxjD9KsCScCMzNIHdCWLUvL//wHxozJO6KqcSIwMyv17LOw444wZQqceWbe0VSFE4GZWam6upQEAH74Q7jllnzjqQInAjOzxrp3h5tvTutf+AL84x95RpM5JwIzs6bsuy9ccEFaHz0a/vWvXMPJkhOBmdnKHH883H9/6m8wdCgsXpx3RJlwIjAzW5VddoETTkjrX/xirqFkxYnAzKw5558Po0bB5Mnwq1/lHU3FORGYmTVHgquuSuvf+ha8+GK+8VSYE4GZWTk22gj+9Ke0PmAAzFvZhIttjxOBmVm59tsv9TwG6NcPHn8833gqxInAzGx1PPdcGqQOYNtt28VjpU4EZmarQ0qD1P31r2l76NB846kAJwIzs5YYPRo23jitz56dbyxryInAzKylzj8/LUePbtPDVjsRmJm11Be/CAcfDC+8AL175x1NizkRmJmticsvT8vXX2+zU106EZiZrYnOnWHWrLR+0EFQX59vPC3gRGBmtqYGD4aLLkrrRx2Vbywt4ERgZlYJxxwDffvCY4/BrbfmHc1qcSIwM6uUYt+CceNSx7M2wonAzKxShg1raDzedFN4++184ymTE4GZWSUdcUTDUNVXX51vLGVyIjAzq7Qjj4R11kntBsuW5R1Ns5wIzMwqra4uTWQDMGZMrqGUw4nAzCwLf/4zbL893H03HHts3tGsUqaJQNJYSbMkzZZ06krO+aqkpyTNlHR9lvGYmVWNlCa+B5g4sVV3NMssEUiqAyYCewLDgAMkDWt0ziDgu8DIiNgcOCGreMzMqm6tteDOO9P6dtvlG8sqZFki2B6YHRFzIuI94EZg30bnHAlMjIg3ACLivxnGY2ZWfWPGQM+eaf3ZZ/ONZSWyTAR9gZdLtucW9pUaDAyW9DdJD0sa29QHSRovqV5S/fz58zMK18wsI488kpatdBKbvBuLOwKDgFHAAcDlktZtfFJEXBYRIyJiRJ8+faoboZnZmhoyJM1xvHw57L57q5u7IMtEMA/oX7Ldr7Cv1FxgUkS8HxHPA8+QEoOZWfty111pOXUqHHpovrE0kmUieBQYJGmgpM7A/sCkRufcTCoNIKk3qapoToYxmZnlY7PN4K23UnvBVVc1JIZWILNEEBHLgGOBycDTwO8iYqakMySNK5w2GVgg6SngHmBCRCzIKiYzs1x17w7TpqX10aPh1VfzjadA0crqqpozYsSIqG/Fz+OamTXrwgvh+OPhM5+B++6ryiUlTYuIEU0dy7ux2Mys9hx3HOyyS+pw9tpreUfjRGBmlovjj0/Lm2/ONQxwIjAzy8c++0CHDmmk0pw5EZiZ5aFzZxhb6EN78sm5huJEYGaWl2uvTcvzzoMlS3ILw4nAzCwv663XMJvZUUflFoYTgZlZno46KrUVXHMNLMinG5UTgZlZnqSGCe/33z+XEJwIzMzyVhx76K670sB0VeZEYGaWt9JSwfjxVb+8E4GZWWtwyCFpecUVMHduVS/tRGBm1hp07Ajf+15a799/1edWmBOBmVlrcdZZMHx4Wp8+vWqXdSIwM2tNrrkmLbfZpmqXdCIwM2tNttgCNt00rS9eXJVLOhGYmbU23/lOWh5zTFUu50RgZtbaHH54Wv7ud1WZ6N6JwMystZHgG99IVUNPPpn55ZwIzMxao+LQ1DvvnPmlnAjMzFqjLbZIy7feyrx6yInAzKy1OuywtHzllUwv40RgZtZajR6dli+8kOllnAjMzFqrIUPS8s47M72ME4GZWWu1+eZpedNNmV7GicDMrLXq1AnWXhuefTbTBmMnAjOz1uzoo9Pyjjsyu4QTgZlZa1acvey55zK7hBOBmVlrNmhQWs6YkdklnAjMzFqzTp3S8tprM7uEE4GZWWu34YaZDkmdaSKQNFbSLEmzJZ3axPFDJM2XNL3wOiLLeMzM2qRix7IFCzL5+MwSgaQ6YCKwJzAMOEDSsCZOvSkihhdev84qHjOzNmuvvdLyoYcy+fgsSwTbA7MjYk5EvAfcCOyb4fXMzNqnYg/jyZMz+fgsE0Ff4OWS7bmFfY19SdI/Jf1BUv+mPkjSeEn1kurnz5+fRaxmZq3XllvChAmwbza/pfNuLL4VGBARWwFTgKuaOikiLouIERExok+fPlUN0Mwsdx06wDnnNLQVVPrjM/nUZB5Q+gu/X2HfhyJiQUS8W9j8NfCpDOMxM7MmZJkIHgUGSRooqTOwPzCp9ARJG5ZsjgOezjAeMzNrQsesPjgilkk6FpgM1AFXRMRMSWcA9RExCThO0jhgGfA6cEhW8ZiZWdMUGU+BVmkjRoyI+vr6vMMwM2tTJE2LiBFNHcu7sdjMzHLmRGBmVuOcCMzMapwTgZlZjWtzjcWS5gMvtvDtvYHXKhhOW+B7rg2+59qwJve8cUQ02SO3zSWCNSGpfmWt5u2V77k2+J5rQ1b37KohM7Ma50RgZlbjai0RXJZ3ADnwPdcG33NtyOSea6qNwMzMPqrWSgRmZtaIE4GZWY1rl4lA0lhJsyTNlnRqE8fXknRT4fgjkgbkEGZFlXHPJ0p6qjAb3FRJG+cRZyU1d88l531JUkhq848alnPPkr5a+LeeKen6asdYaWX8t72RpHskPV7473uvPOKsFElXSPqvpBkrOS5JFxb+Hv+UtO0aXzQi2tWLNOT1c8Angc7AE8CwRud8C7iksL4/cFPecVfhnj8HdCusH10L91w4rwdwP/AwMCLvuKvw7zwIeBxYr7C9ft5xV+GeLwOOLqwPA17IO+41vOfPANsCM1ZyfC/gDkDAjsAja3rN9lgi2B6YHRFzIuI94Eag8USf+9IwLeYfgN0kqYoxVlqz9xwR90TE4sLmw6QZ49qycv6dAc4EfgYsrWZwGSnnno8EJkbEGwAR8d8qx1hp5dxzAD0L6+sAr1QxvoqLiPtJ87OszL7A1ZE8DKzbaJKv1dYeE0Ff4OWS7bmFfU2eExHLgDeBXlWJLhvl3HOpw0m/KNqyZu+5UGTuHxF/qWZgGSrn33kwMFjS3yQ9LGls1aLLRjn3fDpwkKS5wO3At6sTWm5W9//3ZmU2Q5m1TpIOAkYAn807lixJ6gCcT+3NeteRVD00ilTqu1/SlhGxMM+gMnYAcGVEnCdpJ+AaSVtExAd5B9ZWtMcSwTygf8l2v8K+Js+R1JFUnFxQleiyUc49I2l34HvAuIh4t0qxZaW5e+4BbAHcK+kFUl3qpDbeYFzOv/NcYFJEvB8RzwPPkBJDW1XOPR8O/A4gIv4OdCENztZelfX/++poj4ngUWCQpIGSOpMagyc1OmcS8I3C+peBu6PQCtNGNXvPkrYBLiUlgbZebwzN3HNEvBkRvSNiQEQMILWLjIuItjzPaTn/bd9MKg0gqTepqmhOFWOstHLu+SVgNwBJQ0mJYH5Vo6yuScDBhaeHdgTejIh/r8kHtruqoYhYJulYYDLpiYMrImKmpDOA+oiYBPyGVHycTWqU2T+/iNdcmfd8LtAd+H2hXfyliBiXW9BrqMx7blfKvOfJwB6SngKWAxMios2Wdsu855OAyyX9D6nh+JC2/MNO0g2kZN670O7xI6ATQERcQmoH2QuYDSwGDl3ja7bhv5eZmVVAe6waMjOz1eBEYGZW45wIzMxqnBOBmVmNcyIwM6txTgRmTZC0XNJ0STMk3Spp3Qp//guF5/yR9HYlP9tsdTkRmDVtSUQMj4gtSH1Njsk7ILOsOBGYNe/vFAb1krSJpDslTZP0gKQhhf0bSPqzpCcKr08X9t9cOHempPE53oPZSrW7nsVmlSSpjjR8wW8Kuy4DvhkRz0raAbgY2BW4ELgvIvYrvKd74fzDIuJ1SV2BRyX9sS339LX2yYnArGldJU0nlQSeBqZI6g58moZhOgDWKix3BQ4GiIjlpKHNAY6TtF9hvT9pADgnAmtVnAjMmrYkIoZL6kYa5+YY4EpgYUQML+cDJI0Cdgd2iojFku4lDYhm1qq4jcBsFQqzuh1HGthsMfC8pK/Ah3PHbl04dSppClAk1UlahzS8+RuFJDCENBS2WavjRGDWjIh4HPgnaQKUA4HDJT0BzKRh2sTjgc9JehKYRpo7906go6SngbNJQ2GbtToefdTMrMa5RGBmVuOcCMzMapwTgZlZjXMiMDOrcU4EZmY1zonAzKzGORGYmdW4/w/6iuIiB4tXJwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis AUPR score: 0.7948582551191462\n"
     ]
    }
   ],
   "source": [
    "precision, recall, aupr_score = aupr(test_y, ood_scores)\n",
    "\n",
    "plot_precision_recall_curve(precision, recall)\n",
    "print('Mahalanobis AUPR score:', aupr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis fpr95 score: 0.648\n"
     ]
    }
   ],
   "source": [
    "fpr95 = fpr_at_95_tpr(test_y, ood_scores)\n",
    "print('Mahalanobis fpr95 score:', fpr95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n",
      "100%|██████████| 98/98 [00:26<00:00,  3.74it/s]\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.metrics.sml import SML\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "sml = SML(config, model, id_train_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:07<00:00, 14.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 15.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id mean score: 0.8097272615159194 id variance: 2.795628445598447\n",
      "ood mean score: 3.04131169785856 ood variance: 2.7286585372155883\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "test_y, ood_scores, id_ood_scores, ood_ood_scores = compute_ood_scores(\n",
    "    metric=sml,\n",
    "    in_dist_dataloader=id_test_dataloader,\n",
    "    out_of_dist_dataloader=ood_test_dataloader,    \n",
    ")\n",
    "print('id mean score:', np.mean(id_ood_scores), 'id variance:', np.var(id_ood_scores))\n",
    "print('ood mean score:', np.mean(ood_ood_scores), 'ood variance:', np.var(ood_ood_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABCDklEQVR4nO3dd3gU5fbA8e8h9F5VpPeOCBEL0gRpoujPBiKKolykWLBcFRtYrg0LCggqgiiKoGiUpqKIF5QqIFUQpXOBSA8BkpzfH+9E15iyQDaT3T2f59lnd2dmZ87M7s6Zed+Z9xVVxRhjTPTK43cAxhhj/GWJwBhjopwlAmOMiXKWCIwxJspZIjDGmChnicAYY6KcJYIIJyKrRaSN33HkFiLysIi85dOyx4vIU34sO7uJSE8R+fIUP2u/yVzGEkEOEpHfReSoiBwWkV3ejqFoKJepqg1UdW4ol5FKRAqIyH9EZIu3nhtE5H4RkZxYfjrxtBGRbYHDVPUZVb0tRMsTEblTRFaJyBER2SYiU0SkUSiWd6pE5AkRee905qGq76tqhyCW9Y/kd6q/SRHJ78W+wdu+v4vIOBGperLzMn9niSDnXa6qRYEmwLnAQ/6Gc/JEJG8Go6YA7YAuQDGgF9AXeDUEMYiI5Lbf76vAXcCdQGmgNvApcFl2LyiT7yDkfFz2VOAK4AagBHAOsBT3mzspfm6/XElV7ZFDD+B3oH3A++eB6QHvLwAWAPuBFUCbgHGlgXeAHcA+4NOAcV2B5d7nFgCN0y4TOBs4CpQOGHcusBfI572/FVjrzX82UCVgWgUGABuA39JZt3ZAIlApzfDzgWSgpvd+LvAfYBFwEPgsTUyZbYO5wNPAfG9dagK3eDEfAjYB//KmLeJNkwIc9h5nA08A73nTVPXW62Zgi7cthgQsrxAwwdsea4EHgG0ZfLe1vPVsnsn3Px4YCUz34l0I1AgY/yqw1dsuS4GWAeOewO0I3/PG3wY0B37wttVO4HUgf8BnGgBfAX8A/wMeBjoBx4ET3jZZ4U1bAnjbm8924CkgxhvX29vmLwPx3rjewH+98eKN2+3F9jPQEHcQcMJb3mHg87T/AyDGi+tXb5ssJc1vyJuuvfd9/mNcJv+v9L7rPt53PQ+YCQxMM48VwP95r+sGbL/1wHV+70NCtm/yO4BoeqT5A1T0/jCveu8reH+yLrgztUu99+W88dOByUApIB/Q2ht+rvcHPN/7U93sLadAOsv8Brg9IJ4XgDe8192AjUA9IC/wCLAgYFr1/hSlgULprNuzwHcZrPdm/tpBz/V2NA1xO+uPA/6sWW2Dud6fuIEXYz7c0XYN3M6oNZAANPWmb0OaHXcGO4c3cTv9c4BjQL3AdfK2eUVgZdr5Bcy3H7A5i+9/vLc+zb343wc+DBh/I1DGG3cvsAsoGBD3CeBKb9sUAprhEmdeb13WAnd70xfD7dTvBQp6789Puw0Clj0NGON9J2fgEnXqd9YbSAIGecsqxN8TQUfcDryk9z3UA8oHrPNTmfwP7sf9D+p4nz0HKHMyv6/05pvJd/2ut46FgJuA+QHT18cl1QLeNFtxBxp5+eugqb7f+5FQPHLbqXU0+FREDuF+ZLuBx73hNwIzVHWGqqao6lfAEqCLiJQHOgP9VHWfqp5Q1e+8z/UFxqjqQlVNVtUJuJ3ZBeksexLQA1zRCtDdGwZuR/YfVV2rqknAM0ATEakS8Pn/qOofqno0nXmXxe140rPTG59qoqquUtUjwKPAdSISk9k2CPjseFVdrapJ3naYrqq/qvMd8CXQMoM4MjJUVY+q6grcEeE53vDrgGe8bb4NGJHJPMpksv6BpqnqIm8bv48rIgRAVd9T1Xhv3Ybjdkh1Aj77g6p+6m2bo6q6VFV/9Kb/Hbcjb+1N2xXYparDVTVRVQ+p6sL0AhKRM3Hb+G5VPaKqu3FH+N0DJtuhqq95y0r7/Z/AJZq6gHi/oWC2Bbgzm0dUdb33Ha5Q1fh0pgt2+2blCW8dj+KSX+BvvCfwiaoew22/31X1HW+df8IdtFybDTHkOpYIct6VqloMd7Ral792kFWAa0Vkf+oDuBgoD1QC/lDVfenMrwpwb5rPVcIVg6T1MXChl1ha4YpNvg+Yz6sB8/gDd4RWIeDzWzNZr71erOkp741Pbz6bcUf2Zcl8G6Qbg4h0FpEfReQPb/ou/D3pBGNXwOsEILUC/+w0y8ts/ePJeP2DWRYicp+IrBWRA966lODv65J23WuLyBfehQcHcck7dfpKuOKWYFTBfQc7A7b7GNyZQbrLDqSq3+CKpUYCu0VkrIgUD3LZwcYZ7PbNyp/roaqHcGfaqQmvBy45g9sm56f5LfYEzsqGGHIdSwQ+8Y5exwMveoO24o6USwY8iqjqs9640iJSMp1ZbQWeTvO5wqr6QTrL3Ic7Yr4eV+H2oapqwHz+lWY+hVR1QeAsMlmlr3F/nEqBA0XkfNyf/ZuAwYHTVMYdUe7NYhv8IwYRKYBLbi8CZ6pqSWAGLoFlFW8wduKKhNKLO605QEURiT2VBYlIS1wdxHVAKW9dDvDXusA/12c0sA6oparFcWXtqdNvBapnsLi089mKO4ssG7Ddi6tqg0w+8/cZqo5Q1Wa44pXauCKfLD/nLbtGFtOA+301F5GKmUxzBCgc8D69nXbaeD4AeojIhbgitG8D4vouzW+xqKreEUSsYccSgb9eAS4VkXNwlYCXi0hHEYkRkYLe5Y8VvdPsmcAoESklIvlEpJU3jzeBfiJyvnclTRERuUxEimWwzEm4stFr+KtYCOAN4CERaQAgIiVEJOjTYFX9Grcz/FhEGnjrcIG3XqNVdUPA5DeKSH0RKQwMA6aqanJm2yCDxebHFZ/sAZJEpDMQeEnj/4AyIlIi2PVI4yPcNiklIhWAgRlN6K3fKOADL+b8XvzdReTBIJZVDFcOvwfIKyKPAVkdVRfDVc4eFpG6QOBO6gugvIjcLe6y3mJeUga3XaqmXnXl/b6+BIaLSHERySMiNUSkNUEQkfO8318+3M44EXe2mbqsjBISwFvAkyJSy/v9NhaRMmkn8n5fXwHTRKSZiOT11qmfiNzqTbYc6O79P2Jxv/GszMAd/Q8DJqtqatxfALVFpJc3v3zeetYLYp5hxxKBj1R1D67y6jFV3YqrsH0YtzPYijuqSv2OeuGOnNfh6hbu9uaxBLgdd2q+D1fh2zuTxcbhrnDZ5ZWJp8YyDXgO+NArZliFq5c4GVfjjqhm4a4SeQ93JcqgNNNNxJ0N7cIdhd3pxZDVNvgb79T+TtwOex/uLCcuYPw63BHfJu/0Pr3isswMA7YBv+GOSKfijpwzcid/FZHsxxV5XAV8HsSyZuO22y+44rJEMi+KArgPt86HcAcEk1NHeNvmUuBy3HbeALT1Rk/xnuNFZJn3+iZcYl2D25ZTCb4opri3/H1e7PG4CxHAff/1ve3/aTqffQn3/X2JS2pv4ypy03MNbsc9GXe2tAqIxX034OqbanhxDOXvBzrp8uoDPsFdlTQpYPgh3EFFd9yVertw/48CWc0zHMlfJQPGhJ6IzMVdyeHL3b2nQ0TuALqralBHysaECzsjMCYDIlJeRFp4RSV1cJdiTvM7LmOym91dZ0zG8uOunqmGK+r5EFcPYExEsaIhY4yJclY0ZIwxUS7siobKli2rVatW9TsMY4wJK0uXLt2rquXSGxd2iaBq1aosWbLE7zCMMSasiMjmjMZZ0ZAxxkQ5SwTGGBPlLBEYY0yUC7s6gvScOHGCbdu2kZiY6HcoIVOwYEEqVqxIvnz5/A7FGBNhIiIRbNu2jWLFilG1alXEn+5xQ0pViY+PZ9u2bVSrVs3vcIwxESZkRUPiOpXeLSKrMhgvIjJCRDaKyEoRaXqqy0pMTKRMmTIRmQQARIQyZcpE9BmPMcY/oawjGI/rHzUjnXGtYNbC9bI1+nQWFqlJIFWkr58xxj8hKxpS1XkiUjWTSboB73odo/woIiVFpPxJdHFnjDG5nyocPgy7d8OhQ3DiBBw/7p737IH8+d3r1Me2bVC8OCQlufdJSRw5kcCepANUvbwXnHdetofoZx1BBf7e3vo2b9g/EoGI9MWdNVC5cuUcCe5kxcTE0KhRI5KSkqhWrRoTJ06kZMmSAKxevZpBgwaxfft2UlJSuOmmm3jkkUf+PMqfOXMmjz76KAkJCRQoUIBLLrmE4cOH+7g2xkQYVUhMdDvio0fhwAE4dgwSEmDHDhCBX3+FIkX+2kkfOgR790LRon/tpA8dcjvqUqXcjjr1cegQ/O9/UKKEm+7gQdi5EwoVcss9jTbdvqkGt18OJY7BkiV1yRNhiSBoqjoWGAsQGxubK1vJK1SoEMuXLwfg5ptvZuTIkQwZMoSjR49yxRVXMHr0aDp06EBCQgJXX301o0aNYsCAAaxatYqBAwcyffp06tatS3JyMmPHjvV3ZYzxU3Ky27EmJroddmKi23nv2QPr10NMzN+PoHfscJ/Zvt3txJOTYe1aKFkSNmyAfPnczvpUd8YiULq0m0/evC6WIkXgjDP+GlasmIurcGE3vEABl2iKFIGzznLJpGxZSEmBChXcWUC+fO59iRJu+nz53CMmBooWZX/yEe6f/zhv/TyBmqVq8vLlb5KnWpts3NB/8TMRbOfvfcBW9IaFvQsvvJCVK1cCMGnSJFq0aEGHDq4HxcKFC/P666/Tpk0bBgwYwPPPP8+QIUOoW7cu4M4s7rgjIrtFNdEkKQn++MMVh2zfDvv3w5Ytbqe6bh3Ex7sdaOpRd0IC7NvndvxHj57aMitWhCNHoH59iI11y2zd2s2zbl13dJ6QANWquR2viNtJx8S4nXTRoq5IJnUnnS+fm8YHySnJXDS6Fevj1/PARQ/wRJsnKJQvo47bTp+fiSAOGCgiHwLnAweypX7g7rvBOzLPNk2awCuvBDVpcnIyc+bMoU+fPoArFmrWrNnfpqlRowaHDx/m4MGDrFq1invvvTd74zUmOyUluWKP+Hh39L1vn3u9fLnbsS5c6HaeW7e6I/KUFHdUnpX8+aFxY7cTLlbM7YjLlHFH0UePuh123rxuujPOcMMrVnRH0AUL/nU0nidy7ouNT4indKHSxOSJ4elLnqZSiUrEnh0b8uWGLBGIyAdAG6CsiGwDHgfyAajqG7i+R7vg+thNAG4JVSw54ejRozRp0oTt27dTr149Lr30Ur9DMiZriYmwZAn89hts3uwqNffscY9du9zR/O7dLhlkpFo1V+xyzTXu6LtWLbdTP3YMKld2R91lykClSq6IpXBhdxRu/qSqvP/z+9w16y6ebfcstze7navqXZVjyw/lVUM9shivwIBsX3CQR+7ZLbWOICEhgY4dOzJy5EjuvPNO6tevz7x58/427aZNmyhatCjFixenQYMGLF26lHPOOceXuE0USC0/nzvXVYiuXg1xca58PT1nn+0qQ8uXhwYN3NF45cruCLxmTbdjL1XKHcnbne6nbeuBrfSb3o8ZG2ZwQcULaFG5Rc4Hoaph9WjWrJmmtWbNmn8My2lFihT58/WyZcu0cuXKeuLECU1ISNBq1arpV199paqqCQkJetlll+mIESNUVXXFihVao0YNXb9+vaqqJicn6+jRo9NdRm5YT5NLJSaqrlmj+uWXqq+/rtq9u2qFCqpnn62aN6+qO2Z3j1KlVNu1U73hBtWHHlIdO1Z14ULVHTtUk5P9XpOoMmnlJC32TDEt/HRhfeWHVzQpOSlkywKWaAb71bC4aijcnHvuuTRu3JgPPviAXr168dlnnzFo0CAGDBhAcnIyvXr1YuDAgQA0btyYV155hR49epCQkICI0LVrV5/XwORqSUnw88+waBEsWODK5r/99u/TFCgAVaq4o/kePVyxTKNG0Ly5K7YxuUKpQqU4v+L5jO06lmql/Gs+Juz6LI6NjdW0HdOsXbuWevXq+RRRzomW9TS4Y/cNG1xxzrp17uqaVatcEc/27a5CNlWZMnDZZXDOOW5nX6eO2/Hb3ei5TlJKEi//8DLHk48zpNUQwJXK5ETLASKyVFXTrXm2MwJjcoPjx+GHH9yVOLNnu0fgzh6gRQto29aV0desCbVrw7nnukseTa63YtcK+sT1YenOpVzX4Lo/E0BuaD7GEoExOUnVVdb++CNs3Oiue//sM1e8k6pkSejWDerVg4sucgnAu0vdhJ9jScd4at5TPDv/WUoXKs2Ua6dwdb2rc0UCSBUxiSCnTq/8Em5FeCbAsmXwwguuiCe9e1zOPx/atIGmTaFLF6hRwy6vjCAb/tjAc/Of44ZGN/BSh5coU7iM3yH9Q0QkgoIFCxIfHx+xTVGr1x9BwYIF/Q7FZOXECVi8GKZMcRW4qUf94K6fv+kmd4Nio0buuXTpiLohyjiHjx/ms3Wf0bNxTxqe0ZB1A9dRvVR1v8PKUEQkgooVK7Jt2zb27Nnjdyghk9pDmcmF9uyB//4XPv0U3n33r+HFi7vr8C+7DG691d0VayLeV79+Rd8v+rJ5/2aalm9KvXL1cnUSgAhJBPny5bOeu0zO+e03+O47+OQTV96/adNf4zp0cMU711zjGhczUWPf0X3c9+V9jFs+jtplavNd7++oVy48rvKLiERgTEglJsKbb8LEia6tne1e24gFCrjGzJ580jWr0K2bawPHRJ3klGRajGvBL/G/8NDFD/FY68comDd8fguWCIxJz5EjMHo0zJwJ33zjhhUr5lq2vO026NQJmjWzJhai3N6EvX82EvdMu2eoXKIyTcufcq+7vrFEYAy4VjRnznRH/l999dc1/KVKwe23Q7t2cP31/sZocg1VZeLKidw9626ebf8sfZv15cq6V/od1imzRGCiU+r1/F99BePGubt2wV3BU6eOK98fPBg6drSreszfbN6/mX998S9m/zqbiypdRKsqrfwO6bRZIjDRIyUFJk1yV/h8+KFrMhlccwwdO8KNN8J117n2741Jx3sr3+OO6XegqrzW+TX6n9efPBL+BwqWCExkW7sWpk51O/85c/7qMKVCBRg4EK6+2l3PH4H3n5jsV65wOVpUasGYrmOoUrKK3+FkG0sEJvJs3QpvvAHTp8OKFW5YsWKugrd1a+jTx93IZUwWTiSfYPgPwzmRfIJHWz9Kx5od6VCjQ8TduGqJwESGY8fg/ffh7bdd08zgujQcOBB693bNN0TYn9eE1k87f6JPXB9+2vUT3Rt2z1WNxGU3SwQmfKm6u3mnTIGPP3YteFao4C7v7NULWoV/JZ7JeYlJiQz7bhjPz3+esoXL8vF1H/N/9f7P77BCyhKBCT/vvw/jx8PXX/817Oab4Yor4PLL7dp+c1o2/rGRFxe8yE3n3MTwDsMpVaiU3yGFnCUCk/ul9sg1bx68+CJs2+aGd+7sru/v2dO10W/MKTp8/DDT1k6j1zm9aHhGQ9YPXO9rj2E5zRKByb3WrIGnn4bPP3c9dIFruO2OO2D4cChUyN/4TESYvXE2fb/oy9YDW4k9O5Z65epFVRIASwQmt1F1l3pOmOCKf5KTXeudnTq5DlrsUk+TTeIT4hn85WDeXfEudcvW5ftbvg+bRuKymyUCkzts3gyvvw4vvfRX8w6tW7thDRv6G5uJOKmNxG38YyNDWg7hkVaPhFUjcdnNEoHx19Gj8PjjrgcvcDv9Sy+F7t2heXN/YzMRZ8+RPZQpXIaYPDE81/45qpSsQpOzmvgdlu8sERj/bNzorvJZtw5atoQxY1w/vcZkM1Vl/PLxDP5yMM+2e5Z/xf6LbnW7+R1WrhH+jWSY8HPoEDzyiGvDf9MmGDXKdfRiScCEwO/7f6fjex25Ne5WGp3RiLbV2vodUq5jZwQmZz3xBDz3nOvspWlTGDvWtetvTAhMXDGRO6bfgYgwqsso/hX7r4hoJC67WSIwOePAAbjrLnc1UPv28OCDcMkldgWQCakzi55JqyqteKPrG1QuUdnvcHItSwQm9CZNcs0+HD3qmn545x2IifE7KhOBTiSf4Pn5z5OsyTzW+jE61OhAhxod/A4r17NzJBM6P/7o7vrt2dPd/DVtGrz7riUBExLLdi7jvDfP45FvH2F9/HpU1e+QwoadEZjst3499O3rmoQoUMB1+DJyJBQv7ndkJgIdPXGUod8N5cUFL1KuSDmmXT8trLuN9ENIzwhEpJOIrBeRjSLyYDrjK4vItyLyk4isFJEuoYzHhJgq9O8Pdeu6JHDFFbBjB0ycaEnAhMymfZt46YeX6N2kN2v6r7EkcApCdkYgIjHASOBSYBuwWETiVHVNwGSPAB+p6mgRqQ/MAKqGKiYTIqnNQf+f11RvqVIwf75dDmpC5uCxg3yy9hN6N+lNgzMasGHQhojqMSynhfKMoDmwUVU3qepx4EMg7R0cCqQeKpYAdoQwHhMKmzdD7douCeTLB926wd69lgRMyMzYMIOGoxrSJ64Pa/esBbAkcJpCmQgqAFsD3m/zhgV6ArhRRLbhzgYGpTcjEekrIktEZMmePXtCEas5FatXw4UXumTwyCOueehPP4U8dg2CyX57E/bSa1ovLpt0GcUKFGP+rfOjtpG47OZ3ZXEPYLyqDheRC4GJItJQVVMCJ1LVscBYgNjYWLsUIDfYuBHOPx8SEuCrr1y/AMaESGojcZv2beKxVo/xcMuHKZC3gN9hRYxQJoLtQKWA9xW9YYH6AJ0AVPUHESkIlAV2hzAuc7qmTYMePVw/wbNnWxIwIfO/w/+jXJFyxOSJ4cVLX6RKySo0PrOx32FFnFCewy8GaolINRHJD3QH4tJMswVoByAi9YCCgJX95GZPPeXqA/LkcV1GdrCbdUz2U1XeXvY2dV6vw9ilYwG4vM7llgRCJGRnBKqaJCIDgdlADDBOVVeLyDBgiarGAfcCb4rIPbiK495qd4HkTvv2wS23wGefQZcu8MEHdkmoCYlN+zZx++e3881v39C6SmvaV2/vd0gRL6R1BKo6A1cJHDjssYDXa4AWoYzBZIMff4SrroJdu2DwYHj+ebs72ITEhOUT6D+jPzESwxuXvcHtzW63RuJygN+VxSY3S06GQYNcC6ElSsCcOa6hOGNC5OxiZ3NJtUsYfdloKhav6Hc4UcMSgUnfrl1Qpw4cPAhXXun6DChf3u+oTIQ5nnycZ//7LCmawhNtnuDSGpdyaY1L/Q4r6tg5l/mnb791zUQcPOjuE5g2zZKAyXaLty+m2dhmPD73cTbt22SNxPnIEoH5u1GjXPFPSgp88gksWOB3RCbCJJxI4L4v7+OCty9g39F9xHWP492r3kWsbwrfWNGQcU6ccC2Gjh8PF18MU6bAWWf5HZWJQL/t+43XFr3G7U1v57n2z1GiYAm/Q4p6lggMHDnizgIWLXJXB02e7NoNMiabHEg8wCdrP+GWc2+hwRkN2DhoI5VKVMr6gyZHWNFQtPvpJ6he3SWBHj0sCZhsN/2X6TQY1YDbPr+NdXvXAVgSyGUsEUSzKVNcx/FHjsDLL7suJS0JmGyy58geen7Sk64fdKVUoVL80OcH6pat63dYJh1WNBStli51ZwBnnOFuGKta1e+ITARJTknm4ncu5rd9vzG0zVAevPhB8sfk9zsskwFLBNFo2TJXIZw/P3z9tSUBk212Hd7FGUXOICZPDMM7DKdqyao0PKOh32GZLARdNCQihUMZiMkhv/0GrVpBUhJ89x00tD+pOX0pmsKYJWOo/VptxiwZA0DX2l0tCYSJLBOBiFwkImuAdd77c0RkVMgjM9nv66/hvPNcncAHH7jXxpymjX9spN277eg3vR/nVTiPjjU7+h2SOUnBFA29DHTEa0JaVVeISKuQRmWy37p1rvnoggVdf8IXXeR3RCYCvPPTO/Sf0Z/8Mfl58/I36XNuH7sxLAwFVUegqlvTfLnJoQnHhMSRI67zmORklwRq1fI7IhMhKpeoTMcaHRnZZSQViqftidaEi2ASwVYRuQhQEckH3AWsDW1YJtukpLgzgR07YMwYSwLmtBxLOsZ//vsfUjSFYW2H0a56O9pVtx7qwl0wlcX9gAG4jue3A02A/iGMyWSnhx+GL7+Ee+91TUgYc4oWbltIs7HNGPrdULYc2GKNxEWQYM4I6qhqz8ABItICmB+akEy2uesuGDEC2rZ1nckYcwqOHD/Co98+yis/vkKF4hX4oscXXFb7Mr/DMtkomDOC14IcZnKTr792SaBRI3dGkMduIjenZvOBzYxaPIp+sf1Y3X+1JYEIlOEZgYhcCFwElBORwQGjiuP6IDa51bJlrl6gRAmYMQPy2n2D5uTsT9zP1DVTua3pbdQvV5+Nd260HsMiWGZ7iPxAUW+aYgHDDwLXhDIocxqOHoXOneHQIVi+HCran9ecnM/WfcYd0+9g95HdXFz5YuqWrWtJIMJlmAhU9TvgOxEZr6qbczAmc6qOH4fLLoPdu2HoUDjnHL8jMmFk95Hd3DnzTiavnkzjMxsT1yPOGomLEsGUGSSIyAtAA6Bg6kBVtV7Mc5OUFOjSxXUzeeON8NhjfkdkwkhySjItxrVgy4EtPNX2KR5o8QD5Yqwl2mgRTCJ4H5gMdMVdSnozsCeUQZmTpAp33w1z5sCAAfCa1eWb4Ow4tIOzip5FTJ4YXu30KlVLVqV+ufp+h2VyWDCXkpRR1beBE6r6nareCtjZQG6RnAzdurmd/7XXume7xd9kIUVTGL14NHVfr8sbS94AoEutLpYEolQwZwQnvOedInIZsAMoHbqQzEm57Tb4/HPo1w9GjrQkYLL0S/wv3P757czbPI/21dvTuWZnv0MyPgsmETwlIiWAe3H3DxQH7g5lUCZITz7pOpu/4AIYNcqSgMnS28veZuDMgRTMW5BxV4yjd5Pe1kicyToRqOoX3ssDQFv4885i46ekJLfzB5g715KACUrVklXpXLMzI7uMpHyx8n6HY3KJzG4oiwGuw7UxNEtVV4lIV+BhoBBwbs6EaNL15JOwaxe89BIUKOB3NCaXOpZ0jCfnPQnAU5c8ZY3EmXRldkbwNlAJWASMEJEdQCzwoKp+mgOxmYyMGAHDhkH9+u5qIWPSsWDrAvrE9WHd3nXc2uRWVNWKgUy6MksEsUBjVU0RkYLALqCGqsbnTGgmXQ8+CM89BxUqwDvvWJGQ+YfDxw8zZM4QXlv0GpVKVGJWz1nWa5jJVGaXjx5X1RQAVU0ENp1sEhCRTiKyXkQ2isiDGUxznYisEZHVIjLpZOYfdcaPd0mgVSv4/Xdo3tzviEwutOXAFsYsHcOA8waw6o5VlgRMliSjNsVFJAHYmPoWqOG9F0BVtXGmM3Z1DL8AlwLbgMVAD1VdEzBNLeAj4BJV3SciZ6jq7szmGxsbq0uWLAlm3SJPoUKuGYmEBKsXMH+z7+g+pqyZQt9mrs+JHYd2cHaxs32OyuQmIrJUVWPTG5dZ0VC901xuc2Cjqm7ygvgQ6AasCZjmdmCkqu4DyCoJRLUpUyAxEVq3tiRg/mba2mn0n9GfPUf20LpKa+qUrWNJwJyUzBqdO92G5ioAWwPebwPOTzNNbQARmY9r2voJVZ2VdkYi0hfoC1C5cuXTDCsM7d8P3btDmTLwySd+R2NyiV2HdzFo5iCmrplKk7OaMP2G6dQpW8fvsEwY8ruh+rxALaANUBGYJyKNVHV/4ESqOhYYC65oKIdj9N8DD7hG5caMgdJ2U7dxjcS1fKclWw9s5ZlLnuG+i+6zRuLMKQtlItiOu/w0VUVvWKBtwEJVPQH8JiK/4BLD4hDGFV4++gjefBMuvhiuvNLvaIzPth3cxtnFziYmTwwjOo2gWqlq1lS0OW1B9V8oIoVE5GTPORcDtUSkmojkB7oDcWmm+RR3NoCIlMUVFW06yeVErqQkdzZQqxZMnw4x1jFctErRFF5b+Bp1X6/L6MWjAehcq7MlAZMtskwEInI5sByY5b1vIiJpd+j/oKpJwEBgNrAW+EhVV4vIMBG5wptsNhAvImuAb4H77T6FAIMHw+bNrm+B4sX9jsb4ZN3edbR6pxV3zrqTiytfTNfaXf0OyUSYDC8f/XMCkaW4Zqfnquq53rCfVbVRDsT3D1Fz+ejy5XDuudC4sXttN45FpbeWvcXAGQMpnK8wr3R6hV6Ne9ndweaUnOrlo6lOqOqBND++6KuwzWnPPeeex42zJBDFapSqweV1Luf1zq9zZtEz/Q7HRKhgEsFqEbkBiPFuALsTWBDasKLc1q3w4YfQowc0a+Z3NCYHJSYlMuy7YQA80+4Z2lZrS9tqbX2OykS6YCqLB+H6Kz4GTMI1R313CGOKbqpw773u9QMP+BuLyVHzt8ynyRtN+M9//8OeI3vIqtjWmOwSzBlBXVUdAgwJdTAGmDnT3UXcuzc0aeJ3NCYHHDp2iIfnPMzIxSOpUrIKs2+cTYcaHfwOy0SRYBLBcBE5C5gKTFbVVSGOKbo9/bR7fuklf+MwOWbbwW289dNbDGo+iKfbPU3R/EX9DslEmSyLhlS1La5nsj3AGBH5WUQeCXlk0WjSJFiwAK6+GkqV8jsaE0LxCfF/3g9Qr1w9Nt25iVc7v2pJwPgiqBvKVHWXqo4A+uHuKXgslEFFpV274J573E1jb7zhdzQmRFSVqWumUn9Ufe6cdSfr964HsG4jja+CuaGsnog8ISI/4zqvX4BrLsJkp5tvhj17YM4cKFvW72hMCOw8tJOrP7qaa6dcS6XilVhy+xJrJM7kCsHUEYwDJgMdVXVHiOOJTs88A19+CUOHumamTcRJbSRu+6HtPN/+ee658B7y5vG7zUdjnCx/iap6YU4EErWWLIEhQ1wCGGIXZkWarQe2UqF4BWLyxDCyy0iqlapG7TK1/Q7LmL/JsGhIRD7ynn8WkZUBj59FZGXOhRjBEhNdxTDAqFHWqFwESU5JZsTCEdQd+VcjcR1rdrQkYHKlzM4I7vKerYWrUBk0CLZsgddfh/r1/Y7GZJO1e9bSJ64PP2z7gc41O3N5ncv9DsmYTGV4RqCqO72X/VV1c+AD6J8z4UWw6dPhrbegUSPob5szUoxdOpYmY5rwS/wvTLxqItNvmE7lElHYq54JK8FcPnppOsM6Z3cgUUUV7r/fNSb3/ffWqFwEqVW6FlfVvYo1A9ZwY+MbraVQExYyLBoSkTtwR/7V09QJFAPmhzqwiPbgg7B2LbzwApQo4Xc05jQcPXGUJ+Y+gYjwbPtnrZE4E5YyqyOYBMwE/gM8GDD8kKr+EdKoItnatfD8865OYPBgv6Mxp2He5nncFncbG/7YQL9m/VBVOwMwYSmzoiFV1d+BAcChgAciYj2on6rLvYrDjz+GPEHd2G1ymYPHDtJ/en9aj29NsiYz56Y5jO462pKACVtZnRF0BZbiOqIJ/JUrUD2EcUWm3bvh11/dPQN1ra/ZcLXj0A7GLx/P4AsGM6ztMIrkL+J3SMaclgwTgap29Z6r5Vw4Ee6aa9zz8OH+xmFO2t6EvXy0+iP6n9efumXr8ttdv1mPYSZiBNPWUAsRKeK9vlFEXhIRux7uZK1f764Quu8+63UsjKgqk1dNpv7I+tw9625+if8FwJKAiSjBFFKPBhJE5BzgXuBXYGJIo4pEt93mngcO9DcOE7Qdh3Zw5eQr6f5xd6qUrMLSvkvtzmATkYJp9SpJVVVEugGvq+rbItIn1IFFlE2b4L//hUqVoEoVv6MxQUhOSabVO63Yfmg7L176InddcJc1EmciVjC/7EMi8hDQC2gpInmAfKENK8L06uWeP/nE3zhMljbv30zF4hWJyRPDqMtGUb1UdWqWrul3WMaEVDBFQ9fjOq6/VVV34foieCGkUUWS555zvY7deivExvodjclAckoyL/3wEvVG1mP0EtdIXIcaHSwJmKggqpr1RCJnAud5bxep6u6QRpWJ2NhYXbJkiV+LPzn797suJ8uVc43LFSzod0QmHat2r6JPXB8WbV9E19pdGX3ZaCoWt76XTGQRkaWqmu7RaDBXDV0HLAKuBa4DForINdkbYoRK7YD+xRctCeRSbyx5g6ZjmrJp3yYm/d8k4rrHWRIwUSeYOoIhwHmpZwEiUg74GpgaysDC3ubN8OSTUKYM9OzpdzQmjdTmIOqVrce1Da7llY6vUK5IOb/DMsYXwSSCPGmKguIJstP7qNbHu7Bq+nTrcCYXSTiRwGPfPkaMxPDcpc/RumprWle17kFNdAtmhz5LRGaLSG8R6Q1MB2aENqww9/XXrhP6c8+F88/3Oxrjmfv7XBqPbszwH4Zz+PhhgqkfMyYaBNNn8f0i8n/Axd6gsao6LbRhhbkePdzzzJn+xmEAOJB4gAe+eoCxy8ZSo1QNvrnpG2sq2pgAmfVHUAt4EagB/Azcp6rbcyqwsLV7N+zd6y4VPdOaIcgNdh7eyXs/v8d9F97H0LZDKZyvsN8hGZOrZFY0NA74Arga1wLpayc7cxHpJCLrRWSjiDyYyXRXi4iKSPhfaP/qq+75zTf9jSPK7Tmyh9cWup9s3bJ1+f2u33mhwwuWBIxJR2ZFQ8VUNXVvtl5Elp3MjEUkBhiJ6+pyG7BYROJUdU2a6YoBdwELT2b+udaYMa6J6SZN/I4kKqkqH6z6gDtn3snBYwfpWLMjtcvUtiuCjMlEZmcEBUXkXBFpKiJNgUJp3melObBRVTep6nHgQ6BbOtM9CTwHJJ509LnN9OkQHw9duvgdSVTaemArl39wOT0/6UnN0jX56V8/WSNxxgQhszOCncBLAe93BbxX4JIs5l0B2Brwfhvwt0tovIRSSVWni8j9Gc1IRPoCfQEqV87FLWA//PDfn02OSUpJos2ENuw6vIuXO77MoOaDiMljl+0aE4zMOqYJ6WUVXuN1LwG9s5pWVccCY8E1MRHKuE7Z4cOwcqVrXbRMGb+jiRq/7/+dSsUrkTdPXsZ0HUP1UtWpXso6zzPmZITyxrDtQKWA9xW9YamKAQ2BuSLyO3ABEBe2FcapvY4NHepvHFEiKSWJFxe8SL2R9Ri1eBQA7au3tyRgzCkIZQPri4FaIlINlwC6AzekjlTVA0DZ1PciMhd3iWqYtCgXIDERXn4ZGjWCm27yO5qIt/J/K+kT14clO5bQrU43rq5/td8hGRPWQpYIVDVJRAYCs4EYYJyqrhaRYcASVY0L1bJz3L//DQcOuGcRv6OJaKMWj+KuWXdRqmApJl8zmWvrX4vYNjfmtGTZDLW4f1lPoLqqDvP6Kz5LVRflRIBp5cpmqFN3RMeOQf78/sYSoVIbiZu3eR5vLnuTlzu+TNnCZbP+oDEGyLwZ6mDOCEYBKbirhIYBh4CP+at/guj27bfueehQSwIhcOT4ER755hHy5snLCx1eoFWVVrSq0srvsIyJKMFUFp+vqgPwrvNX1X2A7fFSPfssFC4Mgwf7HUnEmbNpDo1GN+KVha9wLPmYNRJnTIgEkwhOeHcJK/zZH0FKSKMKFwsXwpdfwvXXQ9GifkcTMfYn7ue2uNtoP7E9efPkZV7veYzoPMLqAowJkWASwQhgGnCGiDwN/Bd4JqRRhQNVuOUW9/rJJ/2NJcL87/D/+HDVh/y7xb9Z0W8FLau09DskYyJaMM1Qvy8iS4F2gABXqurakEeW2737LqxdC48/DhUq+B1N2Evd+d91wV3UKVuH3+/+3SqDjckhwVw1lG6bDqq6JSQRZSFXXDWk6jqkP3HCtS2UN5S3Y0Q2VeX9n9/nrll3cfj4YVbdsYpaZWr5HZYxEed0rxqajqsfEKAgUA1YDzTItgjDzaRJLgE8+6wlgdOw5cAW+n3Rj5kbZ3JhxQt5+4q3LQkY44NgioYaBb73GorrH7KIwsEzz7jK4fvu8zuSsJWUkkSb8W3YfWQ3IzqNoP95/a2ROGN8ctKHs6q6TESityPefftgzRq44ALrlP4UbNq3iSolqpA3T17evPxNapSuQdWSVf0Oy5iolmUiEJHAC+TzAE2BHSGLKLcbM8Y9P/64v3GEmaSUJIYvGM7jcx/n+Uuf587z76Rd9XZ+h2WMIbgzgmIBr5NwdQYfhyacXO74cRgxAs47Dzp29DuasLF813L6xPVh2c5lXFX3Kq6tf63fIRljAmSaCLwbyYqpqhWGg6sk3rnTVRLbzU1BeX3R69wz+x7KFCrD1GunWkuhxuRCGSYCEcnrtSDaIicDytVec52hc/31/sYRBlIbiWt8ZmN6NurJSx1fonSh0n6HZYxJR2ZnBItw9QHLRSQOmAIcSR2pqp+EOLbcJSkJli2DSy6BAgX8jibXOnz8MEPmDCFfTD5e7PCiNRJnTBgIpomJgkA8rvXRrsDl3nN0mTbNPXfo4G8cudiXv35Jw1ENeW3Ra5xIPmGNxBkTJjI7IzjDu2JoFX/dUJYq+v7hH30E+fLBPff4HUmus+/oPgZ/OZjxy8dTp0wd5t0yj4srX+x3WMaYIGWWCGKAovw9AaSKrkRw8CBMnQqdOlmfA+nYfWQ3U9dM5aGLH+Kx1o9RMG9Bv0MyxpyEzBLBTlUdlmOR5GZvveWe27f3N45cZNfhXXzw8wfcc+E9rpG4u36nTOEyfodljDkFmdUR2PWRqT791D3fdpuvYeQGqsqE5ROoP7I+D815iA3xGwAsCRgTxjJLBHbbJ8CKFfD999C3L5Qo4Xc0vvp9/+90er8TvT/rTf1y9Vneb7k1EmdMBMiwaEhV/8jJQHKtJ55wz/2ju529pJQk2k5oy96EvYzsMpJ+sf3II8FcdGaMye2sDeXMqMKsWVC2LJxzjt/R+GLjHxupVrIaefPkZdwV46heqjpVSlbxOyxjTDayQ7rMfP89JCbCgAF+R5LjTiSf4Jnvn6HBqAaMXDwSgLbV2loSMCYC2RlBZlJbGu3Z0984ctiyncvoE9eH5buWc239a7m+gTWpYUwks0SQmalToVIlqBU9FaIjFo5g8OzBlCtSjk+u+4Sr6l3ld0jGmBCzRJCR+HjX7HSnTn5HkiNSG4k796xzuemcmxjeYTilCpXyOyxjTA6wRJCRoUPdc58+/sYRYoeOHeKhOQ9RIKYAwzsOp2WVlrSs0tLvsIwxOcgqizMyYYJ7bt7c3zhCaNbGWTQc3ZBRi0ehqDUSZ0yUsjOC9Gzd6toX6tw5IjugiU+IZ/CXg3l3xbvUK1uP+bfO58JKF/odljHGJ5YI0jN3rnu+8ko/owiZ+KPxTFs7jUdbPcqQlkMokNf6VzAmmoW0aEhEOonIehHZKCIPpjN+sIisEZGVIjJHRHLHReqTJ7vOZ2680e9Iss3OQzt5ccGLqCq1y9Rm892bGdZ2mCUBY0zoEoHX3/FIoDNQH+ghIvXTTPYTEKuqjYGpwPOhiidoiYnw9dfuktHChf2O5rSpKuN+Gke9kfV49NtH2fjHRgC7IsgY86dQnhE0Bzaq6iZVPQ58CHQLnEBVv1XVBO/tj0DFEMYTnEcfhWPH4L77/I7ktP227zc6vNeBPnF9OOesc1jRb4U1EmeM+YdQ1hFUALYGvN8GnJ/J9H2AmemNEJG+QF+AypUrZ1d86Rs92j3fdFNolxNiSSlJXPLuJcQnxDP6stH0bdbXGokzxqQrV1QWi8iNQCzQOr3xqjoWGAsQGxsbumscFyyAI0fgmmvC9mqhDfEbqF6qOnnz5OWdbu9Qo1QNKpWo5HdYxphcLJSHiNuBwD1QRW/Y34hIe2AIcIWqHgthPFlLLQ569llfwzgVJ5JP8NS8p2g4uiGvL3odgDZV21gSMMZkKZRnBIuBWiJSDZcAugM3BE4gIucCY4BOqro7hLFkbcEC+OEHePhhqFHD11BO1pIdS+gT14eV/1tJ94bd6dGoh98hGWPCSMgSgaomichAYDYQA4xT1dUiMgxYoqpxwAtAUWCKuKKYLap6RahiytTHH7vna6/1ZfGn6tUfX2Xwl4M5q+hZfNb9M66o48/mM8aELwm3ZgViY2N1yZIl2T/j4sWhQQN3VhAGUhuJm79lPhNWTOD5S5+nZMGSfodljMmlRGSpqsamNy5XVBb7btMmOHQImjb1O5IsHTx2kH9/9W8K5i3Iy51epkXlFrSo3MLvsIwxYcyuJwSYMsU95/JLRmdsmEGDUQ0Yu2wsefPktUbijDHZws4IwNUP1KwJ52d2m4N/9ibs5e5Zd/P+z+/ToFwDpl47lfMr5s5YjTHhx84IADZvhmbN/I4iQ/uO7uPzXz7n8daPs+xfyywJGGOylZ0R/PIL7N4NjRr5HcnfbD+4nfd/fp/7L7qfWmVqsfnuzVYZbIwJCTsjmDXLPXft6m8cHlXlzaVvUn9UfZ6Y+wS/7vsVwJKAMSZkLBF8/z2UKweNG/sdCb/+8Svt3m1H3y/60rR8U1besZKapWv6HZYxJsJFd9FQUhLMnOk6oPG5baGklCTavduOP47+wZiuY7it6W3WSJwxJkdEdyKIi3ONzHXo4FsI6/eup0bpGuTNk5cJV06gRukaVCzuf2vcxpjoEd2HnAsWuOe2bXN80ceTjzN07lAajW7EyEUjAWhdtbUlAWNMjovuM4J169xzpZxtoXPR9kX0ievDqt2ruKHRDfRs3DNHl2+MMYGiNxEcOeK6pKxePUcX+8qPr3Dvl/dSvmh5Pu/xOV1r546rlYwx0St6E8Enn7guKR97LEcWl9pIXPMKzbm96e081/45ShQskSPLNsaYzERvIpg8GQoUgJ6hLZY5kHiAB756gEL5CvFKp1e4qNJFXFTpopAu0xhjTkb0Vhb/8guULw95Q5cLP1//OfVH1eetn96iQEwBayTOGJMrRecZwZEjsGED3HxzSGa/58ge7pp1Fx+s+oBGZzTi0+s/5bwK54VkWcYYc7qiMxG8/7577hGaLh0PHDvAjA0zGNpmKA9e/CD5Y/KHZDnGGJMdoq+HMlWoXRsOHoQdOyAmJlvi2npgK++tfI8HL34QEeFA4gGrDDbG5BqZ9VAWfXUEy5bBxo3w4IPZkgRSNIU3lrxBg1ENeOr7p/5sJM6SgDEmXERfIli0yD1fcslpz2pD/AYumXAJd0y/g+YVmvPzHT9bI3HGmLATfXUEs2ZBvnxQv/5pzSYpJYlLJ17K/sT9vH3F29zS5BbE54brjDHmVERfIpg/H+rWdcngFKzds5ZaZWqRN09eJl41kRqla3B2sbOzOUhjjMk50VU0dPw4xMdDkyYn/dFjScd4/NvHafxGY15f9DoALau0tCRgjAl70XVGsHGje27Z8qQ+9uO2H+kT14c1e9bQq3EvejXuFYLgjDHGH9GVCBYudM8n0dro8AXDuf+r+6lYvCIzbphB51qdQxScMcb4I7oSQWqz00F0S5miKeSRPFxY6UL6xfbj2fbPUrxA8RAHaIwxOS+6EsGOHe65fPkMJ9mfuJ97Z99L4XyFea3La9ZInDEm4kVXZfHWre6u4gwu8/x03afUH1mfCSsmUKxAMWskzhgTFaLnjEAVVq+G1q3/MWr3kd0MnDGQKWum0OSsJnxxwxc0Ld/UhyCNMSbnRU8i2LUL9u6Fc875x6iDxw7y1aavePqSp7n/ovvJF3Nq9xgYY0w4ip5EsH27e65cGYAtB7YwccVEHm75MDVL12TL3VsoVqCYjwEaY4w/QlpHICKdRGS9iGwUkQfTGV9ARCZ74xeKSNWQBRMfD0BKqZKMWjyKBqMa8Mx/n/mzkThLAsaYaBWyRCAiMcBIoDNQH+ghImkb+OkD7FPVmsDLwHOhiocNG1hfBtpsGcaAGQO4sOKFrO6/2hqJM8ZEvVCeETQHNqrqJlU9DnwIdEszTTdggvd6KtBOQtRyW1LScTr2gp8P/8o73d5h9o2zqVqyaigWZYwxYSWUdQQVgK0B77cB52c0jaomicgBoAywN3AiEekL9AWo7JXxn6y81Wrw3tKW1Hh8EuVLVjyleRhjTCQKi/sIVHWsqsaqamy5cuVObSbdunHxxHmWBIwxJo1QJoLtQGCjPhW9YelOIyJ5gRJAfAhjMsYYk0YoE8FioJaIVBOR/EB3IC7NNHHAzd7ra4Bv1G7nNcaYHBWyOgKvzH8gMBuIAcap6moRGQYsUdU44G1goohsBP7AJQtjjDE5KKQ3lKnqDGBGmmGPBbxOBK4NZQzGGGMyFxaVxcYYY0LHEoExxkQ5SwTGGBPlLBEYY0yUk3C7WlNE9gCbT/HjZUlz13IUsHWODrbO0eF01rmKqqZ7R27YJYLTISJLVDXW7zhykq1zdLB1jg6hWmcrGjLGmChnicAYY6JctCWCsX4H4ANb5+hg6xwdQrLOUVVHYIwx5p+i7YzAGGNMGpYIjDEmykVkIhCRTiKyXkQ2isiD6YwvICKTvfELRaSqD2FmqyDWebCIrBGRlSIyR0Sq+BFndspqnQOmu1pEVETC/lLDYNZZRK7zvuvVIjIpp2PMbkH8tiuLyLci8pP3++7iR5zZRUTGichuEVmVwXgRkRHe9lgpIk1Pe6GqGlEPXJPXvwLVgfzACqB+mmn6A294r7sDk/2OOwfWuS1Q2Ht9RzSsszddMWAe8CMQ63fcOfA91wJ+Akp578/wO+4cWOexwB3e6/rA737HfZrr3ApoCqzKYHwXYCYgwAXAwtNdZiSeETQHNqrqJlU9DnwIdEszTTdggvd6KtBORCQHY8xuWa6zqn6rqgne2x9xPcaFs2C+Z4AngeeAxJwMLkSCWefbgZGqug9AVXfncIzZLZh1VqC497oEsCMH48t2qjoP1z9LRroB76rzI1BSRMqfzjIjMRFUALYGvN/mDUt3GlVNAg4AZXIkutAIZp0D9cEdUYSzLNfZO2WupKrTczKwEArme64N1BaR+SLyo4h0yrHoQiOYdX4CuFFEtuH6PxmUM6H55mT/71kKacc0JvcRkRuBWKC137GEkojkAV4CevscSk7LiyseaoM765snIo1Udb+fQYVYD2C8qg4XkQtxvR42VNUUvwMLF5F4RrAdqBTwvqI3LN1pRCQv7nQyPkeiC41g1hkRaQ8MAa5Q1WM5FFuoZLXOxYCGwFwR+R1XlhoX5hXGwXzP24A4VT2hqr8Bv+ASQ7gKZp37AB8BqOoPQEFc42yRKqj/+8mIxESwGKglItVEJD+uMjguzTRxwM3e62uAb9SrhQlTWa6ziJwLjMElgXAvN4Ys1llVD6hqWVWtqqpVcfUiV6jqEn/CzRbB/LY/xZ0NICJlcUVFm3IwxuwWzDpvAdoBiEg9XCLYk6NR5qw44Cbv6qELgAOquvN0ZhhxRUOqmiQiA4HZuCsOxqnqahEZBixR1Tjgbdzp40ZcpUx3/yI+fUGu8wtAUWCKVy++RVWv8C3o0xTkOkeUINd5NtBBRNYAycD9qhq2Z7tBrvO9wJsicg+u4rh3OB/YicgHuGRe1qv3eBzIB6Cqb+DqQboAG4EE4JbTXmYYby9jjDHZIBKLhowxxpwESwTGGBPlLBEYY0yUs0RgjDFRzhKBMcZEOUsEJlcSkWQRWR7wqJrJtIezYXnjReQ3b1nLvDtUT3Yeb4lIfe/1w2nGLTjdGL35pG6XVSLyuYiUzGL6JuHeGqcJPbt81ORKInJYVYtm97SZzGM88IWqThWRDsCLqtr4NOZ32jFlNV8RmQD8oqpPZzJ9b1yrqwOzOxYTOeyMwIQFESnq9aOwTER+FpF/tDQqIuVFZF7AEXNLb3gHEfnB++wUEclqBz0PqOl9drA3r1Uicrc3rIiITBeRFd7w673hc0UkVkSeBQp5cbzvjTvsPX8oIpcFxDxeRK4RkRgReUFEFnttzP8riM3yA15jYyLS3FvHn0RkgYjU8e7EHQZc78VyvRf7OBFZ5E2bXoutJtr43fa2PeyR3gN3V+xy7zENdxd8cW9cWdxdlalntIe953uBId7rGFx7Q2VxO/Yi3vB/A4+ls7zxwDXe62uBhUAz4GegCO6u7NXAucDVwJsBny3hPc/F6/MgNaaAaVJjvAqY4L3Oj2tFshDQF3jEG14AWAJUSyfOwwHrNwXo5L0vDuT1XrcHPvZe9wZeD/j8M8CN3uuSuLaIivj9fdvD30fENTFhIsZRVW2S+kZE8gHPiEgrIAV3JHwmsCvgM4uBcd60n6rqchFpjeusZL7XtEZ+3JF0el4QkUdw7dT0wbVfM01Vj3gxfAK0BGYBw0XkOVxx0vcnsV4zgVdFpADQCZinqke94qjGInKNN10JXGNxv6X5fCERWe6t/1rgq4DpJ4hILVwzC/kyWH4H4AoRuc97XxCo7M3LRClLBCZc9ATKAc1U9YS4FkULBk6gqvO8RHEZMF5EXgL2AV+pao8glnG/qk5NfSMi7dKbSFV/EdfXQRfgKRGZo6rDglkJVU0UkblAR+B6XEcr4HqbGqSqs7OYxVFVbSIihXHt7wwARuA64PlWVa/yKtbnZvB5Aa5W1fXBxGuig9URmHBRAtjtJYG2wD/6XBbXD/P/VPVN4C1cd38/Ai1EJLXMv4iI1A5ymd8DV4pIYREpgivW+V5EzgYSVPU9XGN+6fUZe8I7M0nPZFxDYalnF+B26nekfkZEanvLTJe63ubuBO6Vv5pST22KuHfApIdwRWSpZgODxDs9EtcqrYlylghMuHgfiBWRn4GbgHXpTNMGWCEiP+GOtl9V1T24HeMHIrISVyxUN5gFquoyXN3BIlydwVuq+hPQCFjkFdE8DjyVzsfHAitTK4vT+BLXMdDX6rpfBJe41gDLxHVaPoYszti9WFbiOmZ5HviPt+6Bn/sWqJ9aWYw7c8jnxbbae2+inF0+aowxUc7OCIwxJspZIjDGmChnicAYY6KcJQJjjIlylgiMMSbKWSIwxpgoZ4nAGGOi3P8DTMIRoXKOfcIAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SML AUROC score: 0.825673425\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, auroc_score = auroc(test_y, ood_scores)\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "print('SML AUROC score:', auroc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAnn0lEQVR4nO3de5xd0/3/8dc7k5BEBE3imkjUPQhhSDWKxl2JtqoVfFX5UdpUfaupa79urab3Vht1KV9U0VYvoviqS9AQakIQ0hCpS6KIS0RIkOTz+2Pt6UzGJHNmcvbZZ+a8n4/Heex99l5n78/OcD5nr7XXWooIzMysdnUrOgAzMyuWE4GZWY1zIjAzq3FOBGZmNc6JwMysxjkRmJnVOCcC6zQkHSnpbyWUu0TStysRUyVIek7S3tn6uZKuLTom61qcCKwssi+rRZIWSnpF0lWS+pTzHBHx24jYt4RyJ0bEBeU8dyNJIemd7DrnSvqJpLo8ztURkvpK+pmkF7IYn83e9y86NqteTgRWTgdHRB9gR6AeOLtlAUndKx5V+W2fXecewBeAYwuOBwBJqwF3AdsA+wN9gV2B14FdOnC8rvC3shI4EVjZRcRc4DZgW/jPr+ivSnoGeCbbdpCkaZLmS3pA0rDGz0saJOlPkuZJel3SL7Ptx0ianK1L0k8lvSppgaQnJDWe7ypJ32l2vOMlzZL0hqSJkjZsti8knSjpmSyWCZJU4nXOAu4Hdmh2vI5c16aS7s62vSbpt5LWbuc/O8DRwMbAZyLiqYhYFhGvRsQFEXFrs+vdrFlM//m3krSnpDmSTpP0MvC/kmZIOqhZ+e5Z/Dtm7z+WXed8SY9J2rMDcVvBnAis7CQNAg4EHm22+dPACGCopOHAlcCXgX7ApcBESatn1Sx/BZ4HhgAbATe0cpp9gd2BLYC1gM+Tfvm2jGUU8L1s/wbZcVse7yBgZ2BYVm6/Eq9zK+ATwKzsfUevS1mMGwJbA4OAc0uJoYW9gf+LiIUd+Gyj9YGPAIOBE4DrgTHN9u8HvBYRj0jaCLgF+E72mW8Cf5Q0YBXObwVwIrBy+ouk+cBk4F7gwmb7vhcRb0TEItIXzKUR8VBELI2Iq4H3gI+RqjA2BMZFxDsRsTgiJrdyrg+ANYGtAEXEjIj4dyvljgSujIhHIuI94AxgV0lDmpUZHxHzI+IFYBLNfuGvwCOS3gFmAPcAF2fbO3RdETErIu6IiPciYh7wE1K1U3v1A1r7N2iPZcA5WSyLgOuA0ZJ6Z/uPICUHgKOAWyPi1uzu4w6ggfQjwDoRJwIrp09HxNoRMTgivpJ9kTR6sdn6YODUrDphfpY8BpG+KAcBz0fEkpWdKCLuBn4JTABelXSZpL6tFN2Q9Cu88XMLSXcOGzUr83Kz9XeBPgCSnswaXBdK+kSzMjtmZb5AustZY1WuS9J6km7IGp8XANcCHWncfZ1017Mq5kXE4sY3WfXXDODgLBmMJiUHSNd7WIvr3a0MMViFORFYpTQf5vZF4LtZ0mh89Y6I67N9G5fSUBkRF0XETsBQUhXRuFaKvUT6wgJA0hqkX85zSzj+NhHRJ3v9vcW+iIjfA1OA/1nF67qQ9O+zXUT0Jf3SLqmdooU7gf2ya1yRd4Hezd6v32J/a8MRN1YPHQI8lSUHSNf0mxbXu0ZEjO9A7FYgJwIrwuXAiZJGZI2+a0j6lKQ1gX+QqjfGZ9t7ShrZ8gCSds4+3wN4B1hMqtZo6XrgS5J2kLQ66Uv3oYh4rkzXMh44XtL6q3BdawILgbeyevfWElopfkP6cv6jpK0kdZPUT9KZkhqra6YBR0iqk7Q/pVVB3UBqkzmJprsBSHcuB0vaLztez6zBeWAH47eCOBFYxUVEA3A8qWrnTVJj6zHZvqXAwcBmwAvAHFIVTEt9SV+8b5Kqfl4HftjKue4Evg38kfRFvClweBmv5QngPlLdf0ev6zxSddNbpMbXP3UwlvdIDcb/BO4AFpASUH/goazY17M45pPaT/5SwnH/Tbrz+Tjwu2bbXyTdJZwJzCMloXH4e6XTkSemMTOrbc7cZmY1zonAzKzGORGYmdU4JwIzsxrX6QaV6t+/fwwZMqToMMzMOpWpU6e+FhGtDv/R6RLBkCFDaGhoKDoMM7NORdLzK9rnqiEzsxrnRGBmVuOcCMzMalynayMwM+uoDz74gDlz5rB48eK2C3dSPXv2ZODAgfTo0aPkzzgRmFnNmDNnDmuuuSZDhgyhxInoOpWI4PXXX2fOnDlssskmJX8ut6ohSVcqTSM4fQX7JekipSkEH2+c+s7MLC+LFy+mX79+XTIJAEiiX79+7b7jybON4CrSBNorcgCwefY6AfhVjrGYmQF02STQqCPXl1siiIj7gDdWUuQQ4Jpsgo8HgbUl5Tez0eTJcN558MILuZ3CzKwzKvKpoY1YfvrCOSw/feB/SDpBUoOkhnnz5nXsbFOmwLnnwtVXd+zzZmZlUFdXxw477MC2227LwQcfzPz58/+z78knn2TUqFFsueWWbL755lxwwQU0nyrgtttuo76+nqFDhzJ8+HBOPfXUssTUKR4fjYjLIqI+IuoHDGi1h3TbxmWTPi1dWr7AzMzaqVevXkybNo3p06fzkY98hAkTJgCwaNEiRo8ezemnn87MmTN57LHHeOCBB7j44osBmD59OmPHjuXaa6/lqaeeoqGhgc0226wsMRWZCOaSJvRuNJAS5pE1M+sqdt11V+bOTV971113HSNHjmTfffcFoHfv3vzyl79k/Pg0BfQPfvADzjrrLLbaaisg3VmcdNJJZYmjyMdHJwJjJd0AjADeyqbEMzPL3ymnwLRp5T3mDjvAz35WUtGlS5dy1113cdxxxwGpWminnXZarsymm27KwoULWbBgAdOnTy9bVVBLuSUCSdcDewL9Jc0BzgF6AETEJcCtwIGkeV3fBb6UVyxmZtVi0aJF7LDDDsydO5ett96affbZp+iQ8ksEETGmjf0BfDWv85uZrVSJv9zLrbGN4N1332W//fZjwoQJnHzyyQwdOpT77rtvubKzZ8+mT58+9O3bl2222YapU6ey/fbblz2mTtFYbGbW1fTu3ZuLLrqIH//4xyxZsoQjjzySyZMnc+eddwLpzuHkk0/mW9/6FgDjxo3jwgsv5OmnnwZg2bJlXHLJJWWJxYnAzKwgw4cPZ9iwYVx//fX06tWLm266ie985ztsueWWbLfdduy8886MHTsWgGHDhvGzn/2MMWPGsPXWW7Ptttsye/bsssThsYbMzCpo4cKFy72/+eab/7O+3Xbbcc8996zwswcddBAHHXRQ2WPyHYGZWY1zIjAzq3G1lwheegkmTiw6CjMrSPMhG7qijlxf7SWCyy+HQw4pOgozK0DPnj15/fXXu2wyaJyPoGfPnu36nBuLzaxmDBw4kDlz5tDhwSs7gcYZytrDicDMakaPHj3aNXNXrai9qqHWLFtWdARmZoWp3URwwQVpecIJUFcHt91WbDxmZgWp3UTwP/8Dzz6bGo8Bzjmn2HjMzApSu4kA4E9/alqfOrW4OMzMClTbiSAbzAlI7QRvrGyKZTOzrqm2EwGkiSQafepThYVhZlYUJ4JRo2DjjdP6gw8WG4uZWQGcCL78ZXj++aKjMDMrTG13KPv3v2H99dP6mmvCOusUG4+ZWQFq847glFNg0KCmJADw9tvwwguFhWRmVpTaTAQ//emKv/QnTQIp3S2YmdWA2koE//wn/PnPre8bOTItR41Ky3HjKhOTmVnBaisRbLklfPrTre878sjl37/6Kmy9Nfz977mHZWZWpNpKBCvz7LPLv7/jjnQHsfvu8MgjEAFLlhQTm5lZjpwIGh166Ir37bQTdOsGPXrAtdfCa69VLi4zs5w5ETTadVeYNw9efLFp25tvfrjcf/0XDBiQGpR//vPKxWdmlhMngub694eBA9MTRa+8AmuvDVOmrLj8KadUKjIzs9w4EbRm0CBYd920/rGPpfaBCDj6aBg8GHbZpans4sUpcXjAOjPrpGq7Z3F7XX110/phh8GNN0KvXk3bli1LVUZmZp2I7wg66owzPrytWzd4//3Kx2JmtgqcCDpqxx1hm23S+gYbNG33TGdm1sk4EayKJ56AmTPhpZfgrLPStvHji43JzKydck0EkvaXNFPSLEmnt7J/sKS7JD0u6R5JA/OMp+wk2GKLtP6d7zRtf+CBYuIxM+uA3BKBpDpgAnAAMBQYI2loi2I/Aq6JiGHA+cD38oqnIi68MC3/9rdi4zAza4c87wh2AWZFxOyIeB+4ATikRZmhwN3Z+qRW9ncuxx6blnPmFBuHmVk75JkINgKaddNlTratuceAz2brnwHWlNSv5YEknSCpQVLDvHnzcgm2LBr7HlxxRRrO2sysEyi6sfibwB6SHgX2AOYCS1sWiojLIqI+IuoHDBhQ6RhL17wPwahRMGNGcbGYmZUoz0QwFxjU7P3AbNt/RMRLEfHZiBgOnJVtm59jTPm74Yam9aFDYc89U4Lo1y89WhpRWGhmZq3JMxE8DGwuaRNJqwGHAxObF5DUX1JjDGcAV+YYT2V84Qswt1m+u/fetHzjDTj/fDjttGLiMjNbgdwSQUQsAcYCtwMzgN9HxJOSzpc0Oiu2JzBT0tPAesB384qnojbcEK6/vvV9P/wh7LNPZeMxM1sJRSerqqivr4+GhoaiwyjN4sVQV5fmMQDo3h2WZk0gixZBz56pTI8eqfqoW9FNNmbWVUmaGhH1re3zN0+eevZsSgKw/AxnvXqloSl69UoJoq4uJYNOlpjNrPNzIqi0xjYDgJdf/vD+bt1SQpBgr73SHQOk3srnnZeGszAzKyMngkrbffc0w1mjhQvTXcDFF3+47N13pzsGCUaOhHPPhY02Su//+MeKhWxmXZsTQRFefbVpsps11kjbTjoJPvgApk5t6pi2Mp/7XNOdwx/+sHy1k5lZOzgRVJPu3dPw1q+8kpLEK6+kx02nTGlKHL/+9Yc/9/nPp7aIr3wlJRMzs3bwU0Od1cKFcNNNcNRRH953772pCsrMLOOnhrqiPn3gyCOb7hQaRz4F2GOP4uIys07HiaCrOOOM5R89lWD+/MLCMbPOw4mgq7nzzqb1ddZJCeHEE4uLx8yqnhNBV7PXXjB58vLbLr20qbPas8+mbe+8U/nYzKwqORF0RSNHpi/9Dz6As89u2t6tG2y2WUoKffqk5aBBTY+hfvABvP02PP88LFtWXPxmVlFOBF1Z9+5wwQXwwgsrLtN8NrXVVoO+fWHIkKYhLxpfI0bANdfkHrKZVZ4TQS0YNKjp6aJly1Lnswh4+ml45hl47bXUg3ll/vEP+OIXmxLDNde4esmsi3A/Alu5l16Cp55Kyy9+ccXlBg2Cb3wjrX/96ynRLFmS7ko8qqpZ4VbWj8CJwNrvu99dvu2hVDffDJ/61PJTeppZRbhDmZXXWWelX/zvvZeqml5+GQ44AA48cOWfO/jgptFVP/OZVDX117/Cm29WJm4za5UTgXXcaqulL/X11oNbb4Vbbmlqi2j+mjsXzjxz+c/+5S+w5ZYpOXzkI+k448YVchlmtc6JwPK34YapOiki9Xb+4Q+hf3/Ydtvly/3oRykhbL65J+gxqyAnAqustdaCb34T5s2DJ55oalRuPsfzrFmpCmmPPdK6meXKicCKV1cHhx/eVI3U6L770t1B9+7pTuGgg2DYMHj88eJiNeuCnAisumy4YUoIL7yQvvi7dYOlS9O+W25JdxHbb9/Un2H//VNvaDPrMCcCq06DBqXHTZcubRoj6fbbYezY5cvdfnvqDd28F7QEm24K++yTOsp97Wtw3XVNM8OZ2XLcj8A6r0cfTVN2zp7dvs9tthncdltamtUI9yOwrmn48HSn0PJx1XnzYPr01N5w662w667prqHRrFmp7UGCj388JZKlS2HRouKuxaxATgTW9fTvD9tsk9obDjgAHngA3norJYm33kqPqTaaMiVVI3XvDr17N836ZlZDnAistvTtC6eemr7sFy9Ojc39+8NWW6X9113X1Pv5+ONT72mzLs6JwGrX6quntoJ582DGDFi4MI2F1OjXv4aePZdvhF5rLfjnP1OV1MqG9zbrRJwIzBqtsUYa+ygifdm3ZsEC2Hrr1NA8eHBKDscdt3z/B7NOxonArDVbbvnhRuglS+AXv0jtDkcc0VT2yith4MBUxXTssfCb3zT1fTDrBPz4qNmqeOcd+Oxn4W9/+/C+3XaD005LHePMClbY46OS9pc0U9IsSae3sn9jSZMkPSrpcUltjGNsVmXWWCN1aouAF19MYyYNG5b2TZ6cRldda61UhdR8PCWzKpJbIpBUB0wADgCGAmMkDW1R7Gzg9xExHDgcuDiveMxyN3BgGjPpscfSY6pXX50anxcsSPuPOCIlhB494Pnni43VrJk87wh2AWZFxOyIeB+4ATikRZkAGnv6rAW8lGM8ZpXTty8cfXRT4/MDDzTtW7IEhgxJSeHQQ2HSJHj33fQ4q9sWrADdczz2RsCLzd7PAUa0KHMu8DdJXwPWAPZu7UCSTgBOANh4443LHqhZ7nbdtanR+frrU8c1gD/9Kb2a23nn1K9hwID02ndf2HHHysdsNaOkOwJJIyXdIelpSbMl/UtSOwd4adUY4KqIGAgcCPxG0odiiojLIqI+IuoHDBhQhtOaFURKVUTLlqXqoSuvTNubT9Lz8MPpyaOf/ATOOAN22qmpH8MnPpGeaLruumLity6p1KqhK4CfALsBOwP12XJl5gKDmr0fmG1r7jjg9wARMQXoCfQvMSazzkuCjTeGL30p3SU0TtITkeaAnjYNLrsMLrkEPvrRps9Nnpzmej7yyHSMT34y9Wn41a+ayixeXPHLsc6tpMdHJT0UES2rddr6THfgaWAvUgJ4GDgiIp5sVuY24HcRcZWkrYG7gI1iJUH58VGraQsXwlNPwYgS/nc85xw499zcQ7LOoRyPj06S9ENJu0rasfG1sg9ExBJgLHA7MIP0dNCTks6XNDordipwvKTHgOuBY1aWBMxqXp8+sMsu6c6hca6G5j+MDj20af2889Jdg+8QrA2l3hFMamVzRMSo8oe0cr4jMCvBggWp/0KjXr1Sn4e11059G446yg3QNWZldwTuWWzWVUXAd7+bGqT/9a8Vl3vnnTQEt3Vpq1w1JGktST+R1JC9fixprbY/aWaFkeDss9PEO40N0QsXpilABw9uKrfGGulppEceKS5WK1SpbQRXAm8Dn89eC4D/zSsoM8vJGmuksY+eey49wrrXXmn75MnpMdXu3eHyy9M+qxmlJoJNI+KcrJfw7Ig4D/hom58ys+olwZ13pi/9yZNTb+elS+GEE6CuLu0/6aT0uKp1aaUmgkWSdmt8I2kk4AlezboCCUaOTO0ILdvfLrkkdWBr7ND2ve95Ks8uqNREcBIwQdJzkp4HfgmcmF9YZlaInXZafg6Gm2+GMWNS5zeAM89MU3nusAPU18ONN/rx1C6gXU8NSeoLEBELcouoDX5qyKwgM2fCV7+aekG/+uqH9//hD/C5z1U+LivJyp4aWumgc5KOiohrJX2jxXYAIuInZYvSzKrbllumNoVGV12VxkW6OBs9/rDD0nL4cDjrrOU7t1lVa6tqaI1sueYKXmZWq445BiZMSFVIM2ZAv35p+6OPpjsDKU3Sc+mlbleocu5QZmbldf/9aYKeOXOW3/7MM2mAPCtEOTqU/UBSX0k9JN0laZ6ko8obppl1CSNHpmk7G6fvbLT55ukuoeX8C1a4Up8a2jdrID4IeA7YDBiXV1Bm1kUMHJgSwm9/27Tt0EOhZ8/UtuCOa1Wh1ETQ2Kj8KeAPEfFWTvGYWVfUOBnPzJkpObz3XnoCqa4ujZJqhSo1EfxV0j+BnYC7JA0A/PCwmZVOgi22SNVF774Le2cz0557btr39a+7UbkgJTcWS/oI8FZELJXUG+gbES/nGl0r3Fhs1oW8/jr0bzEp4Z13No2BZGXT4cZiSaOy5WeBPYFDsvX9gY+XOU4zqzX9+jVNsvOtb6Vte+8NQ4fC735XbGw1pK2qoT2y5cGtvA7KMS4zqyXdusH3v58eMd1vv9Qv4fDD0/Yjj3Sjcs7cj8DMqs+998Keey6/bdNNU+/ls89Ow2lbu5SjH8GFktZu9n4dSd8pU3xmZsvbY49UZbRsGRx7bNr27LMwfnyat1lKA+JZWZT61NABETG/8U1EvAkcmEtEZmaNJLjiipQUXnoJPv/5pn2jR8Nf/1pcbF1IqYmgTtLqjW8k9QJWX0l5M7Py2mCD1IAcATfdlLYdfHBKFjfeWGxsnVypieC3pP4Dx0k6DrgDuDq/sMzMVmL0aHj88ab3hx0GG27ofggdtNJhqBtFxPclPQZkPUC4ICJuzy8sM7M2bLdd+uJ/6CH42Mfg3/9OTxktWpSGsLCSlZQIMjOAJRFxp6TektaMiLfzCszMrCQjRqRZ0gYNgnnzoFcvWHvtNNfygAFFR9cplPrU0PHAjcCl2aaNgL/kFJOZWfusvjq8/DKccUZ6P38+rLtuGtdo4sRCQ+sMSm0j+CowElgAEBHPAOvmFZSZWbt16wYXXph6Kf/0p7DOOjB3LhxyCPTunbZbq0pNBO9FxPuNbyR1B9wqY2bVp1s3OOUUeOMNeO21tG3RIujeHXbbDV55pdDwqlGpieBeSWcCvSTtA/wBcG8OM6tu/frBBx+kxmRIs6etvz7svnsaAdWA0hPBacA84Angy8CtwNl5BWVmVjbdu8OUKamX8vjxadvf/56GqViypNjYqkSbiUBSHTAjIi6PiMMi4nPZuquGzKzzkOC001JCOPzwtK1Hj2JjqhJtJoKIWArMlLRxBeIxM8uXBNdd1/R+vfXglluKi6cKlFo1tA7wZDZx/cTGV1sfkrS/pJmSZkk6vZX9P5U0LXs9LWl+O+M3M2s/Cd7OukG9+iocdFBqP6hRpXYo+3Z7D5xVKU0A9gHmAA9LmhgRTzWWiYj/blb+a8Dw9p7HzKxD+vRJPZMnTYJRo9ITRUuXpqeOakxbM5T1lHQKcBiwFXB/RNzb+Grj2LsAsyJidvbo6Q3AISspPwa4vvTQzczK4JOfhOOPT+t1dal3co1pK/VdDdSTnhY6APhxO469EfBis/dzsm0fImkwsAlw9wr2nyCpQVLDvBr8I5lZziZMSMNSQOqR/HLFp2MvVFuJYGhEHBURlwKfAz6RUxyHAzdmDdMfEhGXRUR9RNQP8NghZlZuPXqkDmhf+1p6v8EGNdUTua1E8EHjSkS094HbucCgZu8HZttacziuFjKzIklw0UVpdjRI/Q/ef3/ln+ki2koE20takL3eBoY1rkta0MZnHwY2l7SJpNVIX/YfetJI0lakp5KmdOQCzMzKatKkpp7Iq68Oc+YUG08FrDQRRERdRPTNXmtGRPdm633b+OwSYCxwO2kI699HxJOSzpc0ulnRw4Eb3EHNzKqClHoiD88eYhw0CKZPLzamnKmzff/W19dHQ0ND0WGYWS343vfgzDPT+vPPw8adt1+tpKkRUd/avtp7YNbMrFRnnAEXXJDWBw+G994rNp6cOBGYma3M2WfDiSem9W23LTaWnDgRmJm15Ve/SnMkz5oFm2xSdDRl50RgZlaKhx5Ky+eeg/POKzSUcnMiMDMrRa9eTbObnXsuXN91uj45EZiZlWrddWHatLR+xBHw4IOFhlMuTgRmZu2x/fbpaSKAXXftEr2PnQjMzNrrwgvhnHPS+vrrN1UZdVJOBGZmHXHOOTBiBLz5ZkoGnXhkZCcCM7OOkFIbwbHHpvdDh8KiRcXG1EFOBGZmq+KKK+Doo+G112CXXYqOpkOcCMzMVtWVV6bl9OkwdmyxsXSAE4GZ2aqqq4MXXkjrEybAzJnFxtNOTgRmZuUwaBDcfnta32oruLvVmXerkhOBmVm57LtvU9XQXnuleQ06AScCM7Ny+sUv4Oc/T+sf/zgsW1ZsPCVwIjAzK7eTT4Z99knrdXXw7rvFxtMGJwIzszzcdlvT+iGHFBdHCZwIzMzyUFcHEbDFFnDnnfD440VHtEJOBGZmebrmmrRs7IFchZwIzMzyNGIEbLYZTJ0Kf/pT0dG0yonAzCxv996bloceCv/4R7GxtMKJwMwsbxtuCD/6UVofMQKWLi02nhacCMzMKuHUU+FTn0rrX/lKsbG04ERgZlYpN9+clpddVlV9C5wIzMwqRYILLkjrY8YUG0szTgRmZpX0rW+l5cSJ8OyzxcaScSIwM6uk1VaD005L65ttVhVjETkRmJlV2vjxcNhhab0K+hY4EZiZFeHyy2GDDVJCWLy40FCcCMzMirDWWvDlL6f1L3yh0FByTQSS9pc0U9IsSaevoMznJT0l6UlJ1+UZj5lZVRk3Li0nToQ33ywsjNwSgaQ6YAJwADAUGCNpaIsymwNnACMjYhvglLziMTOrOr17w7e/ndZ32qmwMPK8I9gFmBURsyPifeAGoOWg3McDEyLiTYCIeDXHeMzMqs/558P228O//gULFhQSQp6JYCPgxWbv52TbmtsC2ELS/ZIelLR/aweSdIKkBkkN8+bNyylcM7OCNHYyGzSokNMX3VjcHdgc2BMYA1wuae2WhSLisoioj4j6AQMGVDZCM7O8HXxwWnbBO4K5QPP0NjDb1twcYGJEfBAR/wKeJiUGM7Pa0jgQXQHDVOeZCB4GNpe0iaTVgMOBiS3K/IV0N4Ck/qSqotk5xmRmVp0anyA66qiKnzq3RBARS4CxwO3ADOD3EfGkpPMljc6K3Q68LukpYBIwLiJezysmM7OqNWRIWhbwGKkiouInXRX19fXR0NBQdBhmZuU3fjyccQY89hgMG1bWQ0uaGhH1re0rurHYzMwa7bdfWjb2OK4QJwIzs2oxfHhaTp9e0dM6EZiZVZMvfAEWLqzovMZOBGZm1WTrrdPyppsqdkonAjOzanLiiWlZwaksnQjMzKrJeuul5fvvV+yUTgRmZtXmpJPS8uWXK3I6JwIzs2qz225pee21FTmdE4GZWbVpHIRu/vyKnM6JwMys2qy5JgweDPfcU5HTORGYmVWjddeF+++HCgwD5ERgZlaNttwyLR95JPdTORGYmVWjxuGoX3xx5eXKwInAzKwaDR6clpMm5X4qJwIzs2q0xRZp+fDDuZ/KicDMrBp165Yms58yJf9T5X4GMzPrmM9+Ni3nzcv1NE4EZmbVasSItLzttlxP40RgZlatdt45LW+5JdfTOBGYmVWrzTZLyz//OdfTOBGYmdU4JwIzs2q2447Qu3eup3AiMDOrZiNHwltv5TrmkBOBmVk1W7w4LWfMyO0UTgRmZtVs773T8pVXcjuFE4GZWTX76EfTcubM3E7hRGBmVs2GDEnLadNyO4UTgZlZNevfPy3/+c/cTuFEYGZW7Q4+GO69N7fDOxGYmVW79ddPy/fey+XwuSYCSftLmilplqTTW9l/jKR5kqZlr/+XZzxmZp3SRhul5axZuRw+t0QgqQ6YABwADAXGSBraStHfRcQO2evXecVjZtZp7bRTWna2RADsAsyKiNkR8T5wA3BIjuczM+uaBg5MyzvuyOXweSaCjYDmsy7Pyba1dKikxyXdKGlQaweSdIKkBkkN83KeoMHMrOoMGwbjxsEh+fyWLrqx+GZgSEQMA+4Arm6tUERcFhH1EVE/YMCAigZoZla4bt3gBz+AffbJ5/C5HDWZCzT/hT8w2/YfEfF6RDQ2g/8a2CnHeMzMrBV5JoKHgc0lbSJpNeBwYGLzApI2aPZ2NJDfqEpmZtaq7nkdOCKWSBoL3A7UAVdGxJOSzgcaImIicLKk0cAS4A3gmLziMTOz1ilyHOM6D/X19dHQ0FB0GGZmnYqkqRFR39q+ohuLzcysYE4EZmY1zonAzKzGORGYmdW4TtdYLGke8HwHP94feK2M4XQGvuba4GuuDatyzYMjotUeuZ0uEawKSQ0rajXvqnzNtcHXXBvyumZXDZmZ1TgnAjOzGldrieCyogMogK+5Nviaa0Mu11xTbQRmZvZhtXZHYGZmLTgRmJnVuC6ZCCTtL2mmpFmSTm9l/+qSfpftf0jSkALCLKsSrvkbkp7KZoO7S9LgIuIsp7auuVm5QyWFpE7/qGEp1yzp89nf+klJ11U6xnIr4b/tjSVNkvRo9t/3gUXEWS6SrpT0qqTpK9gvSRdl/x6PS9pxlU8aEV3qRRry+lngo8BqwGPA0BZlvgJckq0fDvyu6LgrcM2fBHpn6yfVwjVn5dYE7gMeBOqLjrsCf+fNgUeBdbL36xYddwWu+TLgpGx9KPBc0XGv4jXvDuwITF/B/gOB2wABHwMeWtVzdsU7gl2AWRExOyLeB24AWk70eQhN02LeCOwlSRWMsdzavOaImBQR72ZvHyTNGNeZlfJ3BrgA+D6wuJLB5aSUaz4emBARbwJExKsVjrHcSrnmAPpm62sBL1UwvrKLiPtI87OsyCHANZE8CKzdYpKvduuKiWAj4MVm7+dk21otExFLgLeAfhWJLh+lXHNzx5F+UXRmbV5zdss8KCJuqWRgOSrl77wFsIWk+yU9KGn/ikWXj1Ku+VzgKElzgFuBr1UmtMK09//3NuU2Q5lVJ0lHAfXAHkXHkidJ3YCfUHuz3nUnVQ/tSbrru0/SdhExv8igcjYGuCoifixpV+A3kraNiGVFB9ZZdMU7grnAoGbvB2bbWi0jqTvpdvL1ikSXj1KuGUl7A2cBoyPivQrFlpe2rnlNYFvgHknPkepSJ3byBuNS/s5zgIkR8UFE/At4mpQYOqtSrvk44PcAETEF6EkanK2rKun/9/boiongYWBzSZtIWo3UGDyxRZmJwBez9c8Bd0fWCtNJtXnNkoYDl5KSQGevN4Y2rjki3oqI/hExJCKGkNpFRkdEZ57ntJT/tv9CuhtAUn9SVdHsCsZYbqVc8wvAXgCStiYlgnkVjbKyJgJHZ08PfQx4KyL+vSoH7HJVQxGxRNJY4HbSEwdXRsSTks4HGiJiInAF6fZxFqlR5vDiIl51JV7zD4E+wB+ydvEXImJ0YUGvohKvuUsp8ZpvB/aV9BSwFBgXEZ32brfEaz4VuFzSf5Majo/pzD/sJF1PSub9s3aPc4AeABFxCakd5EBgFvAu8KVVPmcn/vcyM7My6IpVQ2Zm1g5OBGZmNc6JwMysxjkRmJnVOCcCM7Ma50Rg1gpJSyVNkzRd0s2S1i7z8Z/LnvNH0sJyHtusvZwIzFq3KCJ2iIhtSX1Nvlp0QGZ5cSIwa9sUskG9JG0q6f8kTZX0d0lbZdvXk/RnSY9lr49n2/+SlX1S0gkFXoPZCnW5nsVm5SSpjjR8wRXZpsuAEyPiGUkjgIuBUcBFwL0R8ZnsM32y8sdGxBuSegEPS/pjZ+7pa12TE4FZ63pJmka6E5gB3CGpD/BxmobpAFg9W44CjgaIiKWkoc0BTpb0mWx9EGkAOCcCqypOBGatWxQRO0jqTRrn5qvAVcD8iNihlANI2hPYG9g1It6VdA9pQDSzquI2ArOVyGZ1O5k0sNm7wL8kHQb/mTt2+6zoXaQpQJFUJ2kt0vDmb2ZJYCvSUNhmVceJwKwNEfEo8DhpApQjgeMkPQY8SdO0iV8HPinpCWAqae7c/wO6S5oBjCcNhW1WdTz6qJlZjfMdgZlZjXMiMDOrcU4EZmY1zonAzKzGORGYmdU4JwIzsxrnRGBmVuP+P8t9vPxgXDt+AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SML AUPR score: 0.7979512695473273\n"
     ]
    }
   ],
   "source": [
    "precision, recall, aupr_score = aupr(test_y, ood_scores)\n",
    "\n",
    "plot_precision_recall_curve(precision, recall)\n",
    "print('SML AUPR score:', aupr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SML fpr95 score: 0.5594\n"
     ]
    }
   ],
   "source": [
    "fpr95 = fpr_at_95_tpr(test_y, ood_scores)\n",
    "print('SML fpr95 score:', fpr95)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}