ood_cifar10_vs_svhn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Base settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import time\n",
    "from pprint import pprint\n",
    "from pathlib import Path\n",
    "from collections import OrderedDict\n",
    "from tqdm import tqdm\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create ConfigDict from config yaml file\n",
    "import yaml\n",
    "from ml_collections import config_dict\n",
    "\n",
    "config_path = Path('configs') / 'deit_tiny-pretrained-cifar10.yaml'\n",
    "with config_path.open('r') as f:\n",
    "    config = yaml.safe_load(f)\n",
    "    config = config_dict.ConfigDict(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# frequently used variables\n",
    "model_name = config.model.name\n",
    "patch_size = config.model.patch_size\n",
    "summary = config.summary\n",
    "\n",
    "# log directories\n",
    "log_root = Path('./logs') / model_name / summary\n",
    "checkpoint_path = log_root / 'checkpoints'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize ViT model and load pretrained weights\n",
    "from ood_with_vit.models.vit import ViT\n",
    "\n",
    "def initialize_vit_model(config, verbose=0):\n",
    "    n_class = config.dataset.n_class\n",
    "    if config.model.pretrained:\n",
    "        model = torch.hub.load(\n",
    "            repo_or_dir=config.model.repo,\n",
    "            model=config.model.pretrained_model,\n",
    "            pretrained=False,\n",
    "        )\n",
    "        model.head = nn.Linear(model.head.in_features, n_class)\n",
    "    else:\n",
    "        model = ViT(\n",
    "            image_size=config.model.img_size,\n",
    "            patch_size=config.model.patch_size,\n",
    "            num_classes=n_class,\n",
    "            dim=config.model.dim_head,\n",
    "            depth=config.model.depth,\n",
    "            heads=config.model.n_heads,\n",
    "            mlp_dim=config.model.dim_mlp,\n",
    "            dropout=config.model.dropout,\n",
    "            emb_dropout=config.model.emb_dropout,\n",
    "            visualize=True,\n",
    "        )\n",
    "\n",
    "    model = model.to(device=device)\n",
    "    if verbose:\n",
    "        print(model)\n",
    "\n",
    "    checkpoint = torch.load(checkpoint_path / f'{summary}_best.pt')\n",
    "\n",
    "    state_dict = checkpoint['model_state_dict']\n",
    "    trimmed_keys = []\n",
    "    for key in state_dict.keys():\n",
    "        # remove prefix 'module.' for each key (in case of DataParallel)\n",
    "        trimmed_keys.append(key[7:])\n",
    "    trimmed_state_dict = OrderedDict(list(zip(trimmed_keys, state_dict.values())))\n",
    "\n",
    "    model.load_state_dict(trimmed_state_dict)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'airplane': 0, 'automobile': 1, 'bird': 2, 'cat': 3, 'deer': 4, 'dog': 5, 'frog': 6, 'horse': 7, 'ship': 8, 'truck': 9}\n",
      "{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}\n"
     ]
    }
   ],
   "source": [
    "from torchvision.datasets import CIFAR10, SVHN\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "\n",
    "dataset_mean, dataset_std = config.dataset.mean, config.dataset.std\n",
    "dataset_root = config.dataset.root\n",
    "img_size = config.model.img_size\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.Resize(img_size),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(dataset_mean, dataset_std),\n",
    "])      \n",
    "\n",
    "id_dataset = CIFAR10(\n",
    "    root=dataset_root, \n",
    "    train=False, \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "id_test_dataloader = DataLoader(\n",
    "    dataset=id_dataset, \n",
    "    batch_size=config.eval.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "id_dataset_name = 'cifar10'\n",
    "\n",
    "ood_dataset = SVHN(\n",
    "    root=dataset_root, \n",
    "    split='test',\n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "ood_test_dataloader = DataLoader(\n",
    "    dataset=ood_dataset, \n",
    "    batch_size=config.eval.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "ood_dataset_name = 'svhn'\n",
    "\n",
    "print(id_dataset.class_to_idx)\n",
    "print(set(ood_dataset.labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Utility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set seeds\n",
    "import random\n",
    "import numpy as np\n",
    "\n",
    "def set_seed(seed):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "seed = 1234"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.visualizer.feature_extractor import FeatureExtractor\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "# add hooks for feature extraction\n",
    "feature_extractor = FeatureExtractor(\n",
    "    model=model,\n",
    "    layer_name=config.model.layer_name.penultimate,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [00:00<00:00, 89.70it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 102.88it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 105.23it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 105.53it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 104.29it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 106.03it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 105.97it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 105.79it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 105.26it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 105.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [00:00<00:00, 105.08it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 105.82it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 106.23it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 106.23it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 106.10it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 105.81it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 106.31it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 105.87it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 106.48it/s]\n",
      "100%|██████████| 10/10 [00:00<00:00, 106.30it/s]\n"
     ]
    }
   ],
   "source": [
    "# gather penultimate features\n",
    "from ood_with_vit.utils import compute_penultimate_features\n",
    "\n",
    "set_seed(seed)\n",
    "num_samples = 10\n",
    "num_class = 10\n",
    "\n",
    "print('processing in-distribution samples...')\n",
    "X_in, y_in = [], []\n",
    "in_img_paths = []\n",
    "cls_indices = random.sample(range(len(id_dataset.classes)), num_class)\n",
    "for cls_idx in cls_indices:\n",
    "    img_indices = np.where(np.array(id_dataset.targets) == cls_idx)[0]\n",
    "    img_indices = random.sample(list(img_indices), num_samples)\n",
    "    for i in tqdm(img_indices):\n",
    "        img, _ = id_dataset[i]\n",
    "        img = img.to(device)\n",
    "        penultimate_features = compute_penultimate_features(\n",
    "            config=config, \n",
    "            model=model, \n",
    "            imgs=img.unsqueeze(0),\n",
    "            feature_extractor=feature_extractor,    \n",
    "        )\n",
    "        X_in.append(penultimate_features.squeeze().numpy())\n",
    "        y_in.append((cls_idx, id_dataset.classes[cls_idx]))\n",
    "        img_path = f'./assets/{id_dataset_name}/test/{i:05d}.jpg'\n",
    "        # img_path = f'/assets/{id_dataset_name}/test/{i:05d}.jpg'\n",
    "        in_img_paths.append(img_path)\n",
    "        \n",
    "print('processing out-of-distribution samples...')\n",
    "X_out, y_out = [], []\n",
    "out_img_paths = []\n",
    "cls_indices = random.sample(range(len(set(ood_dataset.labels))), num_class)\n",
    "for cls_idx in cls_indices:\n",
    "    img_indices = np.where(np.array(ood_dataset.labels) == cls_idx)[0]\n",
    "    img_indices = random.sample(list(img_indices), num_samples)\n",
    "    for i in tqdm(img_indices):\n",
    "        img, _ = ood_dataset[i]\n",
    "        img = img.to(device)\n",
    "        penultimate_features = compute_penultimate_features(\n",
    "            config=config, \n",
    "            model=model, \n",
    "            imgs=img.unsqueeze(0),\n",
    "            feature_extractor=feature_extractor,    \n",
    "        )\n",
    "        X_out.append(penultimate_features.squeeze().numpy())\n",
    "        y_out.append(str(cls_idx))\n",
    "        img_path = f'./assets/{ood_dataset_name}/test/{i:05d}.jpg'\n",
    "        # img_path = f'/assets/{ood_dataset_name}/test/{i:05d}.jpg'\n",
    "        out_img_paths.append(img_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature visualization by T-SNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ipywidgets import HTML, VBox\n",
    "\n",
    "def interactive_plot(partitioned_info, fig, img_width=240, event=\"click\") :\n",
    "    \"\"\"\n",
    "    Make a plot react on hover or click of a data point and update a HTML preview below it.    \n",
    "    \"\"\"\n",
    "    html = HTML(\"\")\n",
    "    fig = go.FigureWidget(data=fig.data, layout=fig.layout)\n",
    "\n",
    "    if event == \"hover\" :\n",
    "        for data in fig.data:\n",
    "            def update(trace, points, state):\n",
    "                if len(points.point_inds) == 0:\n",
    "                    return\n",
    "                trace_idx, point_idx = points.trace_index, points.point_inds[0]\n",
    "                df_row = partitioned_info[trace_idx].iloc[point_idx]\n",
    "                img_url = df_row['img']\n",
    "                label = df_row['label']\n",
    "                html.value = f'\\\n",
    "                    <body>\\\n",
    "                        <img src=\"{img_url}\" width=\"{img_width}\"><br>\\\n",
    "                        <b>Image: {img_url}</b><br>\\\n",
    "                        <b>Label: {label}</b>\\\n",
    "                    </body>'\n",
    "            data.on_hover(update)\n",
    "    else :\n",
    "        for data in fig.data:\n",
    "            def update(trace, points, state):\n",
    "                if len(points.point_inds) == 0:\n",
    "                    return\n",
    "                trace_idx, point_idx = points.trace_index, points.point_inds[0]\n",
    "                df_row = partitioned_info[trace_idx].iloc[point_idx]\n",
    "                img_url = df_row['img']\n",
    "                label = df_row['label']\n",
    "                html.value = f'\\\n",
    "                    <body>\\\n",
    "                        <img src=\"{img_url}\" width=\"{img_width}\"><br>\\\n",
    "                        <b>Image: {img_url}</b><br>\\\n",
    "                        <b>Label: {label}</b>\\\n",
    "                    </body>'\n",
    "            data.on_click(update)\n",
    "\n",
    "    return VBox([fig, html])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:780: FutureWarning:\n",
      "\n",
      "The default initialization in TSNE will change from 'random' to 'pca' in 1.2.\n",
      "\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:790: FutureWarning:\n",
      "\n",
      "The default learning rate in TSNE will change from 200.0 to 'auto' in 1.2.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cbc2ed20c52243659b1ffc0615ec9689",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/cifar10/test/00198.jpg'],\n",
       "        …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "\n",
    "tsne = TSNE(n_components=2, random_state=seed)\n",
    "X_tsne = tsne.fit_transform(X_in)\n",
    "\n",
    "df_embed = pd.DataFrame(X_tsne)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_in)\n",
    "df_embed = df_embed.assign(img=in_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_in:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "    \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained, CIFAR10 finetuned ViT CIFAR100 T-SNE',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:780: FutureWarning:\n",
      "\n",
      "The default initialization in TSNE will change from 'random' to 'pca' in 1.2.\n",
      "\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:790: FutureWarning:\n",
      "\n",
      "The default learning rate in TSNE will change from 200.0 to 'auto' in 1.2.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "070b4ace0dcb4496bc33435ad6df89f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/svhn/test/19935.jpg'],\n",
       "           …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tsne = TSNE(n_components=2, random_state=seed)\n",
    "X_tsne = tsne.fit_transform(X_out)\n",
    "\n",
    "df_embed = pd.DataFrame(X_tsne)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_out)\n",
    "df_embed = df_embed.assign(img=out_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_out:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "\n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained, CIFAR10 finetuned ViT SVHN T-SNE',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, 64, event='click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:780: FutureWarning:\n",
      "\n",
      "The default initialization in TSNE will change from 'random' to 'pca' in 1.2.\n",
      "\n",
      "/usr/local/lib/python3.9/site-packages/sklearn/manifold/_t_sne.py:790: FutureWarning:\n",
      "\n",
      "The default learning rate in TSNE will change from 200.0 to 'auto' in 1.2.\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d12ed8b9a152461a8f55dfc6f959b89d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/cifar10/test/00198.jpg'],\n",
       "        …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_all = X_in + X_out\n",
    "y_all = y_in + y_out\n",
    "img_all = in_img_paths + out_img_paths\n",
    "tsne = TSNE(n_components=2, random_state=seed)\n",
    "X_tsne = tsne.fit_transform(X_all)\n",
    "\n",
    "df_embed = pd.DataFrame(X_tsne)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_all)\n",
    "df_embed = df_embed.assign(img=img_all)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_all:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "    \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained ViT CIFAR10, SVHN T-SNE',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature visualization by PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b67ead3f55ec408c817c597b11def91f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'hovertemplate': \"class=(7, 'horse')<br>x=%{x}<br>y=%{y}<extra></e…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "\n",
    "pca = PCA()\n",
    "components = pca.fit_transform(X_in)\n",
    "\n",
    "df_embed = pd.DataFrame(components)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_in)\n",
    "df_embed = df_embed.assign(img=in_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_in:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "        \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained, CIFAR10 finetuned ViT CIFAR10 PCA',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "183fddcb2a1f40dc9f3bb2f444ffe4d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'hovertemplate': 'class=7<br>x=%{x}<br>y=%{y}<extra></extra>',\n",
       "   …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pca = PCA()\n",
    "components = pca.fit_transform(X_out)\n",
    "\n",
    "df_embed = pd.DataFrame(components)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_out)\n",
    "df_embed = df_embed.assign(img=out_img_paths)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_out:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "        \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained, CIFAR10 finetuned ViT CIFAR100 PCA',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5de7b870b08d4183b88a9a0e62a0dac7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(FigureWidget({\n",
       "    'data': [{'customdata': array([['./assets/cifar10/test/00198.jpg'],\n",
       "        …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_all = X_in + X_out\n",
    "y_all = y_in + y_out\n",
    "img_all = in_img_paths + out_img_paths\n",
    "pca = PCA()\n",
    "components = pca.fit_transform(X_all)\n",
    "\n",
    "df_embed = pd.DataFrame(components)\n",
    "df_embed = df_embed.rename(columns={0: 'x', 1: 'y'})\n",
    "df_embed = df_embed.assign(label=y_all)\n",
    "df_embed = df_embed.assign(img=img_all)\n",
    "\n",
    "partitioned_info, processed_labels = [], []\n",
    "for label in y_all:\n",
    "    if label not in processed_labels:\n",
    "        processed_labels.append(label)\n",
    "        partitioned_info.append(df_embed.where(df_embed['label'] == label).dropna())\n",
    "    \n",
    "fig = px.scatter(\n",
    "    df_embed,\n",
    "    x='x',\n",
    "    y='y',\n",
    "    color='label',\n",
    "    labels={'label': 'class'},\n",
    "    hover_data=['img'],\n",
    "    title='ImageNet pretrained ViT CIFAR10, SVHN PCA',\n",
    ")\n",
    "interactive_plot(partitioned_info, fig, event='click')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OOD detection baselines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ood_with_vit.utils import compute_ood_scores\n",
    "from ood_with_vit.utils.ood_metrics import auroc, aupr, fpr_at_95_tpr\n",
    "from ood_with_vit.utils.visualization import plot_roc_curve, plot_precision_recall_curve"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### OOD detection by MSP (Maximum over Softmax Probability)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.metrics.msp import MSP\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "msp = MSP(config, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 15.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 261/261 [00:14<00:00, 18.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id mean score: -0.9825615\n",
      "ood mean score: -0.5879053\n"
     ]
    }
   ],
   "source": [
    "test_y, ood_scores, id_ood_scores, ood_ood_scores = compute_ood_scores(\n",
    "    metric=msp,\n",
    "    in_dist_dataloader=id_test_dataloader,\n",
    "    out_of_dist_dataloader=ood_test_dataloader,    \n",
    ")\n",
    "print('id mean score:', np.mean(id_ood_scores))\n",
    "print('ood mean score:', np.mean(ood_ood_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA77UlEQVR4nO3dd3gU5fbA8e9J6NKLjRY6hCJowII0UYoC6s+GBa/3RhEQwWu7KFauYsUOCCpiQfSKDQVFr4p6LfQOIoh0EAgdQklyfn+8E11iygYyO9ns+TzPPrO7MztzZnZ3zrzvzLyvqCrGGGNiV1zQARhjjAmWJQJjjIlxlgiMMSbGWSIwxpgYZ4nAGGNinCUCY4yJcZYIijgRWSIiHYOOo7AQkbtF5OWAlj1eRB4KYtkFTUSuFpHPj/Kz9pssZCwRRJCIrBaRVBHZKyKbvR1DWT+XqapNVXW6n8vIJCIlReQREVnrrecKEblDRCQSy88mno4isj70PVUdrqrX+7Q8EZFBIrJYRPaJyHoReVdEmvuxvKMlIg+IyJvHMg9VnaCqXcJY1l+S39H+JkWkhBf7Cm/7rhaRcSKSkN95mSNZIoi8nqpaFmgJtALuCjac/BORYjmMehfoDJwPlAP6AH2BZ32IQUSksP1+nwUGA4OAykBD4EPggoJeUC7fge8CXPYkoBdwFVABOAWYg/vN5UuQ269QUlV7ROgBrAbODXn9ODAl5PUZwA/ATmAB0DFkXGXgVWAjsAP4MGRcD2C+97kfgBZZlwmcDKQClUPGtQK2AcW91/8AlnnznwbUDplWgZuAFcBv2axbZ+AAUDPL+6cD6UB97/V04BFgJrAb+ChLTLltg+nAw8D33rrUB/7uxbwHWAXc6E17nDdNBrDXe5wMPAC86U2T4K3X34C13rYYGrK80sBr3vZYBtwJrM/hu23grWebXL7/8cBIYIoX7wygXsj4Z4F13naZA7QLGfcAbkf4pjf+eqAN8KO3rTYBLwAlQj7TFPgC2A78DtwNdAMOAYe9bbLAm7YC8Io3nw3AQ0C8N+46b5s/DaR4464D/ueNF2/cFi+2RUAz3EHAYW95e4GPs/4PgHgvrl+9bTKHLL8hb7pzve/zL+Ny+X9l910ne9/1t8CnwMAs81gA/J/3vHHI9lsOXB70PsS3fVPQAcTSI8sfoIb3h3nWe13d+5Odjyupnee9ruaNnwK8A1QCigMdvPdbeX/A070/1d+85ZTMZplfATeExPME8KL3/EJgJdAEKAbcA/wQMq16f4rKQOls1u1R4Jsc1nsNf+6gp3s7mma4nfV7IX/WvLbBdO9P3NSLsTjuaLsebmfUAdgPnOpN35EsO+4cdg4v4Xb6pwAHgSah6+Rt8xrAwqzzC5lvP2BNHt//eG992njxTwDeDhl/DVDFG3cbsBkoFRL3YeAib9uUBk7DJc5i3rosA27xpi+H26nfBpTyXp+edRuELPsDYIz3nRyPS9SZ39l1QBpws7es0hyZCLriduAVve+hCXBSyDo/lMv/4A7c/6CR99lTgCr5+X1lN99cvuvXvXUsDVwLfB8yfSIuqZb0plmHO9Aoxp8HTYlB70f8eBS2onUs+FBE9uB+ZFuA+733rwGmqupUVc1Q1S+A2cD5InIS0B3op6o7VPWwqn7jfa4vMEZVZ6hquqq+htuZnZHNst8CrgRXtQL09t4DtyN7RFWXqWoaMBxoKSK1Qz7/iKpuV9XUbOZdFbfjyc4mb3ymN1R1saruA+4FLheR+Ny2Qchnx6vqElVN87bDFFX9VZ1vgM+BdjnEkZMHVTVVVRfgjghP8d6/HBjubfP1wHO5zKNKLusf6gNVnelt4wm4KkIAVPVNVU3x1m0EbofUKOSzP6rqh962SVXVOar6kzf9atyOvIM3bQ9gs6qOUNUDqrpHVWdkF5CInIDbxreo6j5V3YI7wu8dMtlGVX3eW1bW7/8wLtE0BsT7DYWzLcCVbO5R1eXed7hAVVOymS7c7ZuXB7x1TMUlv9Df+NXA+6p6ELf9Vqvqq946z8MdtFxWADEUOpYIIu8iVS2HO1ptzJ87yNrAZSKyM/MBnA2cBNQEtqvqjmzmVxu4LcvnauKqQbJ6DzjTSyztcdUm34XM59mQeWzHHaFVD/n8ulzWa5sXa3ZO8sZnN581uCP7quS+DbKNQUS6i8hPIrLdm/58jkw64dgc8nw/kHkC/+Qsy8tt/VPIef3DWRYicruILBORXd66VODIdcm67g1F5BPvwoPduOSdOX1NXHVLOGrjvoNNIdt9DK5kkO2yQ6nqV7hqqZHAFhEZKyLlw1x2uHGGu33z8sd6qOoeXEk7M+FdiUvO4LbJ6Vl+i1cDJxZADIWOJYKAeEev44EnvbfW4Y6UK4Y8jlPVR71xlUWkYjazWgc8nOVzZVR1YjbL3IE7Yr4Cd8LtbVXVkPncmGU+pVX1h9BZ5LJK/8X9cWqGvikip+P+7F+FvB06TS3cEeW2PLbBX2IQkZK45PYkcIKqVgSm4hJYXvGGYxOuSii7uLP6EqghIklHsyARaYc7B3E5UMlbl138uS7w1/UZDfwMNFDV8ri69szp1wF1c1hc1vmsw5Uiq4Zs9/Kq2jSXzxw5Q9XnVPU0XPVKQ1yVT56f85ZdL49pwP2+2ohIjVym2QeUCXmd3U47azwTgStF5ExcFdrXIXF9k+W3WFZV+4cRa9SxRBCsZ4DzROQU3EnAniLSVUTiRaSUd/ljDa+Y/SkwSkQqiUhxEWnvzeMloJ+InO5dSXOciFwgIuVyWOZbuLrRS/mzWgjgReAuEWkKICIVRCTsYrCq/he3M3xPRJp663CGt16jVXVFyOTXiEiiiJQBhgGTVDU9t22Qw2JL4KpPtgJpItIdCL2k8XegiohUCHc9svgPbptUEpHqwMCcJvTWbxQw0Yu5hBd/bxEZEsayyuHq4bcCxUTkPiCvo+pyuJOze0WkMRC6k/oEOElEbhF3WW85LymD2y4JmVddeb+vz4ERIlJeROJEpJ6IdCAMItLa+/0Vx+2MD+BKm5nLyikhAbwM/FtEGni/3xYiUiXrRN7v6wvgAxE5TUSKeevUT0T+4U02H+jt/T+ScL/xvEzFHf0PA95R1cy4PwEaikgfb37FvfVsEsY8o44lggCp6lbcyav7VHUd7oTt3bidwTrcUVXmd9QHd+T8M+7cwi3ePGYDN+CK5jtwJ3yvy2Wxk3FXuGz26sQzY/kAeAx426tmWIw7L5Efl+COqD7DXSXyJu5KlJuzTPcGrjS0GXcUNsiLIa9tcASvaD8It8PegSvlTA4Z/zPuiG+VV7zPrrosN8OA9cBvuCPSSbgj55wM4s8qkp24Ko+LgY/DWNY03Hb7BVdddoDcq6IAbset8x7cAcE7mSO8bXMe0BO3nVcAnbzR73rDFBGZ6z2/FpdYl+K25STCr4op7y1/hxd7Cu5CBHDff6K3/T/M5rNP4b6/z3FJ7RXcidzsXIrbcb+DKy0tBpJw3w248031vDge5MgDnWx55wPex12V9FbI+3twBxW9cVfqbcb9P0rmNc9oJH/WDBjjPxGZjruSI5C7e4+FiPQHeqtqWEfKxkQLKxEYkwMROUlE2npVJY1wl2J+EHRcxhQ0u7vOmJyVwF09UwdX1fM27jyAMUWKVQ0ZY0yMs6ohY4yJcVFXNVS1alVNSEgIOgxjjIkqc+bM2aaq1bIbF3WJICEhgdmzZwcdhjHGRBURWZPTOKsaMsaYGGeJwBhjYpwlAmOMiXGWCIwxJsZZIjDGmBjnWyIQ16n0FhFZnMN4EZHnRGSliCwUkVP9isUYY0zO/CwRjMf1j5qT7rhWMBvgetka7WMsxhhjcuDbfQSq+q2IJOQyyYXA617HKD+JSEUROSkfXdwZv6Snw969cOAA7NkD+/ZBRoZ7P6dhejocOgSqRz4yMv76Xrjvb9sGZcpAfHz2cebVPEpu44P6rJ/zjrXP+jnvQvbZfRxiK/tJ6NkHWrfOff5HIcgbyqpzZHvr6733/pIIRKQvrtRArVq1IhJcoaUKO3fCxo2wciWkpUFqqttxL1nixqeluZ3yr79CuXJu/PLlUK2a22GnpR05XLMGypd3zw8edO8bY4IhcsTLrxKUG3pChYMwe3Zj4opYIgibqo4FxgIkJSUV7Vbydu6EuXPdjnvlSndE/uWXEBfnXoejWjUoWRJKlICUFGjaFBIS4PBhqF7dHWEXK+aGmc+3bIGGDd3nihd3yaNOHShdGsqWPXL6uLjshyVKuB9x6CMuLrz3cpu2ePGc1zXLnyZf44P6rJ/zjrXP+jlvPz8bhp0HdnLH53fw8ryXqV+5Pk/3fJm4BH+6wggyEWzgyD5ga3jvxY5Dh2DRIvjpJ3j/fVi9Glat+ut0rVu7H1afPlCqFFSqBM2bw0knuSP+SpXcDvv4493O2BgT1dIz0jnrlbNYnrKcO8+6kwc6PkDp4jl13HbsgkwEk4GBIvI2cDqwKybODyxfDi+84I7yf/3VJQNwVTMJCXDjjdCqldvR163rju5zqiM3xhQpKftTqFy6MvFx8Tx8zsPUrFCTpJOTfF+ub4lARCYCHYGqIrIeuB8oDqCqL+L6Hj0f18fufuDvfsUSuDVr4Jln4L33YJ13WqRCBejSBa68Ek47zVXLFEBx0hgTfVSVCYsmMPizwTza+VFuOO0GLm5yccSW7+dVQ1fmMV6Bm/xafuBUYcIEeOwxWLrUXQ0DcNVV8K9/QYsWwcZnjCkU1u1aR78p/Zi6Yipn1DiDtrXaRjyGqDhZHHU2bIBLL3V1/6VLw//9HzzwgDtpa4wxnomLJnLjJzeSruk80/UZBrYZSHxc5KuCLREUpIwMeOopuOceV/f/+ONw223uyhdjjMmiUulKnF7jdMb2GEudSnUCi8MSQUHZtctV+0ydCmecAS+/bCUAY8wR0jLSePrHpzmUfoih7YfSrX43utbrigR8ftASQUFYuNBd6ZORAT16wEcfWSnAGHOEBZsXkDw5mTmb5nB508tRVUQk8CQA1vrosduxA6691j2fPBk+/tiSgDHmDwfTDnLvV/eS9FIS63av493L3uXtS94uFAkgk5UIjsXq1dCmDWzdCm++CT17Bh2RMaaQWbF9BY99/xhXNb+Kp7o8RZUyVYIO6S8sERyt9HRXEti6FT78EC68MOiIjDGFxN5De/no54+4usXVNDu+GT8P/Jm6leoGHVaOrA7jaN1wA3z3HQwZYknAGPOHL379guajm9Pngz4s27oMoFAnAbBEcHQ++QRefRUuuggeeSToaIwxhcCO1B0kf5RMlze7UCK+BN9c9w1NqjUJOqywWNVQfqWmQt++7oTwaOtLxxjjGolrO64tv6T8wl1n38V9He6jVLFSQYcVNksE+XX//bBpE7z7Lpx4YtDRGGMCtG3/tj8aiRveeTi1KtTi1JOir9ddqxrKj8WL4YknoEkTuOSSoKMxxgREVXl9wes0fL4hL899GYCLGl8UlUkArESQP//4hxuOG2cthRoTo9bsXMONn9zItF+ncVbNs2hfu33QIR0zSwTh+vJLmDULOnVyTUgYY2LOmwvfpP+U/qgqz3d/ngGtBxAn0V+xYokgXGPGuOHEicHGYYwJTLUy1Whbsy1jeoyhdsXaQYdTYCwRhCMtDb75Bs49F044IehojDERcjj9MCN+HMHh9MPc2+FeutbvSpd6XQpV8xAFwRJBON5+23XufsUVQUdijImQeZvmkTw5mXmb59G7We9C1UhcQYv+yq1IeP11N7z00mDjMMb47kDaAe7+8m5av9SajXs28t7l7zHxkolFMgFkshJBXnbtctVCbdpAxYpBR2OM8dnK7St58ocnufaUaxnRZQSVSlcKOiTfWSLIyxtvuN7G7rsv6EiMMT7Ze2gvHyz7gD6n9KHZ8c1YPnB5oD2GRZpVDeVl1Cg37No12DiMMb6YtnIaTUc15W8f/u2PRuJiKQmAJYK8LVsG1apBMSs8GVOUpOxP4W8f/o1uE7pRpngZvvv7d1HTSFxBs71bblJS3LB99N85aIz5U2YjcSu3r2Rou6Hc0/6eqGokrqBZIsjNW2+5YXJysHEYYwrE1n1bqVKmCvFx8Tx27mPUrliblie2DDqswFnVUG5mzHDDjh0DDcMYc2xUlVfnvUrDFxry0pyXALiw8YWWBDxWIsiJquuApmlTKF066GiMMUdp9c7V9P24L1+s+oJ2tdrRqU6noEMqdCwR5GTZMncPQf/+QUdijDlKbyx4g/5T+iMijDp/FDcm3VgkGokraJYIcjJ2rBv26RNsHMaYo3ZC2RNoX7s9L/Z4kVoVagUdTqElqhp0DPmSlJSks2fP9n9BTZvCzp2wYYP/yzLGFIjD6Yd5/PvHSdd07utgN4GGEpE5qpqU3TgrI2VnxgxYutSqhYyJInM3zaX1S6255+t7WJ6ynGg7yA2SJYLs3HabG152WbBxGGPylHo4lSH/HUKbl9rw+77f+eCKD5jwfxOKdCNxBc3XRCAi3URkuYisFJEh2YyvJSJfi8g8EVkoIuf7GU9YMjJg7lx3N3GjRkFHY4zJw6odq3jqx6e4ruV1LB2wlIsaXxR0SFHHt0QgIvHASKA7kAhcKSKJWSa7B/iPqrYCegOj/IonbEuXQmoq3Hhj0JEYY3Kw++Buxs8fD0DT45uy4uYVvNzr5ZhoKdQPfpYI2gArVXWVqh4C3gYuzDKNAuW95xWAjT7GE55333XDCy4INg5jTLamrphKs1HNSJ6c/EcjcUWp28gg+JkIqgPrQl6v994L9QBwjYisB6YCN2c3IxHpKyKzRWT21q1b/Yj1TwsXuuHpp/u7HGNMvmzbv40+H/ThgrcuoFzJcnz/j+9jtpG4ghb0yeIrgfGqWgM4H3hD5K93e6jqWFVNUtWkatWq+RvRwoWupVE70WRMoZHZSNzbi9/mvvb3MbfvXM6ocUbQYRUZft5QtgGoGfK6hvdeqGSgG4Cq/igipYCqwBYf48rdqlVwzjmBLd4Y86ff9/5OteOqER8Xz5PnPUntirVpcUKLoMMqcvwsEcwCGohIHREpgTsZPDnLNGuBzgAi0gQoBfhc95OL/fvdMCEhsBCMMa6RuFfmvkKjFxoxdo67y79no56WBHziW4lAVdNEZCAwDYgHxqnqEhEZBsxW1cnAbcBLIvJP3Inj6zTIu0B++80NGzcOLARjYt2qHau44eMb+Oq3r+hQuwPn1j036JCKPF/bGlLVqbiTwKHv3RfyfCnQ1s8Y8uWbb9ywTZtg4zAmRr02/zUGTB1AvMTz4gUvcsNpN1gjcRFgjc6FmjvXDS0RGBOIk8udzDl1zmH0BaOpUb5G0OHEDGt0LpSIu5v455/9mb8x5giH0g/x6P8eJUMzeKDjA0GHU6RZo3PhSE11w0p2Z6IxkTBrwyxOG3sa90+/n1U7VlkjcQGyRJAp80Tx9dcHG4cxRdz+w/u5/fPbOeOVM9iRuoPJvSfz+sWvWyNxAbJzBJlmznTDOnWCjcOYIu63Hb/x/MznueHUG3js3MeoUKpC0CHFPEsEmX791Q2bNQs2DmOKoF0HdvH+svf5e6u/0/T4pqy8eSU1K9TM+4MmIqxqKNPSpRAXB8cfH3QkxhQpU36ZQtNRTbn+4+v5eZu7EMOSQOFiiSDT+vVQokTQURhTZGzdt5Wr37+aHhN7UKl0JX5M/pHGVe1mzcLIqoYyrVwJNe0oxZiCkJ6Rztmvns1vO37jwY4PMuTsIZSItwOtwsoSQabt2+HUU4OOwpiotnnvZo4/7nji4+IZ0WUECRUTaHa8nXcr7MKuGhKRMn4GEqjMewisa0pjjkqGZjBm9hgaPt+QMbPHANCjYQ9LAlEiz0QgImeJyFLgZ+/1KSISfJeSBWnFCjds3TrYOIyJQiu3r6Tz653pN6Ufrau3pmv9rkGHZPIpnKqhp4GueE1Iq+oCEWnva1SRtn69G9aqFWwcxkSZV+e9yoCpAygRX4KXer5EcqtkuzEsCoV1jkBV12X5ctP9CScg8+e7YW3r99SY/KhVoRZd63Vl5PkjqV4+a0+0JlqEkwjWichZgIpIcWAwsMzfsCIs82Yyu2rImFwdTDvII/97hAzNYFinYXSu25nOdTsHHZY5RuGcLO4H3ITreH4D0BIY4GNMkbfV6xStePFg4zCmEJuxfganjT2NB795kLW71lojcUVIOCWCRqp6degbItIW+N6fkAKwYQMkJgYdhTGF0r5D+7j363t55qdnqF6+Op9c+QkXNLwg6LBMAQqnRPB8mO9FL7uZzJgcrdm1hlGzRtEvqR9LBiyxJFAE5VgiEJEzgbOAaiJya8io8rg+iIuGtDTYu9euGDImxM4DO5m0dBLXn3o9idUSWTlopfUYVoTlVjVUAijrTVMu5P3dwKV+BhVR69dDRoa1OmqM56OfP6L/lP5s2beFs2udTeOqjS0JFHE5JgJV/Qb4RkTGq+qaCMYUWZndUlawNtFNbNuybwuDPh3EO0veocUJLZh85WRrJC5GhHOyeL+IPAE0BUplvqmq5/gWVSRlJoL2ReseOWPyIz0jnbbj2rJ211oe6vQQd7a9k+LxdhVdrAgnEUwA3gF64C4l/Ruw1c+gIuqnn9wwISHQMIwJwsY9Gzmx7InEx8XzbLdnSaiYQGI1u4Iu1oRz1VAVVX0FOKyq36jqP4CiURoAOHQIypQBuy3exJAMzWD0rNE0fqExL85+EYDzG5xvSSBGhVMiOOwNN4nIBcBGoLJ/IUXY1q3Qpk3QURgTMb+k/MINH9/At2u+5dy659K9fvegQzIBCycRPCQiFYDbcPcPlAdu8TOoiNq+3aqFTMx4Ze4rDPx0IKWKlWJcr3Fc1/I6ayTO5J0IVPUT7+kuoBP8cWdx0bBund1MZmJGQsUEutfvzsjzR3JSuZOCDscUErndUBYPXI5rY+gzVV0sIj2Au4HSQKvIhOgjVdizB6pUCToSY3xxMO0g//723wA8dM5D1kicyVZuJYJXgJrATOA5EdkIJAFDVPXDCMTmvwMH3PD444ONwxgf/LDuB5InJ/Pztp/5R8t/oKpWDWSylVsiSAJaqGqGiJQCNgP1VDUlMqFFQGaro/XrBxuHMQVo76G9DP1yKM/PfJ6aFWry2dWfWa9hJle5XT56SFUzAFT1ALAqv0lARLqJyHIRWSkiQ3KY5nIRWSoiS0TkrfzM/5gtX+6GJUtGdLHG+GntrrWMmTOGm1rfxOL+iy0JmDzlViJoLCILvecC1PNeC6Cq2iK3GXvnGEYC5wHrgVkiMllVl4ZM0wC4C2irqjtEJLJ1NKtWuaH1VWyi3I7UHby79F36ntaXxGqJrBq8ipPLnRx0WCZK5JYImhzjvNsAK1V1FYCIvA1cCCwNmeYGYKSq7gBQ1S3HuMz82bPHDe1ksYliHyz7gAFTB7B131Y61O5Ao6qNLAmYfMmt0bljbWiuOrAu5PV64PQs0zQEEJHvcU1bP6Cqn2WdkYj0BfoC1CrI5qKXejmpWrWCm6cxEbJ572Zu/vRmJi2dRMsTWzLlqik0qtoo6LBMFAqr83qfl98A6AjUAL4VkeaqujN0IlUdC4wFSEpKKrj+8TZscEM7R2CiTHpGOu1ebce6XesYfs5wbj/rdmskzhw1PxPBBtzlp5lqeO+FWg/MUNXDwG8i8gsuMczyMa4/paTAyVaENtFj/e71nFzuZOLj4nmu23PUqVTHmoo2xyycRucQkdIikt8y5yyggYjUEZESQG9gcpZpPsSVBhCRqriqolX5XM7RW7ECGjaM2OKMOVoZmsHzM56n8QuNGT1rNADdG3S3JGAKRJ6JQER6AvOBz7zXLUUk6w79L1Q1DRgITAOWAf9R1SUiMkxEenmTTQNSRGQp8DVwR0TvUzh8GEqUiNjijDkaP2/7mfavtmfQZ4M4u9bZ9GjYI+iQTBETTtXQA7grgKYDqOp8EakTzsxVdSowNct794U8V+BW7xFZhw9DairUrRvxRRsTrpfnvszAqQMpU7wMr130Gn1a9LG7g02BC6sZalXdleXHV3AnbIOSecVQ9erBxmFMLupVqkfPRj15ofsLnFD2hKDDMUVUOIlgiYhcBcR7N4ANAn7wN6wIWLTIDU87Ldg4jAlxIO0Aw74ZBsDwzsPpVKcTnep0CjgqU9SFc7L4Zlx/xQeBt3DNUd/iY0yRscW7d63Jsd43Z0zB+H7t97R8sSWP/O8Rtu7biqs5NcZ/4ZQIGqvqUGCo38FE1DffuGGNGsHGYWLenoN7uPvLuxk5ayS1K9Zm2jXT6FKvS9BhmRgSTiIYISInApOAd1R1sc8xRcamTVChAhQL+p46E+vW717Py/Ne5uY2N/Nw54cpW6Js0CGZGJNn1ZCqdsL1TLYVGCMii0TkHt8j89uyZVAnrIufjClwKftT/rgfoEm1JqwatIpnuz9rScAEIqwbylR1s6o+B/TD3VNwX+6fiBJWLWQiTFWZtHQSiaMSGfTZIJZvc02hW7eRJkjh3FDWREQeEJFFuM7rf8A1FxG9duyAvXuhTZugIzExZNOeTVzyn0u47N3LqFm+JrNvmG2NxJlCIZwK8nHAO0BXVd3oczyRsXq1G1oXlSZCMhuJ27BnA4+f+zj/PPOfFIuz81OmcMjzl6iqZ0YikIiaP98NrYtK47N1u9ZRvXx14uPiGXn+SOpUqkPDKta+lSlccqwaEpH/eMNFIrIw5LEopOey6LRzpxtaIjA+Sc9I57kZz9F45J+NxHWt39WSgCmUcisRDPaGRa+Fq8zmJawJauODZVuXkTw5mR/X/0j3+t3p2ahn0CEZk6scSwSqusl7OkBV14Q+gAGRCc8nK1ZAXBwUt448TMEaO2csLce05JeUX3jj4jeYctUUalUowF71jPFBOJePnpfNe90LOpCIWrQImjULOgpTBDWo3ICLG1/M0puWck2La6ylUBMVcqwaEpH+uCP/ulnOCZQDvvc7MF9t3w7t2wcdhSkCUg+n8sD0BxARHj33UWskzkSl3M4RvAV8CjwCDAl5f4+qbvc1Kj/t3u2Gtay4bo7Nt2u+5frJ17Ni+wr6ndYPVbUSgIlKuVUNqaquBm4C9oQ8EJHK/ofmk7173bBp02DjMFFr98HdDJgygA7jO5Cu6Xx57ZeM7jHakoCJWnmVCHoAc3Ad0YT+yhWIzq691q51w7LWpos5Ohv3bGT8/PHcesatDOs0jONKHBd0SMYckxwTgar28IZFq2W2efPcsEGDYOMwUWXb/m38Z8l/GNB6AI2rNua3wb9Zj2GmyAinraG2InKc9/waEXlKRKK3gn3HDjesVy/YOExUUFXeWfwOiSMTueWzW/gl5RcASwKmSAnn8tHRwH4ROQW4DfgVeMPXqPx06JAbVqwYaBim8Nu4ZyMXvXMRvd/rTe2KtZnTd47dGWyKpHBavUpTVRWRC4EXVPUVEUn2OzDfLPb61YkLqwVuE6PSM9Jp/2p7NuzZwJPnPcngMwZbI3GmyArnl71HRO4C+gDtRCQOiN5bcjduhJIlg47CFFJrdq6hRvkaxMfFM+qCUdStVJf6la1NKlO0hXNYfAWu4/p/qOpmXF8ET/galZ82brR7CMxfpGek89SPT9FkZBNGz3aNxHWp18WSgIkJ4XRVuRmYAFQQkR7AAVV93ffI/FK6NJQrF3QUphBZvGUxZ407i9s+v43OdTtzUeOLgg7JmIgK56qhy4GZwGXA5cAMEbnU78B8s2MHnHJK0FGYQuLF2S9y6phTWbVjFW/931tM7j2ZGuWjuwM+Y/IrnHMEQ4HWqroFQESqAf8FJvkZmC9SU+H3361qyPzRHESTqk24rOllPNP1GaodVy3osIwJRDiJIC4zCXhSCLPT+0Jn2zY3LGZXf8Sq/Yf3c9/X9xEv8Tx23mN0SOhAh4QOQYdlTKDC2aF/JiLTROQ6EbkOmAJM9Tcsn6SmumGdonWztAnP9NXTaTG6BSN+HMHeQ3tR1aBDMqZQCKfP4jtE5P+As723xqrqB/6G5ZPNm93QdgAxZdeBXdz5xZ2MnTuWepXq8dW1X1lT0caEyK0/ggbAk0A9YBFwu6puiFRgvlizxg1r1w42DhNRm/Zu4s1Fb3L7mbfzYKcHKVO8TNAhGVOo5FY1NA74BLgE1wLp8/mduYh0E5HlIrJSRIbkMt0lIqIikpTfZeTLxo1uWLOmr4sxwdu6byvPz3A/2cZVG7N68Gqe6PKEJQFjspFb1VA5VX3Je75cRObmZ8YiEg+MxHV1uR6YJSKTVXVplunKAYOBGfmZ/1FJS3PDqlV9X5QJhqoycfFEBn06iN0Hd9O1flcaVmloVwQZk4vcSgSlRKSViJwqIqcCpbO8zksbYKWqrlLVQ8DbwIXZTPdv4DHgQL6jz68lS9zQ+iIoktbtWkfPiT25+v2rqV+5PvNunGeNxBkThtxKBJuAp0Jebw55rcA5ecy7OrAu5PV64PTQCbyEUlNVp4jIHTnNSET6An0Bah3LPQDlyx/9Z02hlpaRRsfXOrJ572ae7vo0N7e5mfi4+KDDMiYq5NYxja+XVXiN1z0FXJfXtKo6FhgLkJSUdPSX/Bw+DDXsrtGiZPXO1dQsX5NiccUY02MMdSvVpW6l6Ow8z5ig+Hlj2AYg9KxsDe+9TOWAZsB0EVkNnAFM9vWE8aFDUDx6G041f0rLSOPJH56kycgmjJo1CoBz655rScCYo+DnLbazgAYiUgeXAHoDV2WOVNVdwB9nbUVkOu4S1dm+RbRsmd1VXAQs/H0hyZOTmb1xNhc2upBLEi8JOiRjoppve0VVTRORgcA0IB4Yp6pLRGQYMFtVJ/u17BylpsL+/RFfrCk4o2aNYvBng6lUqhLvXPoOlyVehogEHZYxUS3PRCDuX3Y1UFdVh3n9FZ+oqjPz+qyqTiVLcxSqel8O03YMK+JjoQoVKvi+GFPwMhuJa3Z8M3o3683TXZ+mahm7DNiYghBOiWAUkIG7SmgYsAd4D2jtY1z+EIEmTYKOwuTDvkP7uOereygWV4wnujxB+9rtaV+7fdBhGVOkhHOy+HRVvQnvOn9V3QGU8DUqv6SnQ7xdUhgtvlz1Jc1HN+eZGc9wMP2gNRJnjE/CKREc9u4SVvijP4IMX6PyS0aGJYIosPPATm7//HZemfcKDSo34NvrvqVd7XZBh2VMkRVOieA54APgeBF5GPgfMNzXqPySng5x0dmVQiz5fe/vvL34bf7V9l8s6LfAkoAxPgunGeoJIjIH6AwIcJGqLvM9Mj9Y1VChlbnzH3zGYBpVbcTqW1bbyWBjIiScq4ZqAfuBj0PfU9W1fgbmC0sEhY6qMmHRBAZ/Npi9h/ZyfoPzaVClgSUBYyIonHMEU3DnBwQoBdQBlgNNfYzLHxkZVjVUiKzdtZZ+n/Tj05WfcmaNM3ml1ys0qNIg6LCMiTnhVA01D33tNRQ3wLeI/LRxo5UICom0jDQ6ju/Iln1beK7bcwxoPcAaiTMmIPm+s1hV54rI6XlPWQgVLw5btwYdRUxbtWMVtSvUplhcMV7q+RL1KtcjoWJC0GEZE9PCOUdwa8jLOOBUYKNvEfmtUaOgI4hJaRlpjPhhBPdPv5/Hz3ucQacPonPdzkGHZYwhvBJBuZDnabhzBu/5E46PVF0z1CVLBh1JzJm/eT7Jk5OZu2kuFze+mMsSLws6JGNMiFwTgXcjWTlVvT1C8fjngNcBmp0jiKgXZr7AP6f9kyqlqzDpsknWUqgxhVCOiUBEinktiLaNZEC+SU11wwP+94hp/mwkrsUJLbi6+dU81fUpKpeuHHRYxphs5FYimIk7HzBfRCYD7wL7Mkeq6vs+x1aw0tPd8KSTgo2jiNt7aC9DvxxK8fjiPNnlSWskzpgoEM5F9aWAFFzroz2Ant4wumR4zSNZ1ZBvPv/1c5qNasbzM5/ncPphayTOmCiRW4ngeO+KocX8eUNZpuj7h2cmAruhrMDtSN3BrZ/fyvj542lUpRHf/v1bzq51dtBhGWPClFsiiAfKcmQCyBR9iSCzasgSQYHbsm8Lk5ZO4q6z7+K+DvdRqlipoEMyxuRDbolgk6oOi1gkfrOqoQK1ee9mJi6ayD/P/KdrJG7waqqUqRJ0WMaYo5Db4XHR6gjWSgQFQlV5bf5rJI5M5K4v72JFygoASwLGRLHc9opF67ZPO0dwzFbvXE23Cd247qPrSKyWyPx+862ROGOKgByrhlR1eyQD8Z1VDR2TtIw0Or3WiW37tzHy/JH0S+pHnFhSNaYoyHejc1FrwwY3TEsLNo4os3L7SupUrEOxuGKM6zWOupXqUrti7aDDMsYUoNg5pMssEdSsGWwcUeJw+mGGfzecpqOaMnLWSAA61elkScCYIih2SgSZJQFrdC5PczfNJXlyMvM3z+eyxMu4oukVQYdkjPFR7CSCzKuGisXOKh+N52Y8x63TbqXacdV4//L3ubjJxUGHZIzxWezsFTNLBJYIspXZSFyrE1tx7SnXMqLLCCqVrhR0WMaYCIidvWJmIrCrho6w5+Ae7vryLkrGl2RE1xG0q92OdrXbBR2WMSaCYudk8e+/u6GVCP7w2crPaDa6GaNmjUJRayTOmBgVO3vFUl77N1YiIGV/Crd+fiuvL3idJlWb8P0/vufMmmcGHZYxJiCxkwjEazGjlDWIlpKawgfLPuDe9vcytN1QShazK6mMiWW+Vg2JSDcRWS4iK0VkSDbjbxWRpSKyUES+FBH/LlKP8WqPTXs28eQPT6KqNKzSkDW3rGFYp2GWBIwx/iUCr7/jkUB3IBG4UkQSs0w2D0hS1RbAJOBxv+IJCcz3RRQmqsq4eeNoMrIJ9359Lyu3rwSwK4KMMX/ws0TQBlipqqtU9RDwNnBh6ASq+rWq7vde/gTU8C2aGCwR/LbjN7q82YXkycmccuIpLOi3wBqJM8b8hZ/nCKoD60JerwdOz2X6ZODT7EaISF+gL0CtWrWOLaoYKRGkZaRxzuvnkLI/hdEXjKbvaX2tkThjTLYKxcliEbkGSAI6ZDdeVccCYwGSkpKO7tA+RkoEK1JWULdSXYrFFePVC1+lXqV61Kxg7SsZY3Lm5yHiBiB0D1TDe+8IInIuMBTopaoHfYwnc4G+LyIIh9MP89C3D9FsdDNemPkCAB0TOloSMMbkyc8SwSyggYjUwSWA3sBVoROISCtgDNBNVbf4GEuRLhHM3jib5MnJLPx9Ib2b9ebK5lcGHZIxJor4lghUNU1EBgLTgHhgnKouEZFhwGxVnQw8AZQF3hV3pL5WVXv5FRNQ5EoEz/70LLd+fisnlj2Rj3p/RK9G/m4+Y0zR4+s5AlWdCkzN8t59Ic/P9XP5WYKJ2KIiIbORuKSTk0hulczj5z1OxVIVgw7LGBOFCsXJ4ojITARRXiLYfXA3//riX5QqVoqnuz1N21ptaVurbdBhGWOiWOxdTxjFiWDqiqk0HdWUsXPHUiyumDUSZ4wpELFXIohC2/Zv45bPbmHCogk0rdaUSZdN4vQaud2SYYwx4YudRJApCksEO1J38PEvH3N/h/u5u93dlIgvEXRIxpgiJHYSQZSVCDbs3sCERRO446w7aFClAWtuWWMng40xvrBzBIWMqvLSnJdIHJXIA9Mf4NcdvwJYEjDG+CZ2EkEUlAh+3f4rnV/vTN9P+nLqSaeysP9C6leuH3RYxpgiLnaqhjIV0hJBWkYanV/vzPbU7YzpMYbrT73eGokzxkRE7CSCQloiWL5tOfUq16NYXDFeu+g16lWuR43y/rXGbYwxWcXeIWchKREcSj/Eg9MfpPno5oycORKADgkdLAkYYyLOSgQBmLlhJsmTk1m8ZTFXNb+Kq1tcHXRIxpgYFjuJIFPAJYJnfnqG2z6/jZPKnsTHV35Mj4Y9Ao3HGGNiJxEEXCLIbCSuTfU23HDqDTx27mNUKFUh0JiMMQZiKRFkinCJYNeBXdz5xZ2ULl6aZ7o9w1k1z+KsmmdFNAZjjMlN7JwsDqBE8PHyj0kclcjL816mZHxJayTOGFMoWYnAB1v3bWXwZ4OZuHgizY9vzodXfEjr6q19X64xxhyN2EkEETwa33VwF1NXTOXBjg8y5Owh1kicMaZQi51EkMmnEsG6Xet4c+GbDDl7CPUr12fNLWvsZLAxJirYOYJjlKEZvDj7RZqOaspD3z30RyNxlgSMMdEidhJBpgIsEaxIWcE5r51D/yn9aVO9DYv6L7JG4owxUSd2qoYKuESQlpHGeW+cx84DO3ml1yv8veXfkULSfIUxxuRH7CSCTMe4s162dRkNqjSgWFwx3rj4DepVrsfJ5U4uoOCMMX46fPgw69ev58CBA0GH4ptSpUpRo0YNihcvHvZnYicRHGOJ4GDaQYZ/N5zh/xvOE+c9wS1n3EK72u0KKDhjTCSsX7+ecuXKkZCQUCRL8KpKSkoK69evp06dOmF/LnYSQaaj+PJ/Wv8TyZOTWbp1KX1a9KFPiz4+BGaM8duBAweKbBIAEBGqVKnC1q1b8/W52EkER1kiGPHDCO744g5qlK/B1Kum0r1B9wIOzBgTSUU1CWQ6mvWLnUSQKcyNlKEZxEkcZ9Y8k35J/Xj03EcpX7K8z8EZY0zkxc7lo2GWCHYe2EnyR8kM/nQwAGfVPItRF4yyJGCMKRDx8fG0bNmSZs2a0bNnT3bu3PnHuCVLlnDOOefQqFEjGjRowL///e8j2ij79NNPSUpKIjExkVatWnHbbbcVSEyxkwgy5VIi+PDnD0kcmchrC16jXMly1kicMabAlS5dmvnz57N48WIqV67MyJGuh8LU1FR69erFkCFDWL58OQsWLOCHH35g1KhRACxevJiBAwfy5ptvsnTpUmbPnk39+gVz31LsVA3lslPfsm8LA6cO5N2l79LyxJZ8ctUnnHrSqREMzhgTcbfcAvPnF+w8W7aEZ54Je/IzzzyThQsXAvDWW2/Rtm1bunTpAkCZMmV44YUX6NixIzfddBOPP/44Q4cOpXHjxoArWfTv379AwrYSAbD74G6+WPUFD5/zMDOvn2lJwBjju/T0dL788kt69eoFuGqh00477Yhp6tWrx969e9m9ezeLFy/+y/iCErMlgrW71vLGgje4u93d1K9cn7W3rKVcyXIBBWeMibh8HLkXpNTUVFq2bMmGDRto0qQJ5513XiBxhPK1RCAi3URkuYisFJEh2YwvKSLveONniEiCn/EAZKCMmjWKpqOaMvx/w/9oJM6SgDEmEjLPEaxZswZV/eMcQWJiInPmzDli2lWrVlG2bFnKly9P06ZN/zK+oPiWCEQkHhgJdAcSgStFJDHLZMnADlWtDzwNPOZXPKiyvAp0fKc7N029iTNrnMmSAUuskThjTCDKlCnDc889x4gRI0hLS+Pqq6/mf//7H//9738BV3IYNGgQd955JwB33HEHw4cP55dffgEgIyODF198sUBi8bNE0AZYqaqrVPUQ8DZwYZZpLgRe855PAjqLT3d7pGk6XfvAom1LefXCV5l2zTQSKib4sShjjAlLq1ataNGiBRMnTqR06dJ89NFHPPTQQzRq1IjmzZvTunVrBg4cCECLFi145plnuPLKK2nSpAnNmjVj1apVBRKHn+cIqgPrQl6vB07PaRpVTRORXUAVYFvoRCLSF+gLUKtWraMKpljjRN5c2ol6Q1/hpKrht8FhjDEFae/evUe8/vjjj/943rx5c6ZPn57jZ3v06EGPHj0KPKaouGpIVceqapKqJlWrVu3oZtKrF2eP/8qSgDHGZOFnItgA1Ax5XcN7L9tpRKQYUAFI8TEmY4wxWfiZCGYBDUSkjoiUAHoDk7NMMxn4m/f8UuArtdt5jTE+Kuq7mKNZP98SgaqmAQOBacAy4D+qukREholIL2+yV4AqIrISuBX4yyWmxhhTUEqVKkVKSkqRTQaZ/RGUKlUqX5+TaNsgSUlJOnv27KDDMMZEoVjuoUxE5qhqUnafiZ07i40xMa948eL56rkrVkTFVUPGGGP8Y4nAGGNinCUCY4yJcVF3slhEtgJrjvLjVcly13IMsHWODbbOseFY1rm2qmZ7R27UJYJjISKzczprXlTZOscGW+fY4Nc6W9WQMcbEOEsExhgT42ItEYwNOoAA2DrHBlvn2ODLOsfUOQJjjDF/FWslAmOMMVlYIjDGmBhXJBOBiHQTkeUislJE/tKiqYiUFJF3vPEzRCQhgDALVBjrfKuILBWRhSLypYjUDiLOgpTXOodMd4mIqIhE/aWG4ayziFzufddLROStSMdY0ML4bdcSka9FZJ73+z4/iDgLioiME5EtIrI4h/EiIs9522OhiJx6zAtV1SL1AOKBX4G6QAlgAZCYZZoBwIve897AO0HHHYF17gSU8Z73j4V19qYrB3wL/AQkBR13BL7nBsA8oJL3+vig447AOo8F+nvPE4HVQcd9jOvcHjgVWJzD+POBTwEBzgBmHOsyi2KJoA2wUlVXqeoh4G3gwizTXAi85j2fBHQWEYlgjAUtz3VW1a9Vdb/38idcj3HRLJzvGeDfwGNAUWh3OJx1vgEYqao7AFR1S4RjLGjhrLMC5b3nFYCNEYyvwKnqt8D2XCa5EHhdnZ+AiiJy0rEssygmgurAupDX6733sp1GXQc6u4AqEYnOH+Gsc6hk3BFFNMtznb0ic01VnRLJwHwUzvfcEGgoIt+LyE8i0i1i0fkjnHV+ALhGRNYDU4GbIxNaYPL7f8+T9UcQY0TkGiAJ6BB0LH4SkTjgKeC6gEOJtGK46qGOuFLftyLSXFV3BhmUz64ExqvqCBE5E3hDRJqpakbQgUWLolgi2ADUDHldw3sv22lEpBiuOJkSkej8Ec46IyLnAkOBXqp6MEKx+SWvdS4HNAOmi8hqXF3q5Cg/YRzO97wemKyqh1X1N+AXXGKIVuGsczLwHwBV/REohWucragK6/+eH0UxEcwCGohIHREpgTsZPDnLNJOBv3nPLwW+Uu8sTJTKc51FpBUwBpcEor3eGPJYZ1XdpapVVTVBVRNw50V6qWo093Mazm/7Q1xpABGpiqsqWhXBGAtaOOu8FugMICJNcIlga0SjjKzJwLXe1UNnALtUddOxzLDIVQ2papqIDASm4a44GKeqS0RkGDBbVScDr+CKjytxJ2V6BxfxsQtznZ8AygLveufF16pqr8CCPkZhrnOREuY6TwO6iMhSIB24Q1WjtrQb5jrfBrwkIv/EnTi+LpoP7ERkIi6ZV/XOe9wPFAdQ1Rdx50HOB1YC+4G/H/Myo3h7GWOMKQBFsWrIGGNMPlgiMMaYGGeJwBhjYpwlAmOMiXGWCIwxJsZZIjCFkoiki8j8kEdCLtPuLYDljReR37xlzfXuUM3vPF4WkUTv+d1Zxv1wrDF688ncLotF5GMRqZjH9C2jvTVO4z+7fNQUSiKyV1XLFvS0ucxjPPCJqk4SkS7Ak6ra4hjmd8wx5TVfEXkN+EVVH85l+utwra4OLOhYTNFhJQITFUSkrNePwlwRWSQif2lpVEROEpFvQ46Y23nvdxGRH73Pvisiee2gvwXqe5+91ZvXYhG5xXvvOBGZIiILvPev8N6fLiJJIvIoUNqLY4I3bq83fFtELgiJebyIXCoi8SLyhIjM8tqYvzGMzfIjXmNjItLGW8d5IvKDiDTy7sQdBlzhxXKFF/s4EZnpTZtdi60m1gTd9rY97JHdA3dX7Hzv8QHuLvjy3riquLsqM0u0e73hbcBQ73k8rr2hqrgd+3He+/8C7stmeeOBS73nlwEzgNOARcBxuLuylwCtgEuAl0I+W8EbTsfr8yAzppBpMmO8GHjNe14C14pkaaAvcI/3fklgNlAnmzj3hqzfu0A373V5oJj3/FzgPe/5dcALIZ8fDlzjPa+Ia4vouKC/b3sE+yhyTUyYIiNVVVtmvhCR4sBwEWkPZOCOhE8ANod8ZhYwzpv2Q1WdLyIdcJ2VfO81rVECdySdnSdE5B5cOzXJuPZrPlDVfV4M7wPtgM+AESLyGK466bt8rNenwLMiUhLoBnyrqqledVQLEbnUm64CrrG437J8vrSIzPfWfxnwRcj0r4lIA1wzC8VzWH4XoJeI3O69LgXU8uZlYpQlAhMtrgaqAaep6mFxLYqWCp1AVb/1EsUFwHgReQrYAXyhqleGsYw7VHVS5gsR6ZzdRKr6i7i+Ds4HHhKRL1V1WDgroaoHRGQ60BW4AtfRCrjepm5W1Wl5zCJVVVuKSBlc+zs3Ac/hOuD5WlUv9k6sT8/h8wJcoqrLw4nXxAY7R2CiRQVgi5cEOgF/6XNZXD/Mv6vqS8DLuO7+fgLaikhmnf9xItIwzGV+B1wkImVE5Dhctc53InIysF9V38Q15pddn7GHvZJJdt7BNRSWWboAt1Pvn/kZEWnoLTNb6nqbGwTcJn82pZ7ZFPF1IZPuwVWRZZoG3Cxe8Uhcq7QmxlkiMNFiApAkIouAa4Gfs5mmI7BARObhjrafVdWtuB3jRBFZiKsWahzOAlV1Lu7cwUzcOYOXVXUe0ByY6VXR3A88lM3HxwILM08WZ/E5rmOg/6rrfhFc4loKzBXXafkY8iixe7EsxHXM8jjwiLfuoZ/7GkjMPFmMKzkU92Jb4r02Mc4uHzXGmBhnJQJjjIlxlgiMMSbGWSIwxpgYZ4nAGGNinCUCY4yJcZYIjDEmxlkiMMaYGPf/A/ia2uiWSr0AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSP AUROC score: 0.97953201\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, auroc_score = auroc(test_y, ood_scores)\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "print('MSP AUROC score:', auroc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAiM0lEQVR4nO3de7hU1XnH8e9PONwExABRuQkolktU1BOvTTDGCxrFqNVoo0ZrtVoksU18Yqq5VG2SpjHJk0diJNVqjJeYmFoSUZOgqdGgBRQRJCri7eANQUA83Hn7x9oThuMBBph9ZubM7/M88+yZPWv2fheXeWettfdaigjMzKx+7VTpAMzMrLKcCMzM6pwTgZlZnXMiMDOrc04EZmZ1zonAzKzOORFYzZD0WUm/LaHcjyV9tS1iaguSXpZ0dPb8G5J+VumYrH1xIrCyyL6sVkpaIektSbdI6l7Oc0TE7RFxbAnlLo6Ia8p57gJJIen9rJ4LJX1PUoc8zrU9JPWU9ANJr2Yxvpi97lPp2Kx6ORFYOZ0UEd2BA4FG4KqWBSR1bPOoym//rJ5jgM8Af1fheACQ1AmYCowCxgI9gcOAxcDB23G89vB3ZSVwIrCyi4iFwP3AR+Avv6LHS3oBeCHbd6KkWZKWSvqTpP0Kn5c0UNKvJC2StFjS9dn+8yQ9mj2XpO9LelvScknPSCqc7xZJ1xYd70JJ8yUtkTRZUr+i90LSxZJeyGKZKEkl1nM+8Bgwuuh421OvvSQ9lO17R9Ltknpt4x87wLnAIOCUiHg2IjZExNsRcU1ETCmq795FMf3lz0rSkZKaJH1Z0pvAf0maJ+nEovIds/gPzF4fmtVzqaSnJR25HXFbhTkRWNlJGgicADxVtPvTwCHASEkHADcD/wD0Bm4EJkvqnHWz/AZ4BRgM9AfuauU0xwIfB/YBdgHOIP3ybRnLUcC3svf3yI7b8ngnAh8F9svKHVdiPYcDHwPmZ6+3t17KYuwHjAAGAt8oJYYWjgYeiIgV2/HZgt2BDwF7AhcBdwJnFb1/HPBORDwpqT9wH3Bt9pkvAfdI6rsD57cKcCKwcrpX0lLgUeB/gW8WvfetiFgSEStJXzA3RsQTEbE+Im4FVgOHkrow+gGXR8T7EbEqIh5t5VxrgR7AcEARMS8i3mil3GeBmyPiyYhYDXwFOEzS4KIy346IpRHxKvAwRb/wN+NJSe8D84A/AD/K9m9XvSJifkT8LiJWR8Qi4Hukbqdt1Rto7c9gW2wAvp7FshK4AxgnqVv2/t+SkgPA2cCUiJiStT5+B8wg/QiwGuJEYOX06YjoFRF7RsQ/Zl8kBa8VPd8T+GLWnbA0Sx4DSV+UA4FXImLdlk4UEQ8B1wMTgbclTZLUs5Wi/Ui/wgufW0FqOfQvKvNm0fNmoDuApLnZgOsKSR8rKnNgVuYzpFbOzjtSL0m7SborG3xeDvwM2J7B3cWkVs+OWBQRqwovsu6vecBJWTIYR0oOkOp7eov6/nUZYrA25kRgbaV4mtvXgH/Lkkbh0S0i7szeG1TKQGVE/DAiDgJGkrqILm+l2OukLywAJO1M+uW8sITjj4qI7tnjjy3ei4i4G5gGfG0H6/VN0p/PvhHRk/RLu6RxihZ+DxyX1XFzmoFuRa93b/F+a9MRF7qHTgaezZIDpDrd1qK+O0fEt7cjdqsgJwKrhJ8AF0s6JBv03VnSpyT1AP6P1L3x7Wx/F0lHtDyApI9mn28A3gdWkbo1WroTOF/SaEmdSV+6T0TEy2Wqy7eBCyXtvgP16gGsAJZl/e6tJbRS3Eb6cr5H0nBJO0nqLelfJBW6a2YBfyupg6SxlNYFdRdpTOYSNrYGILVcTpJ0XHa8LtmA84DtjN8qxInA2lxEzAAuJHXtvEsabD0ve289cBKwN/Aq0ETqgmmpJ+mL911S189i4D9aOdfvga8C95C+iPcCzixjXZ4BHiH1/W9vvf6V1N20jDT4+qvtjGU1acD4z8DvgOWkBNQHeCIr9oUsjqWk8ZN7SzjuG6SWz+HAz4v2v0ZqJfwLsIiUhC7H3ys1R16Yxsysvjlzm5nVOScCM7M650RgZlbnnAjMzOpczU0q1adPnxg8eHClwzAzqykzZ858JyJanf6j5hLB4MGDmTFjRqXDMDOrKZJe2dx77hoyM6tzTgRmZnXOicDMrM7V3BiBmdn2Wrt2LU1NTaxatWrrhWtUly5dGDBgAA0NDSV/xonAzOpGU1MTPXr0YPDgwZS4EF1NiQgWL15MU1MTQ4YMKflzuXUNSbpZaRnBOZt5X5J+qLSE4OzC0ndmZnlZtWoVvXv3bpdJAEASvXv33uYWT55jBLeQFtDenOOBYdnjIuCGHGMxMwNot0mgYHvql1vXUEQ80mI5wJZOBn4aafrTxyX1krTHZpYb3HGPPgq//W0uh94hZ58NvXrBhz+848fasAHWr4cOHWAnXwdgZqWp5BhBfzZdvrAp2/eBRCDpIlKrgUGDBm3f2aZNg2uv3b7P5qEw/fc116TtwIHQu3f6Ii9+rFtX+r5iBxywMTGsX7/p88LrNWvg7bdh9Gg49ti0f+lSOPzwdLzCedatgx494O//vi3/hMzapQ4dOrDvvvuybt06hgwZwm233UavXr0AmDt3LhMmTGDhwoVs2LCBc889l6uuuuovv/Lvv/9+vvrVr9Lc3Eznzp056qijuO6663Y4ppoYLI6IScAkgMbGxu1bQOHyy9OjWqxdC//93/DiizB5MnzoQ+mXfPGjY8dt3zd9evribmhIrYLici1fP/tsSgSzZsG8ebB6dYrtpptaj/m734Xu3VPsW3q8/34qv9tuMGxYqtu++8KAAWm7ejUMGgR77pniLFZIUGvWQLduqW5m7UjXrl2ZNWsWAJ/73OeYOHEiV155JStXrmTcuHHccMMNHHvssTQ3N3Paaafxox/9iPHjxzNnzhwuvfRS7rvvPoYPH8769euZNGlSWWKq5P+yhaQFvQsGUMI6su1GQwOccUZ6/pWvVDaWggh45ZWUMArJpWPHlKy+9CXo0iXFvbXH88+nrri33koPSMmuNT16pPOsWZMSxPr1rZfr1w+am2HZshTnxz8OBx+cksXQoelza9em4w0fDkOGpOdOJFbFDjvsMGbPng3AHXfcwRFHHMGxxx4LQLdu3bj++us58sgjGT9+PN/5zne48sorGT58OJBaFpdccklZ4qjk/5LJwKWS7gIOAZblNj5gpZGgtQn9eveGRx7Z/uOuWgV//jO89hp07pxaH08+mVoOffpAp05pf6dOG5+/9FL6kn/xRdhll5SEOnSAG29Mx3zkkdJiOvTQ1BIZMyYlhqOPTsfr3n3762Ptw2WXpdZwOY0eDT/4QUlF169fz9SpU7nggguA1C100EEHbVJmr732YsWKFSxfvpw5c+bwxS9+sbzxZnJLBJLuBI4E+khqAr4ONABExI+BKcAJpHVdm4Hz84rFKqxLl/QfZPTo9Dr7xbNdfvzjtG1uhhUr4N13UwJrbk4tihkzoKkpJZAnnoDHH0/lf/nLDx5r111h1CgYOxZOOSUlpd69U8Ixy8nKlSsZPXo0CxcuZMSIERxzzDGVDinXq4bO2sr7AYzP6/zWznXrlh4tr7Y65JBNX2/YAMuXw8qVcM89KWlMm5a6r+bOTV1Yjz4KV1216ef22gtGjEgtip12Sl1NQ4fCRz+ajmG1r8Rf7uVWGCNobm7muOOOY+LEiXz+859n5MiRPNKilbtgwQK6d+9Oz549GTVqFDNnzmT//fcve0w1t3h9Y2NjeBpqK5s1a+AXv0hXSz3/PLzwQurGeuut1Mpozc47w/nnw8c+lrqYVq2Cgw5Kg99W1ebNm8eIESMqGkP37t1ZsWIFAE899RSf/vSnefHFF1m7di2jRo1i0qRJHH300axcuZLTTz+d4447jgkTJjB79mxOPfVUpkyZwj777MOGDRuYNGkSF1988QfO0Vo9Jc2MiMbWYvJImtW3Tp3gs59t/b3m5vQl//bb8OabqUUxZQosWADXX58eLQ0alLrArrgitSbcerAtOOCAA9hvv/248847Oeecc/if//kfJkyYwPjx41m/fj3nnHMOl156KQD77bcfP/jBDzjrrLNobm5GEieeeGJZ4nCLwGxbrV0LM2embqcOHdLg9913w6JFaYyipQED0vjD2LHpKqajjoIjj3SSqIBqaBG0BbcIzPLW0JB+7Rcccgicd97G17//fbpctnv3NP7wxz+mAezCFSrFNzbusUf6/Ec+Avvsk5JF31ZXEzTLjROBWbkdfXR6FIuAd95JN/FNmgRdu6YbCt94A+69Nz0KCjcX7r03nHVWui/isMPS2AS4JWFl50Rg1hak9Et/zJj0APjP/0wJYtWq1GL4+c9h/nyYMycNWk+blh4tde2aWhujRqV5qgYMSFc5deuW7gNZvTqd66Mf9aWwrYiIdj3x3PZ09zsRmFWSlL7Yhw374CWs69bB00+nlkNTU7qM9U9/SgPczzwDf/jD1o/fu3e6kW758pQYVq5M90/ssUe6oe+NNzZO/dGvXxq/KMcEiFWqS5cuLF68uN1ORV1Yj6BLly7b9DkPFpvVuiVLUqJ45ZX0Jb9oEUydmq5uWrgQXn01tRJWrkzlO3bcWG5Lhg5NSaJ//zSJ4ZAh6a7sESPS4HfnzjXXTVXPK5RtabDYicCsnkWkeyZeeSXdfT1rVrqf4tVXSz9Gv35w4IEbWxsjRqRWzoABaWxj0KDUirGK8lVDZtY6CXbfPT1a3pVd8Oab8PLLqStp+vT0esmSdKnshz6UXv/mN1s/V7duKVG8+SaMHJkGwffeOyWKj30s7auxFkZ74URgZltWSBQAn/zklsu+916aIfb111OimD07jU8UZpW9++6NYxzPPPPBz3/iE7D//mmson//NMYxaJATRM7cNWRmlbFsWZpl9vnn013af/zj1j+z225pOo/GxnRD32WXpWRhW+UxAjOrDRFp5tjZs9MVTc88k1oQ06enS2Efe6z1z/XsCRddBBMmpBaEfYATgZm1D8Xrct9+OzzwANxxx6ZlRo5MCyl96lPt+lLYbeVEYGbt3/33w+mnp0RRuDy0Z8+0/sU++8DFF6e1weuUE4GZ1Y+1a1Nr4fxW1rrq3j1dqTRqFJx4Ihx3XJq6ow4ub3UiMLP6tXp1Goy+7bZ0p3ZrevZM9z6cdBJ85jPp6qV2Nj2HE4GZWbHXX4fvfCfdKT1vXlqcqKXDD4dzzkldS0OHtn2MZeZEYGZWiqefhnPPTVctFevSBf7mb9IYw/HHpxvgasyWEsFObR2MmVnV2n//lAwi0qDzT38KJ5yQBp9/9jP41rfg4x9PrYXbb09XMbUDTgRmZq3ZaafUNXTffSkxrF4NTz6Z3ps2Dc4+O02Z8etfVzbOMnAiMDMrRadOaRbWiDSr6xlnpOkzxo1LYw3XXJOmDq9BTgRmZtuqX7+0kNDcuXDqqSkhfO1raRnT889PrYca4kRgZra9Ro6Ee+6BNWvSetMAt9ySBpdvvbWioW0LJwIzsx3V0JDubF69Gs48M+0777x0P8KCBRUNrRROBGZm5dKpE9x5ZxpDgLSc6F57VX3rwInAzKzc+vVL3UXf/356fd556Qa2KuVEYGaWh4aGtF5CoXXw5S9Dr17w6KOVjKpVTgRmZnnq1w8WLUr3HSxblu5KXrq00lFtwonAzCxvffqkSe8KXUW77prGD6qEE4GZWVu57LKUBCBdUbRkSUXDKXAiMDNrS0uWwFVXpee9e1fFfEVOBGZmbe2aa2D06PT8xBMrGgo4EZiZVcb06Wnxm/vvhy98oaKhOBGYmVVCx46wcmW6zPSHP4Q5cyoWihOBmVmlNDSkKa0BTjmlYmE4EZiZVdJBB8Huu8P8+dDUVJEQck0EksZKek7SfElXtPL+npKmSpot6Q+SBuQZj5lZVbr55rS97LKKnD63RCCpAzAROB4YCZwlaWSLYt8FfhoR+wFXA9/KKx4zs6p1/PFpW5jSuo3l2SI4GJgfEQsiYg1wF3ByizIjgYey5w+38r6ZWX24+uq0veaaNj91nomgP/Ba0eumbF+xp4FTs+enAD0k9W55IEkXSZohacaiRYtyCdbMrKIK3ULXXtvmN5lVerD4S8AYSU8BY4CFwPqWhSJiUkQ0RkRj37592zpGM7P89egBRx2Vnt9xR5ueOs9EsBAYWPR6QLbvLyLi9Yg4NSIOAK7M9i3NMSYzs+p1991pe845bXraPBPBdGCYpCGSOgFnApOLC0jqI6kQw1eAm3OMx8ysuvXundY7Bli3rs1Om1siiIh1wKXAg8A84O6ImCvpaknjsmJHAs9Jeh7YDfi3vOIxM6sJ3/xm2k6evOVyZaSIaLOTlUNjY2PMmDGj0mGYmeXjrbfSDWZ77w0vvFC2w0qaGRGNrb1X6cFiMzMrtttuaTt/fpud0onAzKzajBmTtmVsEWyJE4GZWbUp3Fx23XVtcjonAjOzanPYYWnbRvcTOBGYmVWbhgYYOhTee69NTudEYGZWjU44IW3ffTf3UzkRmJlVo6FD0/a117ZcrgycCMzMqtGhh6bt1Km5n8qJwMysGjVm935997u5n8qJwMysGjU0pO3OO+d+KicCM7NqNW4cSLmfxonAzKxa7bknPP987jOROhGYmVWr7t3T9o03cj2NE4GZWbUaPDhtFy7cYrEd5URgZlathg9P2z/9KdfTOBGYmVWr0aPT9rHHcj2NE4GZWbXq2TNt+/fP9TROBGZm1axvX2huzvUUTgRmZtWsc2d44IFcT9Ex16ObmdmOkXK/qcwtAjOzajZmTO4zkDoRmJlVs8KA8fr1uZ3CicDMrJp17Zq2OS5Q40RgZlbN9t47bXOcb8iJwMysmhWmo16zJrdTOBGYmVWzTp3Sdu3a3E7hRGBmVs06dEjbV17J7RROBGZm1Wz33dP2rbdyO0VJN5RJOgL4BrBn9hkBERFDc4vMzMw2JoKd8vvdXuqdxTcB/wTMBPK7mNXMzDbVBmMEpSaCZRFxf25RmJlZ6wpXDb33Xm6nKDURPCzpP4BfAasLOyPiyVyiMjOzpEuXtM1xucpSE8Eh2baxaF8AR5U3HDMz20S3bmm7yy65naKkRBARn8gtAjMz27yO2dd0pe8slrSLpO9JmpE9rpOUX3oyM7OkWhIBcDPwHnBG9lgO/FdeQZmZWaZwQ1mOiaDUMYK9IuK0otf/KmlWDvGYmVmxwv0DL72U3ylKLLdS0l8XXmQ3mK3c2ockjZX0nKT5kq5o5f1Bkh6W9JSk2ZJOKD10M7M68vrruR261BbBJcCt2biAgCXAeVv6gKQOwETgGKAJmC5pckQ8W1TsKuDuiLhB0khgCjB4m2pgZlYPKt01FBGzgP0l9cxeLy/hYwcD8yNiAYCku4CTgeJEEEC2/A67APmlPDOzWnXggZW7oUzS2RHxM0n/3GI/ABHxvS18vD9QvNBmExvvRyj4BvBbSROAnYGjNxPHRcBFAIMGDdpSyGZm7c+6dbneULa1MYKds22PzTx21FnALRExADgBuE3SB2KKiEkR0RgRjX379i3Dac3MakjPntCnT26H32KLICJuzLb/uh3HXggMLHo9INtX7AJgbHaOaZK6AH2At7fjfGZm7VO/fvDOO7kdvtQbyr4jqaekBklTJS2SdPZWPjYdGCZpiKROwJnA5BZlXgU+mZ1jBNAFWLRtVTAza+caGqpiqcpjswHiE4GXgb2By7f0gYhYB1wKPAjMI10dNFfS1ZLGZcW+CFwo6WngTuC8iIhtr4aZWTu2006wYEFuhy/18tFCuU8Bv4iIZYUB4y2JiCmkS0KL932t6PmzwBElxmBmVp+WLNm4LkEOSm0R/EbSn4GDgKmS+gKrcovKzMw2+qu/2rguQQ5KSgQRcQVwONAYEWuB90n3BJiZWd46darcCmWSjoqIhySdWrSvuMiv8grMzMwyhcHiCCihW35bbW2MYAzwEHBSK+8FTgRmZvlbvDhtV6/euGJZGW3tPoKvZ9vzy35mMzMrTb9+aZvTfEOl3kfwTUm9il7vKunaXCIyM7NNFVoBGzbkcvhSrxo6PiKWFl5ExLukKSHMzCxvhTUJcrrNqtRE0EFS58ILSV2Bzlsob2Zm5VIYIM6pRVDqDWW3k+4fKCxPeT5way4RmZnZpnJuEZS6HsG/Z9NAFKaJviYiHswlIjMz21SVtAggzRe0LiJ+L6mbpB4Rkd9KCWZmllTDGIGkC4FfAjdmu/oD9+YSkZmZbSrnFkGpg8XjSZPDLQeIiBeAD+cSkZmZbaoaWgTA6oj4y2TYkjqS7iw2M7O8VUmL4H8l/QvQVdIxwC+AX+cSkZmZbaqQCCrcIvgyaeWwZ4B/IK0xcFUuEZmZ2aYKXUOVumpIUgdgbkQMB36SSxRmZrZ5lW4RRMR64DlJg3KJwMzMtqwabigDdgXmSvo/0qI0WUwxbvMfMTOzsii0CHKafbTURPDVXM5uZmZbt3x52r7//pbLbaetrVDWBbgY2Js0UHxTROSTkszMrHX9+6dtDquTwdbHCG4FGklJ4HjgulyiMDOzzSuMEaxfn8vht9Y1NDIi9gWQdBPwf7lEYWZmm9ehQ9rmlAi21iJYW3jiLiEzswopJIIK3Uewv6RslAKR7ixenj2PiOiZS1RmZrZRJbuGIqJDLmc1M7PSVbhryMzMKs2JwMyszhXuKF68OJfDOxGYmVW7hoa07dIll8M7EZiZVbucEkCBE4GZWa2o8HoEZmZWKTlNLVHgRGBmVuecCMzM6pwTgZlZrfAYgZlZnarlMQJJYyU9J2m+pCtaef/7kmZlj+clLc0zHjMz+6BSVyjbZtmi9xOBY4AmYLqkyRHxbKFMRPxTUfkJwAF5xWNmVvNqsGvoYGB+RCyIiDXAXcDJWyh/FnBnjvGYmdWmGu4a6g+8VvS6Kdv3AZL2BIYAD23m/YskzZA0Y9GiRWUP1MysnlXLYPGZwC8jotWp9SJiUkQ0RkRj37592zg0M7MqUYNdQwuBgUWvB2T7WnMm7hYyM2tdDXcNTQeGSRoiqRPpy35yy0KShgO7AtNyjMXMzDYjt0SQrXF8KfAgMA+4OyLmSrpa0riiomcCd0Xk1OYxM2svcvqazO3yUYCImAJMabHvay1efyPPGMzMal4Ndw2ZmVkNcCIwM6sVNXjVkJmZlYO7hszMLE9OBGZmtcJdQ2ZmdcpdQ2ZmlicnAjOzWuGuITOzOuWuITMzy5MTgZlZnXMiMDOrFR4jMDOrUx4jMDOzPDkRmJnVCncNmZnVKXcNmZlZnpwIzMxqhbuGzMzqlLuGzMwsT04EZma1wl1DZmZ1yl1DZmaWJycCM7Na4a4hM7M65a4hMzPLkxOBmVmtcNeQmVmdcteQmZnlyYnAzKxWuGvIzKxOuWvIzMzy5ERgZlYr3DVkZlan3DVkZmZ5ciIwM6tzTgRmZrWiFscIJI2V9Jyk+ZKu2EyZMyQ9K2mupDvyjMfMrCblPEbQMa8DS+oATASOAZqA6ZImR8SzRWWGAV8BjoiIdyV9OK94zMysdXm2CA4G5kfEgohYA9wFnNyizIXAxIh4FyAi3s4xHjOz2laDXUP9gdeKXjdl+4rtA+wj6TFJj0sa29qBJF0kaYakGYsWLcopXDOzKtXOLx/tCAwDjgTOAn4iqVfLQhExKSIaI6Kxb9++bRuhmVk7l2ciWAgMLHo9INtXrAmYHBFrI+Il4HlSYjAzs5ZqsGtoOjBM0hBJnYAzgcktytxLag0gqQ+pq2hBjjGZmdWeWu0aioh1wKXAg8A84O6ImCvpaknjsmIPAoslPQs8DFweEYvzisnMzD4ot8tHASJiCjClxb6vFT0P4J+zh5mZbUkNdg2ZmVk51GrXkJmZ1QYnAjOzWuGuITOzOuWuITMzy5MTgZlZrXDXkJlZnXLXkJmZ5cmJwMysVjz6aC6HzfXOYjMzK4NBg+Dii+G003I5vBOBmVm169ABbrght8O7a8jMrM45EZiZ1TknAjOzOudEYGZW55wIzMzqnBOBmVmdcyIwM6tzTgRmZnVOkdNsdnmRtAh4ZTs/3gd4p4zh1ALXuT64zvVhR+q8Z0T0be2NmksEO0LSjIhorHQcbcl1rg+uc33Iq87uGjIzq3NOBGZmda7eEsGkSgdQAa5zfXCd60Muda6rMQIzM/ugemsRmJlZC04EZmZ1rl0mAkljJT0nab6kK1p5v7Okn2fvPyFpcAXCLKsS6vzPkp6VNFvSVEl7ViLOctpanYvKnSYpJNX8pYal1FnSGdnf9VxJd7R1jOVWwr/tQZIelvRU9u/7hErEWS6Sbpb0tqQ5m3lfkn6Y/XnMlnTgDp80ItrVA+gAvAgMBToBTwMjW5T5R+DH2fMzgZ9XOu42qPMngG7Z80vqoc5ZuR7AI8DjQGOl426Dv+dhwFPArtnrD1c67jao8yTgkuz5SODlSse9g3X+OHAgMGcz758A3A8IOBR4YkfP2R5bBAcD8yNiQUSsAe4CTm5R5mTg1uz5L4FPSlIbxlhuW61zRDwcEc3Zy8eBAW0cY7mV8vcMcA3w78CqtgwuJ6XU+UJgYkS8CxARb7dxjOVWSp0D6Jk93wV4vQ3jK7uIeARYsoUiJwM/jeRxoJekPXbknO0xEfQHXit63ZTta7VMRKwDlgG92yS6fJRS52IXkH5R1LKt1jlrMg+MiPvaMrAclfL3vA+wj6THJD0uaWybRZePUur8DeBsSU3AFGBC24RWMdv6/32rvHh9nZF0NtAIjKl0LHmStBPwPeC8CofS1jqSuoeOJLX6HpG0b0QsrWRQOTsLuCUirpN0GHCbpI9ExIZKB1Yr2mOLYCEwsOj1gGxfq2UkdSQ1Jxe3SXT5KKXOSDoauBIYFxGr2yi2vGytzj2AjwB/kPQyqS91co0PGJfy99wETI6ItRHxEvA8KTHUqlLqfAFwN0BETAO6kCZna69K+v++LdpjIpgODJM0RFIn0mDw5BZlJgOfy57/DfBQZKMwNWqrdZZ0AHAjKQnUer8xbKXOEbEsIvpExOCIGEwaFxkXETMqE25ZlPJv+15SawBJfUhdRQvaMMZyK6XOrwKfBJA0gpQIFrVplG1rMnBudvXQocCyiHhjRw7Y7rqGImKdpEuBB0lXHNwcEXMlXQ3MiIjJwE2k5uN80qDMmZWLeMeVWOf/ALoDv8jGxV+NiHEVC3oHlVjndqXEOj8IHCvpWWA9cHlE1Gxrt8Q6fxH4iaR/Ig0cn1fLP+wk3UlK5n2ycY+vAw0AEfFj0jjICcB8oBk4f4fPWcN/XmZmVgbtsWvIzMy2gROBmVmdcyIwM6tzTgRmZnXOicDMrM45EZi1QtJ6SbMkzZH0a0m9ynz8l7Pr/JG0opzHNttWTgRmrVsZEaMj4iOke03GVzogs7w4EZht3TSySb0k7SXpAUkzJf1R0vBs/26S/lvS09nj8Gz/vVnZuZIuqmAdzDar3d1ZbFZOkjqQpi+4Kds1Cbg4Il6QdAjwI+Ao4IfA/0bEKdlnumfl/y4ilkjqCkyXdE8t3+lr7ZMTgVnrukqaRWoJzAN+J6k7cDgbp+kA6JxtjwLOBYiI9aSpzQE+L+mU7PlA0gRwTgRWVZwIzFq3MiJGS+pGmudmPHALsDQiRpdyAElHAkcDh0VEs6Q/kCZEM6sqHiMw24JsVbfPkyY2awZeknQ6/GXt2P2zolNJS4AiqYOkXUjTm7+bJYHhpKmwzaqOE4HZVkTEU8Bs0gIonwUukPQ0MJeNyyZ+AfiEpGeAmaS1cx8AOkqaB3ybNBW2WdXx7KNmZnXOLQIzszrnRGBmVuecCMzM6pwTgZlZnXMiMDOrc04EZmZ1zonAzKzO/T9J805RJkTo8QAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSP AUPR score: 0.9757307094062874\n"
     ]
    }
   ],
   "source": [
    "precision, recall, aupr_score = aupr(test_y, ood_scores)\n",
    "\n",
    "plot_precision_recall_curve(precision, recall)\n",
    "print('MSP AUPR score:', aupr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSP fpr95 score: 0.0756\n"
     ]
    }
   ],
   "source": [
    "fpr95 = fpr_at_95_tpr(test_y, ood_scores)\n",
    "print('MSP fpr95 score:', fpr95)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OOD detection by Mahalanobis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n",
      "100%|██████████| 98/98 [00:24<00:00,  3.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "covairance norms: [58.213383, 111.35267, 49.149143, 44.527348, 99.387886, 59.178444, 104.78499, 80.21675, 87.104904, 63.132908]\n",
      "covariance norm: 29.670227021092874\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.metrics.mahalanobis import Mahalanobis\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.Resize(img_size),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(dataset_mean, dataset_std),\n",
    "])      \n",
    "cifar10 = CIFAR10(\n",
    "    root=dataset_root, \n",
    "    train=True, \n",
    "    download=False, \n",
    "    transform=transform_test\n",
    ")\n",
    "id_train_dataloader = DataLoader(\n",
    "    dataset=cifar10, \n",
    "    batch_size=config.train.batch_size, \n",
    "    shuffle=False, \n",
    "    num_workers=8\n",
    ")\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "feature_extractor = FeatureExtractor(\n",
    "    model=model,\n",
    "    layer_name=config.model.layer_name.penultimate,\n",
    ")\n",
    "mahalanobis = Mahalanobis(config, model, id_train_dataloader, feature_extractor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 15.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 261/261 [00:13<00:00, 18.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id mean score: 215.34715 id variance: 21136.686\n",
      "ood mean score: 693.95 ood variance: 44660.0\n"
     ]
    }
   ],
   "source": [
    "test_y, ood_scores, id_ood_scores, ood_ood_scores = compute_ood_scores(\n",
    "    metric=mahalanobis,\n",
    "    in_dist_dataloader=id_test_dataloader,\n",
    "    out_of_dist_dataloader=ood_test_dataloader,\n",
    ")\n",
    "print('id mean score:', np.mean(id_ood_scores), 'id variance:', np.var(id_ood_scores))\n",
    "print('ood mean score:', np.mean(ood_ood_scores), 'ood variance:', np.var(ood_ood_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 16.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0, airplane: 1018, mean: 220.49879455566406, var: 23541.91015625\n",
      "1, automobile: 964, mean: 195.30661010742188, var: 18286.0078125\n",
      "2, bird: 1058, mean: 243.42190551757812, var: 30870.400390625\n",
      "3, cat: 1026, mean: 248.9990234375, var: 29320.724609375\n",
      "4, deer: 962, mean: 197.35235595703125, var: 13548.0478515625\n",
      "5, dog: 1033, mean: 217.2900848388672, var: 16825.470703125\n",
      "6, frog: 962, mean: 196.2056121826172, var: 19143.857421875\n",
      "7, horse: 964, mean: 204.74002075195312, var: 18380.091796875\n",
      "8, ship: 982, mean: 206.94052124023438, var: 18990.318359375\n",
      "9, truck: 1031, mean: 217.32928466796875, var: 18014.09375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 261/261 [00:13<00:00, 18.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0, airplane: 1880, mean: 698.2384033203125, var: 33407.17578125\n",
      "2, bird: 5769, mean: 660.5449829101562, var: 30104.4140625\n",
      "3, cat: 16120, mean: 702.5452880859375, var: 47090.9609375\n",
      "4, deer: 1, mean: 888.8529663085938, var: 0.0\n",
      "5, dog: 894, mean: 534.7012329101562, var: 22808.62890625\n",
      "6, frog: 80, mean: 707.6121215820312, var: 19461.208984375\n",
      "7, horse: 3, mean: 621.4541625976562, var: 12089.70703125\n",
      "8, ship: 17, mean: 639.4715576171875, var: 41562.0859375\n",
      "9, truck: 1268, mean: 861.5384521484375, var: 52534.41015625\n"
     ]
    }
   ],
   "source": [
    "mahalanobis.compute_dataset_ood_score(id_test_dataloader)\n",
    "classes = set(mahalanobis.closest_classes)\n",
    "for c in classes:\n",
    "    tmd = np.array(mahalanobis.total_mahalanobis_distances)\n",
    "    cc = np.array(mahalanobis.closest_classes)\n",
    "    c_tmd = tmd[list(np.where(cc == c)[0])]\n",
    "    n_samples = mahalanobis.closest_classes.count(c)\n",
    "    print(f'{c}, {cifar10.classes[c]}: {n_samples}, mean: {c_tmd.mean()}, var: {c_tmd.var()}')\n",
    "\n",
    "mahalanobis.compute_dataset_ood_score(ood_test_dataloader)\n",
    "classes = set(mahalanobis.closest_classes)\n",
    "for c in classes:\n",
    "    tmd = np.array(mahalanobis.total_mahalanobis_distances)\n",
    "    cc = np.array(mahalanobis.closest_classes)\n",
    "    c_tmd = tmd[list(np.where(cc == c)[0])]\n",
    "    n_samples = mahalanobis.closest_classes.count(c)\n",
    "    print(f'{c}, {cifar10.classes[c]}: {n_samples}, mean: {c_tmd.mean()}, var: {c_tmd.var()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA8d0lEQVR4nO3dd5gUVdbA4d9hyEgGRckZhiDoiAEJipJE1M+wmFZ2UUREYE2Loq6yypowoKCgKKiIrrgqCoquiqyKSs5pRMkIDDkzM+f741ZrO07ogamu6e7zPk8/1d1VXXWqurtO3VtV94qqYowxJnEVCToAY4wxwbJEYIwxCc4SgTHGJDhLBMYYk+AsERhjTIKzRGCMMQnOEkGcE5GlItIx6DgKCxG5V0ReDmjZ40Xk4SCWXdBE5FoR+fQYP2u/yULGEkEUicjPInJQRPaJyBZvx3CCn8tU1WaqOsPPZYSISAkR+ZeIrPPWc7WI3CUiEo3lZxNPRxHZEP6eqg5X1Rt9Wp6IyEARWSIi+0Vkg4i8IyIt/FjesRKRB0XkjeOZh6pOVNXOESzrD8nvWH+TIlLci321t31/FpFXRKROfudlfs8SQfRdrKonAK2A1sA9wYaTfyJSNIdR7wCdgO5AWeB6oC/wrA8xiIgUtt/vs8AgYCBQCWgEvA9cVNALyuU78F2Ay54M9ASuAcoDpwJzcb+5fAly+xVKqmqPKD2An4ELwl4/DkwNe30W8C2wC1gIdAwbVwl4FdgE7ATeDxvXA1jgfe5boGXWZQKnAAeBSmHjWgPbgWLe678Cy735Twdqh02rwK3AauCnbNatE3AIqJnl/TOBDKCB93oG8C/gB2AP8EGWmHLbBjOAR4BvvHVpAPzFi3kvsAa42Zu2jDdNJrDPe5wCPAi84U1Tx1uvG4B13rYYGra8UsAEb3ssB+4GNuTw3Tb01rNNLt//eGAUMNWL93ugftj4Z4H13naZC7QLG/cgbkf4hjf+RqANMMvbVpuB54HiYZ9pBnwG7AB+Ae4FugJHgKPeNlnoTVseGOfNZyPwMJDkjevtbfOngTRvXG/ga2+8eOO2erEtBprjDgKOesvbB3yY9X8AJHlx/ehtk7lk+Q15013gfZ9/GJfL/yu777qP913PBD4GBmSZx0Lg/7znTcK230rgqqD3Ib7tm4IOIJEeWf4ANbw/zLPe6+ren6w7rqR2ofe6qjd+KvA2UBEoBnTw3m/t/QHP9P5UN3jLKZHNMr8AbgqL5wngRe/5JUAq0BQoCtwHfBs2rXp/ikpAqWzW7VHgqxzWey2/7aBneDua5rid9bthf9a8tsEM70/czIuxGO5ouz5uZ9QBOACc5k3fkSw77hx2Di/hdvqnAoeBpuHr5G3zGsCirPMLm28/YG0e3/94b33aePFPBN4KG38dUNkbdwewBSgZFvdR4FJv25QCTsclzqLeuiwHBnvTl8Xt1O8ASnqvz8y6DcKW/R4wxvtOTsQl6tB31htIB27zllWK3yeCLrgdeAXve2gKnBy2zg/n8j+4C/c/aOx99lSgcn5+X9nNN5fv+jVvHUsBfwa+CZs+GZdUS3jTrMcdaBTlt4Om5KD3I348ClvROhG8LyJ7cT+yrcA/vPevA6ap6jRVzVTVz4A5QHcRORnoBvRT1Z2qelRVv/I+1xcYo6rfq2qGqk7A7czOymbZbwJXg6taAXp574Hbkf1LVZerajowHGglIrXDPv8vVd2hqgezmXcV3I4nO5u98SGvq+oSVd0P3A9cJSJJuW2DsM+OV9WlqprubYepqvqjOl8BnwLtcogjJw+p6kFVXYg7IjzVe/8qYLi3zTcAI3OZR+Vc1j/ce6r6g7eNJ+KqCAFQ1TdUNc1btxG4HVLjsM/OUtX3vW1zUFXnqup33vQ/43bkHbxpewBbVHWEqh5S1b2q+n12AYnISbhtPFhV96vqVtwRfq+wyTap6nPesrJ+/0dxiaYJIN5vKJJtAa5kc5+qrvS+w4WqmpbNdJFu37w86K3jQVzyC/+NXwv8R1UP47bfz6r6qrfO83EHLVcWQAyFjiWC6LtUVcvijlab8NsOsjZwpYjsCj2Ac4GTgZrADlXdmc38agN3ZPlcTVw1SFbvAmd7iaU9rtrkf2HzeTZsHjtwR2jVwz6/Ppf12u7Fmp2TvfHZzWct7si+Crlvg2xjEJFuIvKdiOzwpu/O75NOJLaEPT8AhE7gn5Jlebmtfxo5r38ky0JE7hSR5SKy21uX8vx+XbKueyMR+ci78GAPLnmHpq+Jq26JRG3cd7A5bLuPwZUMsl12OFX9AlctNQrYKiJjRaRchMuONM5It29efl0PVd2LK2mHEt7VuOQMbpucmeW3eC1QrQBiKHQsEQTEO3odDzzpvbUed6RcIexRRlUf9cZVEpEK2cxqPfBIls+VVtVJ2SxzJ+6I+U+4E25vqaqGzefmLPMpparfhs8il1X6L+6PUzP8TRE5E/dn/yLs7fBpauGOKLfnsQ3+EIOIlMAltyeBk1S1AjANl8DyijcSm3FVQtnFndXnQA0RSTmWBYlIO9w5iKuAit667Oa3dYE/rs8LwAqgoaqWw9W1h6ZfD9TLYXFZ57MeV4qsErbdy6lqs1w+8/sZqo5U1dNx1SuNcFU+eX7OW3b9PKYB9/tqIyI1cplmP1A67HV2O+2s8UwCrhaRs3FVaF+GxfVVlt/iCap6SwSxxhxLBMF6BrhQRE7FnQS8WES6iEiSiJT0Ln+s4RWzPwZGi0hFESkmIu29ebwE9BORM70racqIyEUiUjaHZb6Jqxu9gt+qhQBeBO4RkWYAIlJeRCIuBqvqf3E7w3dFpJm3Dmd56/WCqq4Om/w6EUkWkdLAMGCyqmbktg1yWGxxXPXJNiBdRLoB4Zc0/gJUFpHyka5HFv/GbZOKIlIdGJDThN76jQYmeTEX9+LvJSJDIlhWWVw9/DagqIg8AOR1VF0Wd3J2n4g0AcJ3Uh8BJ4vIYHGX9Zb1kjK47VIndNWV9/v6FBghIuVEpIiI1BeRDkRARM7wfn/FcDvjQ7jSZmhZOSUkgJeBf4pIQ+/321JEKmedyPt9fQa8JyKni0hRb536ichfvckWAL28/0cK7jeel2m4o/9hwNuqGor7I6CRiFzvza+Yt55NI5hnzLFEECBV3YY7efWAqq7HnbC9F7czWI87qgp9R9fjjpxX4M4tDPbmMQe4CVc034k74ds7l8VOwV3hssWrEw/F8h7wGPCWV82wBHdeIj8uxx1RfYK7SuQN3JUot2WZ7nVcaWgL7ihsoBdDXtvgd7yi/UDcDnsnrpQzJWz8CtwR3xqveJ9ddVluhgEbgJ9wR6STcUfOORnIb1Uku3BVHpcBH0awrOm47bYKV112iNyrogDuxK3zXtwBwduhEd62uRC4GLedVwPneaPf8YZpIjLPe/5nXGJdhtuWk4m8Kqact/ydXuxpuAsRwH3/yd72fz+bzz6F+/4+xSW1cbgTudm5ArfjfhtXWloCpOC+G3Dnm+p7cTzE7w90suWdD/gP7qqkN8Pe34s7qOiFu1JvC+7/USKvecYi+a1mwBj/icgM3JUcgdzdezxE5Bagl6pGdKRsTKywEoExORCRk0WkrVdV0hh3KeZ7QcdlTEGzu+uMyVlx3NUzdXFVPW/hzgMYE1esasgYYxKcVQ0ZY0yCi7mqoSpVqmidOnWCDsMYY2LK3Llzt6tq1ezGxVwiqFOnDnPmzAk6DGOMiSkisjancVY1ZIwxCc4SgTHGJDhLBMYYk+AsERhjTIKzRGCMMQnOt0QgrlPprSKyJIfxIiIjRSRVRBaJyGl+xWKMMSZnfpYIxuP6R81JN1wrmA1xvWy94GMsxhhjcuDbfQSqOlNE6uQyySXAa17HKN+JSAUROTkfXdyZY6EKR4/+9jhyBPbsgcxMyMjI/rFrFxQv7qaJ5LFtG5QpAyJueaHlhseQ13vHOs7v6XMbt3kzVKwIRQKocQ2yqZiglp1Ay93PEbZxgDoXXw9nnFHg8w/yhrLq/L699Q3ee39IBCLSF1dqoFatWlEJrtA5cgT27oUdO2D/fti0yT3S02HlSihZEubPh4MHYd8+95kVK6BChd92+Lt3B7oKxsQNkbynKSBf1FFuuhjKH4Y5c5pQJM4SQcRUdSwwFiAlJSV+W8lThdRUWLQIpkyBH3907y1c6Hb+eSlf3h2Rn3QS1KkDF10E27dD48buiL5ECUhLg3r1oFix3x5790KtWpCUlP0jI8MlmpIl3dFuJA9wy4Pf/jThf55I3jvWcX5Pn9u4okWDKRGEx5Boy45Tuw7t4q5P7+Ll+S/ToFIDnr74ZYrU8acrjCATwUZ+3wdsDe+9xJGZCTNmwEcfwfffw7ff/n58mTJQtSpccglUrgw1a0KpUlClijvSr1TJ7fQrV/6tKsYYE/MyMjM4Z9w5rExbyd3n3M2DHR+kVLGcOm47fkEmginAABF5CzgT2J0Q5wdU4d134YEHYPny394vVw4uvxxOPRXOPx9at4bSpXOejzEm7qQdSKNSqUokFUnikfMfoWb5mqSckuL7cn1LBCIyCegIVBGRDcA/gGIAqvoiru/R7rg+dg8Af/ErlkJhzx644QZ4/333unp16NYNOnRwR/xNmgQanjEmOKrKxMUTGfTJIB7t9Cg3nX4TlzW9LGrL9/OqoavzGK/ArX4tv1BJTYWGDd3zE06Am2+GRx75rQ7dGJOw1u9eT7+p/Zi2ehpn1TiLtrXaRj2GmDhZHNM+/BB69nTP//lPuO++YOMxxhQakxZP4uaPbiZDM3imyzMMaDOApCJJUY/DEoFfVGHIEHj8cXcyd+RIuOaaoKMyxhQiFUtV5MwaZzK2x1jqVqwbWByWCPzSuze89po7DzBxorvRyBiT0NIz03l61tMcyTjC0PZD6dqgK13qd0ECvuLPEoEf5s51SaBOHZg61S7rNMawcMtC+kzpw9zNc7mq2VWoKiISeBIAa3204K1bB2ed5W6++u9/LQkYk+AOpx/m/i/uJ+WlFNbvWc87V77DW5e/VSgSQIiVCArSjh1w9tmu2Yc33oD69YOOyBgTsNU7VvPYN49xTYtreKrzU1QuXTnokP7AEkFBUYXLLnPt/3z0kWvewRiTkPYd2ccHKz7g2pbX0vzE5qwYsIJ6FesFHVaOrGqooHz8McycCXfcYUnAmAT22Y+f0eKFFlz/3vUs3+ZaDyjMSQAsERScIUPc8J57go3DGBOInQd30ueDPnR+ozPFk4rzVe+vaFq1adBhRcSqhgrCwoWweDGcfrq7Z8AYk1AyMjNo+0pbVqWt4p5z7+GBDg9QsmjJoMOKmCWCgvD3v7vhO+8EG4cxJqq2H9j+ayNxwzsNp1b5Wpx2cuz1umtVQ8dr6FCYPh369oW6wd0ZaIyJHlXltYWv0ei5Rrw872UALm1yaUwmAbASwfHZsAGGD4dGjWDUqKCjMcZEwdpda7n5o5uZ/uN0zql5Du1rtw86pONmieB4TJjghvff73qmMsbEtTcWvcEtU29BVXmu23P0P6M/RST2K1Zs73Ws0tNhzBjXeczVuba4bYyJE1VLV6VtzbaM6TGG2hVqBx1OgbFEcKymToX16+HNN12/vsaYuHM04ygjZo3gaMZR7u9wP10adKFz/c6FqnmIgmCJ4Fg9+qgbdu4cbBzGGF/M3zyfPlP6MH/LfHo171WoGokraLFfuRWEjAz47jvo3t3uGzAmzhxKP8S9n9/LGS+dwaa9m3j3qneZdPmkuEwAIVYiOBahTufPPTfYOIwxBS51RypPfvskfz71z4zoPIKKpeK/LxFLBMdi3Dg3DHVBaYyJafuO7OO95e9x/anX0/zE5qwcsDLQHsOizaqG8uvoURg9GmrUgGbNgo7GGHOcpqdOp9noZtzw/g2/NhKXSEkALBHk34wZcOQIDB4cdCTGmOOQdiCNG96/ga4Tu1K6WGn+95f/xUwjcQXNqobyK9Se0F//GmwcxphjFmokLnVHKkPbDeW+9vfFVCNxBc0SQX699BKceKJ1Rm9MDNq2fxuVS1cmqUgSj13wGLUr1KZVtVZBhxU4qxrKj6lT3TA5Odg4jDH5oqq8Ov9VGj3fiJfmvgTAJU0usSTgsRJBfoSqhSZNCjYOY0zEft71M30/7Mtnaz6jXa12nFf3vKBDKnQsEUQqI8M1Mle3LlSrFnQ0xpgIvL7wdW6Zegsiwujuo7k55ea4aCSuoFkiiNSCBW543XWBhmGMidxJJ5xE+9rtebHHi9QqXyvocAotSwSRWrHCDc85J9g4jDE5OppxlMe/eZwMzeCBDg/QuX5nOte39sDyYmWkSH3+uRumpAQbhzEmW/M2z+OMl87gvi/vY2XaSlQ16JBihiWCSIVKBFWqBBuHMeZ3Dh49yJD/DqHNS234Zf8vvPen95j4fxPjupG4guZrIhCRriKyUkRSRWRINuNriciXIjJfRBaJSHc/4zlmBw/CrFlw/vlBR2KMyWLNzjU8NesperfqzbL+y7i0yaVBhxRzfEsEIpIEjAK6AcnA1SKS9QL8+4B/q2proBcw2q94jsusWW54xRXBxmGMAWDP4T2MXzAegGYnNmP1bat5uefLCdFSqB/8LBG0AVJVdY2qHgHeAi7JMo0C5bzn5YFNPsZz7ObNc8O2bYONwxjDtNXTaD66OX2m9Pm1kbh46jYyCH4mgurA+rDXG7z3wj0IXCciG4BpwG3ZzUhE+orIHBGZs23bNj9izd2MGSACLVpEf9nGGAC2H9jO9e9dz0VvXkTZEmX55q/fJGwjcQUt6JPFVwPjVbUG0B14XeSPd3uo6lhVTVHVlKpVq0Y9SDZvdknATj4ZE4hQI3FvLXmLB9o/wLy+8zirxllBhxU3/LyPYCNQM+x1De+9cH2ArgCqOktESgJVgK0+xpV/a9daJzTGBOCXfb9QtUxVkook8eSFT1K7Qm1antQy6LDijp8lgtlAQxGpKyLFcSeDp2SZZh3QCUBEmgIlgQDqfnKxYwekpbmmJYwxUaGqjJs3jsbPN2bs3LEAXNz4YksCPvGtRKCq6SIyAJgOJAGvqOpSERkGzFHVKcAdwEsi8jfciePeWtjuApk82Q3btAk2DmMSxJqda7jpw5v44qcv6FC7AxfUuyDokOKer01MqOo03Eng8PceCHu+DCjcl+J8/bUbWiIwxncTFkyg/7T+JEkSL170IjedfpM1EhcF1tZQXmbNgqQk64jGmCg4pewpnF/3fF646AVqlKsRdDgJwxJBbg4dgtRUa2jOGJ8cyTjCo18/SqZm8mDHB7mw/oVcWP/CoMNKOFbmyk3ojuJrrgk2DmPi0OyNszl97On8Y8Y/WLNzjTUSFyBLBLlZutQNrY0hYwrMgaMHuPPTOzlr3FnsPLiTKb2m8Nplr1kjcQGyqqHczJoFJUtCkyZBR2JM3Php508898Nz3HTaTTx2wWOUL1k+6JASniWC3Eyf7pqdtiMVY47L7kO7+c/y//CX1n+h2YnNSL0tlZrla+b9QRMVVjWUE1V3I1m5cnlPa4zJ0dRVU2k2uhk3fngjK7a7fj0sCRQulghysmWLG3bpEmwcxsSobfu3ce1/rqXHpB5ULFWRWX1m0aSKVbMWRlY1lJNly9ywpd3Sbkx+ZWRmcO6r5/LTzp94qONDDDl3CMWTigcdlsmBJYKcrFrlhvXrBxuHMTFky74tnFjmRJKKJDGi8wjqVKhD8xObBx2WyUPEVUMiUtrPQAqdRYvcsKm1d25MXjI1kzFzxtDouUaMmTMGgB6NelgSiBF5JgIROUdElgErvNenikjh7FKyIIXaGLLO6o3JVeqOVDq91ol+U/txRvUz6NLAzqvFmkiqhp4GuuA1Ia2qC0Wkva9RFQYiUNu6vzMmN6/Of5X+0/pTPKk4L138En1a97Ebw2JQROcIVHV9li83w59wCon0dFi8GPr3DzoSYwq1WuVr0aV+F0Z1H0X1cll7ojWxIpJEsF5EzgFURIoBg4Dl/oYVsB9+cMMa1vqhMeEOpx/mX1//i0zNZNh5w+hUrxOd6nUKOixznCI5WdwPuBXX8fxGoBUQ34fKr7/uhlddFWwcxhQi32/4ntPHns5DXz3Eut3rrJG4OBJJiaCxql4b/oaItAW+8SekQuCzz9zQLh01hv1H9nP/l/fzzHfPUL1cdT66+iMuanRR0GGZAhRJieC5CN+LH6pQqVLQURhTKKzdvZbRs0fTL6UfS/svtSQQh3IsEYjI2cA5QFURuT1sVDlcH8Txa80auPrqoKMwJjC7Du1i8rLJ3HjajSRXTSZ1YKr1GBbHcqsaKg6c4E1TNuz9PcAVfgYVqF273LBMmUDDMCYoH6z4gFum3sLW/Vs5t9a5NKnSxJJAnMsxEajqV8BXIjJeVddGMaZgzZnjhu3j/1YJY8Jt3b+VgR8P5O2lb9PypJZMuXqKNRKXICI5WXxARJ4AmgElQ2+qanx22zV/vhu2bRtsHMZEUUZmBm1facu63et4+LyHubvt3RRLKhZ0WCZKIkkEE4G3gR64S0lvALb5GVSgVq92w7p1g43DmCjYtHcT1U6oRlKRJJ7t+ix1KtQhuWpy0GGZKIvkqqHKqjoOOKqqX6nqX4H4LA2Au6O4enXrlczEtUzN5IXZL9Dk+Sa8OOdFALo37G5JIEFFUiI46g03i8hFwCYgfq+tXL7c+ig2cW1V2ipu+vAmZq6dyQX1LqBbg25Bh2QCFkkieFhEygN34O4fKAcM9jOowKSnw969lghM3Bo3bxwDPh5AyaIleaXnK/Ru1dsaiTN5JwJV/ch7uhs4D369szj+LF8OmZnQrl3QkRjjizoV6tCtQTdGdR/FyWVPDjocU0jkdkNZEnAVro2hT1R1iYj0AO4FSgGtoxNiFKWmumFN61jbxIfD6Yf558x/AvDw+Q9bI3EmW7mVCMYBNYEfgJEisglIAYao6vtRiC36lnuNqjZoEGwcxhSAb9d/S58pfVixfQV/bfVXVNWqgUy2cksEKUBLVc0UkZLAFqC+qqZFJ7QArFvnhtYhjYlh+47sY+jnQ3nuh+eoWb4mn1z7ifUaZnKV2+WjR1Q1E0BVDwFr8psERKSriKwUkVQRGZLDNFeJyDIRWSoib+Zn/gVu3jxISYGk+G5KycS3dbvXMWbuGG4941aW3LLEkoDJU24lgiYi4vXgjgD1vdcCqKq2zG3G3jmGUcCFwAZgtohMUdVlYdM0BO4B2qrqThE58TjW5fjNnm2NzZmYtPPgTt5Z9g59T+9LctVk1gxawyllTwk6LBMjcksETY9z3m2AVFVdAyAibwGXAMvCprkJGKWqOwFUdetxLvPYHTrkhqVLBxaCMcfiveXv0X9af7bt30aH2h1oXKWxJQGTL7k1One8Dc1VB9aHvd4AnJllmkYAIvINrmnrB1X1k6wzEpG+QF+AWrVqHWdYOVixwg3POsuf+RtTwLbs28JtH9/G5GWTaVWtFVOvmUrjKo2DDsvEoIg6r/d5+Q2BjkANYKaItFDVXeETqepYYCxASkqKP/3jrfXyXrVqvszemIKUkZlBu1fbsX73eoafP5w7z7nTGokzx8zPRLARd/lpSA3vvXAbgO9V9Sjwk4iswiWG2T7Glb39+93QLh01hdiGPRs4pewpJBVJYmTXkdStWNeaijbHLZJG5xCRUiKS3zLnbKChiNQVkeJAL2BKlmnex5UGEJEquKqiNflcTsEI3UNg5whMIZSpmTz3/XM0eb4JL8x+AYBuDbtZEjAFIs9EICIXAwuAT7zXrUQk6w79D1Q1HRgATAeWA/9W1aUiMkxEenqTTQfSRGQZ8CVwV2D3KahX41S9eiCLNyYnK7avoP2r7Rn4yUDOrXUuPRr1CDokE2ciqRp6EHcF0AwAVV0gIhE11q+q04BpWd57IOy5Ard7j2CtWgWnnGL3EJhC5eV5LzNg2gBKFyvNhEsncH3L6+3uYFPgImqGWlV3Z/nx+XPCNkhr1liro6bQqV+xPhc3vpjnuz3PSSecFHQ4Jk5FkgiWisg1QJJ3A9hA4Ft/w4qyzExYuhR69sx7WmN8dCj9EMO+GgbA8E7DOa/ueZxX97yAozLxLpKTxbfh+is+DLyJa456sI8xRd+8ee6GMruHwATom3Xf0OrFVvzr63+xbf82VOOv4G0Kp0hKBE1UdSgw1O9gArNggRtaPwQmAHsP7+Xez+9l1OxR1K5Qm+nXTadz/c5Bh2USSCSJYISIVAMmA2+r6hKfY4q+PXvc0K4YMgHYsGcDL89/mdva3MYjnR7hhOInBB2SSTB5Vg2p6nm4nsm2AWNEZLGI3Od7ZNG0cqUbVqkSbBwmYaQdSPv1foCmVZuyZuAanu32rCUBE4iIbihT1S2qOhLoh7un4IHcPxFjNmxww2J2i77xl6oyedlkkkcnM/CTgazc7g5CrNtIE6RIbihrKiIPishiXOf13+Kai4gfRYpA+fJBR2Hi3Oa9m7n835dz5TtXUrNcTebcNMcaiTOFQiTnCF4B3ga6qOomn+MJxsyZdqLY+CrUSNzGvRt5/ILH+dvZf6NokaDbfDTGyfOXqKpnRyOQwNmlesYH63evp3q56iQVSWJU91HUrViXRpUbBR2WMb+TY9WQiPzbGy4WkUVhj8VhPZfFhz17oHnzoKMwcSQjM4OR34+kyajfGonr0qCLJQFTKOVWIhjkDeO7hasDB9zQ2m8xBWT5tuX0mdKHWRtm0a1BNy5ufHHQIRmTqxxLBKq62XvaX1XXhj+A/tEJLwp+/NENK1QINAwTH8bOHUurMa1YlbaK1y97nanXTKVWeZ961TOmgERy+eiF2bzXraADCcyWLW7YunWwcZi40LBSQy5rchnLbl3GdS2vs5ZCTUzIsWpIRG7BHfnXy3JOoCzwjd+BRc3ChW546qnBxmFi0sGjB3lwxoOICI9e8Kg1EmdiUm7nCN4EPgb+BQwJe3+vqu7wNapoCjUvcZI18WvyZ+bamdw45UZW71hNv9P7oapWAjAxKbeqIVXVn4Fbgb1hD0Skkv+hRcnSpVC0qJ0sNhHbc3gP/af2p8P4DmRoBp//+XNe6PGCJQETs/IqEfQA5uI6ogn/lStQz8e4omfTJuuVzOTLpr2bGL9gPLefdTvDzhtGmeJlgg7JmOOSYyJQ1R7eMKJuKWPWunXW6qjJ0/YD2/n30n/T/4z+NKnShJ8G/WQ9hpm4EUlbQ21FpIz3/DoReUpE4ud6OBGoXTvoKEwhpaq8veRtkkclM/iTwaxKWwVgScDElUguH30BOCAipwJ3AD8Cr/saVbSowsaNUC8+arlMwdq0dxOXvn0pvd7tRe0KtZnbd67dGWziUiStXqWrqorIJcDzqjpORPr4HVhUbN3qhjXiqzFVc/wyMjNo/2p7Nu7dyJMXPsmgswZZI3EmbkXyy94rIvcA1wPtRKQIEB8N969yxXyqVQs2DlNorN21lhrlapBUJInRF42mXsV6NKjUIOiwjPFVJFVDf8J1XP9XVd2C64vgCV+jipYd3u0QLVsGG4cJXEZmBk/Neoqmo5rywhzXSFzn+p0tCZiEEElXlVuAiUB5EekBHFLV13yPLBpmz3ZDO1mc0JZsXcI5r5zDHZ/eQad6nbi0yaVBh2RMVEVy1dBVwA/AlcBVwPcicoXfgUVFZqYbnnJKsHGYwLw450VOG3Maa3au4c3/e5MpvaZQo5ydMzKJJZJzBEOBM1R1K4CIVAX+C0z2M7Co+OUXKF3a7ipOQKHmIJpWacqVza7kmS7PULVM1aDDMiYQkSSCIqEk4Ekjwk7vC70dO+Dw4aCjMFF04OgBHvjyAZIkiccufIwOdTrQoU6HoMMyJlCR7NA/EZHpItJbRHoDU4Fp/oYVJevWQa34uTfO5G7GzzNo+UJLRswawb4j+1DrntQYILI+i+8Skf8DzvXeGquq7/kbVpQcPQrF4uNKWJOz3Yd2c/dndzN23ljqV6zPF3/+wpqKNiZMbv0RNASeBOoDi4E7VXVjtALz3YEDruXRAQOCjsT4bPO+zbyx+A3uPPtOHjrvIUoXKx10SMYUKrlVDb0CfARcjmuB9Ln8zlxEuorIShFJFZEhuUx3uYioiKTkdxnHbNcud9VQ48ZRW6SJnm37t/Hc9+4n26RKE34e9DNPdH7CkoAx2citaqisqr7kPV8pIvPyM2MRSQJG4bq63ADMFpEpqrosy3RlgUHA9/mZ/3Fbt84NS9uOIZ6oKpOWTGLgxwPZc3gPXRp0oVHlRnZFkDG5yK1EUFJEWovIaSJyGlAqy+u8tAFSVXWNqh4B3gIuyWa6fwKPAYfyHf3xCN1MVqVKVBdr/LN+93ounnQx1/7nWhpUasD8m+dbI3HGRCC3EsFm4Kmw11vCXitwfh7zrg6sD3u9ATgzfAIvodRU1akicldOMxKRvkBfgFoFdZXPwYNu2LZtwczPBCo9M52OEzqyZd8Wnu7yNLe1uY2kItbhkDGRyK1jGl8vq/Aar3sK6J3XtKo6FhgLkJKSUjDX/C1Z4oYVKhTI7Ewwft71MzXL1aRokaKM6TGGehXrUa+iNStuTH74eWPYRqBm2Osa3nshZYHmwAwR+Rk4C5gStRPGGza4od1VHJPSM9N58tsnaTqqKaNnjwbggnoXWBIw5hj42cD6bKChiNTFJYBewDWhkaq6G/i1gl5EZuAuUZ3jY0y/2bbNuqiMUYt+WUSfKX2Ys2kOlzS+hMuTLw86JGNimm+JQFXTRWQAMB1IAl5R1aUiMgyYo6pT/Fp2RA4dgoYNAw3B5N/o2aMZ9MkgKpasyNtXvM2VyVciVqoz5rjkmQjE/cuuBeqp6jCvv+JqqvpDXp9V1WlkaY5CVR/IYdqOEUVcUA4ftuanY0iokbjmJzanV/NePN3laaqUtiu+jCkIkZQIRgOZuKuEhgF7gXeBM3yMy3/r10OJEkFHYfKw/8h+7vviPooWKcoTnZ+gfe32tK/dPuiwjIkrkZwsPlNVb8W7zl9VdwLFfY3Kb6F+CPbsCTYOk6vP13xOixda8Mz3z3A447A1EmeMTyIpERz17hJW+LU/gkxfo/Lb9u1uWLdusHGYbO06tIs7P72TcfPH0bBSQ2b2nkm72u2CDsuYuBVJiWAk8B5woog8AnwNDPc1Kr+FEkHTpsHGYbL1y75feGvJW/y97d9Z2G+hJQFjfBZJM9QTRWQu0AkQ4FJVXe57ZH7a6N3OUK1asHGYX4V2/oPOGkTjKo35efDPdjLYmCiJ5KqhWsAB4MPw91R1nZ+B+erIETcsXz7YOAyqysTFExn0ySD2HdlH94bdaVi5oSUBY6IoknMEU3HnBwQoCdQFVgLNfIzLX+npbljUz/vpTF7W7V5Hv4/68XHqx5xd42zG9RxHw8p2b4cx0RZJ1VCL8NdeQ3H9fYsoGo4edUPrnSww6ZnpdBzfka37tzKy60j6n9HfGokzJiD5PiRW1XkicmbeUxZioXaGLBFE3Zqda6hdvjZFixTlpYtfon6l+tSpUCfosIxJaJGcI7g97GUR4DRgk28RRcOKFW5Y1ToriZb0zHRGfDuCf8z4B49f+DgDzxxIp3qdgg7LGENkJYKyYc/TcecM3vUnnCjZudMNK1cONo4EsWDLAvpM6cO8zfO4rMllXJl8ZdAhGWPC5JoIvBvJyqrqnVGKJ3pOPjnoCBLC8z88z9+m/43KpSoz+crJ1lKoMYVQjolARIp6LYjGXxde27dbtZDPQo3EtTypJde2uJanujxFpVKVgg7LGJON3EoEP+DOBywQkSnAO8D+0EhV/Y/Psfln69bfrhwyBWrfkX0M/XwoxZKK8WTnJ62ROGNiQCTnCEoCabjWR0P3EygQu4mgVCkoVy7oKOLOpz9+St8P+7Ju9zpua3Pbr6UCY0zhllsiONG7YmgJvyWAkNhuBnLxYujWLego4sbOgzu5/dPbGb9gPI0rN2bmX2Zybq1zgw7LGBOh3BJBEnACv08AIbGbCFRd72RWNVRgtu7fyuRlk7nn3Ht4oMMDlCxaMuiQjDH5kFsi2Kyqw6IWSbQcOOCGNWoEG0eM27JvC5MWT+JvZ//NNRI36Gcql7bLcY2JRbk1Qx2flbvbtrmhdVx/TFSVCQsmkDwqmXs+v4fVaasBLAkYE8NySwTxedtnqFeyxo2DjSMG/bzrZ7pO7ErvD3qTXDWZBf0WWCNxxsSBHKuGVHVHNAOJmlCJwLo9zJf0zHTOm3Ae2w9sZ1T3UfRL6UcRiaRfI2NMYZd47TCH+is+5ZRg44gRqTtSqVuhLkWLFOWVnq9Qr2I9aleoHXRYxpgClHiHdIcPu2GJEsHGUcgdzTjK8P8Np9noZoyaPQqA8+qeZ0nAmDiUeCWC9evd0DqlydG8zfPoM6UPC7Ys4MrkK/lTsz8FHZIxxkeJtzcMlQQqVgw2jkJq5PcjuX367VQtU5X/XPUfLmt6WdAhGWN8lniJ4NAhNyxVKtg4CplQcxCtq7Xmz6f+mRGdR1CxlCVLYxJB4iWCH390w5J29yvA3sN7uefzeyiRVIIRXUbQrnY72tVuF3RYxpgoSryTxWXKuKE1OscnqZ/Q/IXmjJ49GkVRu6TWmISUeCWCVavceYIEbhUz7UAat396O68tfI2mVZryzV+/4eyaZwcdljEmIImXCNavh/T0oKMIVNrBNN5b/h73t7+foe2GUqKoXUprTCLztWpIRLqKyEoRSRWRIdmMv11ElonIIhH5XET8v0h98WKoU8f3xRQ2m/du5slvn0RVaVS5EWsHr2XYecMsCRhj/EsEXn/Ho4BuQDJwtYgkZ5lsPpCiqi2BycDjfsXzq717oVo13xdTWKgqr8x/haajmnL/l/eTuiMVwK4IMsb8ys8SQRsgVVXXqOoR4C3gkvAJVPVLVfXaheY7IDptQydnzUfx6aedP9H5jc70mdKHU6udysJ+C62ROGPMH/h5jqA6sD7s9QbgzFym7wN8nN0IEekL9AWoVavWsUeUkeGGNWse+zxiRHpmOue/dj5pB9J44aIX6Ht6X2skzhiTrUJxslhErgNSgA7ZjVfVscBYgJSUlGO/xjHUK1nx4sc8i8Juddpq6lWsR9EiRXn1klepX7E+NcvHf+Izxhw7Pw8RNwLhe6Aa3nu/IyIXAEOBnqp62Md43PkB+O3u4jhyNOMoD898mOYvNOf5H54HoGOdjpYEjDF58rNEMBtoKCJ1cQmgF3BN+AQi0hoYA3RV1a0+xuKEEkGcnSyes2kOfab0YdEvi+jVvBdXt7g66JCMMTHEt0SgqukiMgCYDiQBr6jqUhEZBsxR1SnAE8AJwDvibvBap6o9/YqJHV5fO5Xjp1vFZ797lts/vZ1qJ1Tjg14f0LOxf5vPGBOffD1HoKrTgGlZ3nsg7PkFfi7/D3btcsMqVaK6WD+EGolLOSWFPq378PiFj1OhZIWgwzLGxKBCcbI4akKd0oTaG4pBew7v4e+f/Z2SRUvydNenaVurLW1rtQ06LGNMDEus6wlDndLE6FVD01ZPo9noZoydN5aiRYpaI3HGmAKRWCWCYsXcsGzZYOPIp+0HtjP4k8FMXDyRZlWbMfnKyZxZI7dbMowxJnKJlQj273fDGEsEOw/u5MNVH/KPDv/g3nb3UjwpNks0xpjCKbESQaprZycWzhFs3LORiYsnctc5d9GwckPWDl5rJ4ONMb5IrHMEoURQunSwceRCVXlp7kskj07mwRkP8uNO16OaJQFjjF8SKxEULVqoeyb7ccePdHqtE30/6stpJ5/GolsW0aBSg6DDMsbEucSqGlq5Emr73+XBsUjPTKfTa53YcXAHY3qM4cbTbrRG4owxUZFYiaBs2ULXReXK7SupX6k+RYsUZcKlE6hfqT41ykWnNW5jjIFEqxpasqTQlAiOZBzhoRkP0eKFFoz6YRQAHep0sCRgjIm6xCkRqLo7i48cCToSftj4A32m9GHJ1iVc0+Iarm15bdAhGWMSWOIkglACCLhE8Mx3z3DHp3dw8gkn8+HVH9KjUY9A4zHGmMRJBKE+COrVC2TxoUbi2lRvw02n3cRjFzxG+ZLlA4nFGGPCJU4iCLU8mp4e1cXuPrSbuz+7m1LFSvFM12c4p+Y5nFPznKjGYIwxuUmck8Wh5iWi2CnNhys/JHl0Mi/Pf5kSSSWskThjTKGUOCWCLVvcMAp3FW/bv41Bnwxi0pJJtDixBe//6X3OqH6G78s1xphjkTiJIHQ0HoU7i3cf3s201dN4qONDDDl3iDUSZ4wp1BInEYSuGvKp5dH1u9fzxqI3GHLuEBpUasDawWvtZLAxJiYkzjmCo0fdMNQnQQHJ1ExenPMizUY34+H/PfxrI3GWBIwxsSJxEkHoHEEB9k62Om015084n1um3kKb6m1YfMtiayTOGBNzEqdqKFQSKFowq5yemc6Fr1/IrkO7GNdzHH9p9RekkLVjZIwxkUicRBBynFcNLd+2nIaVG1K0SFFev+x16leqzyllTymg4Iwxfjp69CgbNmzgUOgG0zhUsmRJatSoQbF8VIMnTiLIzHTDYzxqP5x+mOH/G87wr4fzxIVPMPiswbSr3a4AAzTG+G3Dhg2ULVuWOnXqxGUJXlVJS0tjw4YN1K1bN+LPJU4iCF0+WiT/p0W+2/Adfab0Ydm2ZVzf8nqub3l9AQdnjImGQ4cOxW0SABARKleuzLZt2/L1ucRJBMdYIhjx7Qju+uwuapSrwbRrptGtYTcfgjPGREu8JoGQY1m/xEsEEZYIMjWTIlKEs2ueTb+Ufjx6waOUK1F4u7k0xphjlTiXj0ZYNbTr0C76fNCHQR8PAuCcmucw+qLRlgSMMQUiKSmJVq1a0bx5cy6++GJ2hRrEBJYuXcr5559P48aNadiwIf/85z9/10bZxx9/TEpKCsnJybRu3Zo77rijQGJKnEQQQdXQ+yveJ3lUMhMWTqBsibLWSJwxpsCVKlWKBQsWsGTJEipVqsSoUa6HwoMHD9KzZ0+GDBnCypUrWbhwId9++y2jR48GYMmSJQwYMIA33niDZcuWMWfOHBo0KJj7lhKnaiiXEsHW/VsZMG0A7yx7h1bVWvHRNR9x2smnRTlAY0xUDR4MCxYU7DxbtYJnnol48rPPPptFixYB8Oabb9K2bVs6d+4MQOnSpXn++efp2LEjt956K48//jhDhw6lSZMmgCtZ3HLLLQUSduKVCLJJBHsO7+GzNZ/xyPmP8MONP1gSMMb4LiMjg88//5yePXsCrlro9NNP/9009evXZ9++fezZs4clS5b8YXxBSZwSQZaqoXW71/H6wte5t929NKjUgHWD11G2hD8N0hljCqF8HLkXpIMHD9KqVSs2btxI06ZNufDCCwOJI5yvJQIR6SoiK0UkVUSGZDO+hIi87Y3/XkTq+BaMVzWUKTB69miajW7G8K+H/9pInCUBY0w0hM4RrF27FlX99RxBcnIyc+fO/d20a9as4YQTTqBcuXI0a9bsD+MLim+JQESSgFFANyAZuFpEkrNM1gfYqaoNgKeBx/yKh8xMVlaGjpN7cOu0Wzm7xtks7b/UGokzxgSidOnSjBw5khEjRpCens61117L119/zX//+1/AlRwGDhzI3XffDcBdd93F8OHDWbVqFQCZmZm8+OKLBRKLnyWCNkCqqq5R1SPAW8AlWaa5BJjgPZ8MdBKf7vZIzzhKl+th8fZlvHrJq0y/bjp1KtTxY1HGGBOR1q1b07JlSyZNmkSpUqX44IMPePjhh2ncuDEtWrTgjDPOYMCAAQC0bNmSZ555hquvvpqmTZvSvHlz1qxZUyBx+HmOoDqwPuz1BuDMnKZR1XQR2Q1UBraHTyQifYG+ALVq1TqmYIo2SeaNZedRf+g4Tq4SeRscxhhTkPbt2/e71x9++OGvz1u0aMGMGTNy/GyPHj3o0aNHgccUE1cNqepYVU1R1ZSqVase20x69uTc8V9YEjDGmCz8TAQbgZphr2t472U7jYgUBcoDaT7GZIwxJgs/E8FsoKGI1BWR4kAvYEqWaaYAN3jPrwC+ULud1xjjo3jfxRzL+vmWCFQ1HRgATAeWA/9W1aUiMkxEenqTjQMqi0gqcDvwh0tMjTGmoJQsWZK0tLS4TQah/ghKliyZr89JrG2QlJQUnTNnTtBhGGNiUCL3UCYic1U1JbvPJM6dxcaYhFesWLF89dyVKGLiqiFjjDH+sURgjDEJzhKBMcYkuJg7WSwi24C1x/jxKmS5azkB2DonBlvnxHA861xbVbO9IzfmEsHxEJE5OZ01j1e2zonB1jkx+LXOVjVkjDEJzhKBMcYkuERLBGODDiAAts6JwdY5Mfiyzgl1jsAYY8wfJVqJwBhjTBaWCIwxJsHFZSIQka4islJEUkXkDy2aikgJEXnbG/+9iNQJIMwCFcE63y4iy0RkkYh8LiK1g4izIOW1zmHTXS4iKiIxf6lhJOssIld53/VSEXkz2jEWtAh+27VE5EsRme/9vrsHEWdBEZFXRGSriCzJYbyIyEhveywSkdOOe6GqGlcPIAn4EagHFAcWAslZpukPvOg97wW8HXTcUVjn84DS3vNbEmGdvenKAjOB74CUoOOOwvfcEJgPVPRenxh03FFY57HALd7zZODnoOM+znVuD5wGLMlhfHfgY0CAs4Dvj3eZ8VgiaAOkquoaVT0CvAVckmWaS4AJ3vPJQCcRkSjGWNDyXGdV/VJVD3gvv8P1GBfLIvmeAf4JPAbEQ7vDkazzTcAoVd0JoKpboxxjQYtknRUo5z0vD2yKYnwFTlVnAjtymeQS4DV1vgMqiMjJx7PMeEwE1YH1Ya83eO9lO426DnR2A5WjEp0/IlnncH1wRxSxLM919orMNVV1ajQD81Ek33MjoJGIfCMi34lI16hF549I1vlB4DoR2QBMA26LTmiBye//PU/WH0GCEZHrgBSgQ9Cx+ElEigBPAb0DDiXaiuKqhzriSn0zRaSFqu4KMiifXQ2MV9URInI28LqINFfVzKADixXxWCLYCNQMe13Dey/baUSkKK44mRaV6PwRyTojIhcAQ4Geqno4SrH5Ja91Lgs0B2aIyM+4utQpMX7COJLveQMwRVWPqupPwCpcYohVkaxzH+DfAKo6CyiJa5wtXkX0f8+PeEwEs4GGIlJXRIrjTgZPyTLNFOAG7/kVwBfqnYWJUXmus4i0BsbgkkCs1xtDHuusqrtVtYqq1lHVOrjzIj1VNZb7OY3kt/0+rjSAiFTBVRWtiWKMBS2SdV4HdAIQkaa4RLAtqlFG1xTgz97VQ2cBu1V18/HMMO6qhlQ1XUQGANNxVxy8oqpLRWQYMEdVpwDjcMXHVNxJmV7BRXz8IlznJ4ATgHe88+LrVLVnYEEfpwjXOa5EuM7Tgc4isgzIAO5S1Zgt7Ua4zncAL4nI33AnjnvH8oGdiEzCJfMq3nmPfwDFAFT1Rdx5kO5AKnAA+MtxLzOGt5cxxpgCEI9VQ8YYY/LBEoExxiQ4SwTGGJPgLBEYY0yCs0RgjDEJzhKBKZREJENEFoQ96uQy7b4CWN54EfnJW9Y87w7V/M7jZRFJ9p7fm2Xct8cbozef0HZZIiIfikiFPKZvFeutcRr/2eWjplASkX2qekJBT5vLPMYDH6nqZBHpDDypqi2PY37HHVNe8xWRCcAqVX0kl+l741pdHVDQsZj4YSUCExNE5ASvH4V5IrJYRP7Q0qiInCwiM8OOmNt573cWkVneZ98Rkbx20DOBBt5nb/fmtUREBnvvlRGRqSKy0Hv/T977M0QkRUQeBUp5cUz0xu3zhm+JyEVhMY8XkStEJElEnhCR2V4b8zdHsFlm4TU2JiJtvHWcLyLfikhj707cYcCfvFj+5MX+ioj84E2bXYutJtEE3fa2PeyR3QN3V+wC7/Ee7i74ct64Kri7KkMl2n3e8A5gqPc8CdfeUBXcjr2M9/7fgQeyWd544Arv+ZXA98DpwGKgDO6u7KVAa+By4KWwz5b3hjPw+jwIxRQ2TSjGy4AJ3vPiuFYkSwF9gfu890sAc4C62cS5L2z93gG6eq/LAUW95xcA73rPewPPh31+OHCd97wCri2iMkF/3/YI9hF3TUyYuHFQVVuFXohIMWC4iLQHMnFHwicBW8I+Mxt4xZv2fVVdICIdcJ2VfOM1rVEcdySdnSdE5D5cOzV9cO3XvKeq+70Y/gO0Az4BRojIY7jqpP/lY70+Bp4VkRJAV2Cmqh70qqNaisgV3nTlcY3F/ZTl86VEZIG3/suBz8KmnyAiDXHNLBTLYfmdgZ4icqf3uiRQy5uXSVCWCEysuBaoCpyuqkfFtShaMnwCVZ3pJYqLgPEi8hSwE/hMVa+OYBl3qerk0AsR6ZTdRKq6SlxfB92Bh0Xkc1UdFslKqOohEZkBdAH+hOtoBVxvU7ep6vQ8ZnFQVVuJSGlc+zu3AiNxHfB8qaqXeSfWZ+TweQEuV9WVkcRrEoOdIzCxojyw1UsC5wF/6HNZXD/Mv6jqS8DLuO7+vgPaikiozr+MiDSKcJn/Ay4VkdIiUgZXrfM/ETkFOKCqb+Aa88uuz9ijXskkO2/jGgoLlS7A7dRvCX1GRBp5y8yWut7mBgJ3yG9NqYeaIu4dNuleXBVZyHTgNvGKR+JapTUJzhKBiRUTgRQRWQz8GViRzTQdgYUiMh93tP2sqm7D7RgnicgiXLVQk0gWqKrzcOcOfsCdM3hZVecDLYAfvCqafwAPZ/PxscCi0MniLD7FdQz0X3XdL4JLXMuAeeI6LR9DHiV2L5ZFuI5ZHgf+5a17+Oe+BJJDJ4txJYdiXmxLvdcmwdnlo8YYk+CsRGCMMQnOEoExxiQ4SwTGGJPgLBEYY0yCs0RgjDEJzhKBMcYkOEsExhiT4P4fjHeFFsGBX/4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis AUROC score: 0.97032881\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, auroc_score = auroc(test_y, ood_scores)\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "print('Mahalanobis AUROC score:', auroc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAo00lEQVR4nO3debxd873/8dc740lkkImSQSKiSQjBQXGVGiJUpa0OtK6hKqho6/b6ldZ0g1b1R11XKL1SqipXqz83bakqMbSlzUGiGRoihpygIgMyyvD5/fFdR7aTnZyd5Kyzzz7n/Xw89mPttdZ3rfX5JuxPvt/vWt+liMDMzKy+NuUOwMzMmicnCDMzK8oJwszMinKCMDOzopwgzMysKCcIMzMrygnCKp6kL0v6Qwnlfizp0qaIqSlIekXSUdn3KyT9vNwxWcviBGG5yn7EVkpaJumfku6Q1KUxrxERd0fEqBLKnRMRVzbmtetICknLs3oukHS9pLZ5XGtrSOom6QZJr2UxvpSt9y53bNZ8OUFYU/hURHQB9gWqgUvqF5DUrsmjanx7Z/U8DPgi8JUyxwOApA7AI8AewGigG3AQsAg4YCvO1xL+rqwEThDWZCJiAfAgsCd88K/u8yS9CLyYbTte0jRJSyX9RdJedcdL6i/p15IWSlok6aZs++mS/pR9l6QfSXpL0ruS/i6p7np3SLqq4HxnSZorabGkyZJ2LtgXks6R9GIWywRJKrGec4E/AyMLzrc19Ros6dFs29uS7pa0/Rb+sQOcCgwAPhMRsyJifUS8FRFXRsQDBfXdrSCmD/6sJB0uqVbStyW9CfxU0mxJxxeUb5fFv2+2/rGsnkslTZd0+FbEbWXmBGFNRlJ/4DjguYLNnwYOBIZL2geYCJwN9AJuBSZL6ph11/wWeBUYCPQFJhW5zCjg48DuQHfgC6R/KdeP5Qjg+9n+nbLz1j/f8cD+wF5ZuWNKrOdQ4FBgbra+tfVSFuPOwDCgP3BFKTHUcxTw+4hYthXH1vkI0BPYBRgL3AOcXLD/GODtiHhWUl/gd8BV2TH/Dtwnqc82XN/KwAnCmsL9kpYCfwIeB75XsO/7EbE4IlaSfnhujYi/RsS6iLgTWA18jNQVsjNwYUQsj4hVEfGnItdaA3QFhgKKiNkR8UaRcl8GJkbEsxGxGrgYOEjSwIIy10TE0oh4DZhCQYtgE56VtByYDTwG3Jxt36p6RcTciHg4IlZHxELgelL31ZbqBRT7M9gS64HLs1hWAr8ATpDUOdv/JVLSADgFeCAiHshaKw8DNaR/HFgFcYKwpvDpiNg+InaJiK9lPzB15hd83wX4VtYtsTRLKv1JP6D9gVcjYu3mLhQRjwI3AROAtyTdJqlbkaI7k/7VXnfcMlJLo29BmTcLvq8AugBImpkN9C6TdGhBmX2zMl8ktYq225Z6SdpR0qRs0Ptd4OfA1gwqLyK1krbFwohYVbeSdaPNBj6VJYkTSEkDUn0/X6++/9IIMVgTc4KwciucTng+cHWWTOo+nSPinmzfgFIGSCPixojYDxhO6mq6sEix10k/ZABI2o70L+0FJZx/j4jokn2erLcvIuJe4Cngsm2s1/dIfz4jIqIb6V/mJY2D1PNH4JisjpuyAuhcsP6RevuLTftc1800BpiVJQ1IdbqrXn23i4hrtiJ2KyMnCGtOfgKcI+nAbLB5O0mflNQV+Bupm+SabHuVpEPqn0DS/tnx7YHlwCpS90h99wBnSBopqSPpx/ivEfFKI9XlGuAsSR/Zhnp1BZYB72T9+sUSXSnuIv1o3ydpqKQ2knpJ+o6kum6facCXJLWVNJrSurImkcZ8zmVD6wFSS+dTko7JzleVDXT328r4rUycIKzZiIga4CxSF9ES0iDv6dm+dcCngN2A14BaUldOfd1IP8hLSF1Ii4AfFrnWH4FLgftIP9CDgZMasS5/B54gjS1sbb3+g9Rt9Q5p0PfXWxnLatJA9T+Ah4F3SYmpN/DXrNg3sjiWksZn7i/hvG+QWkoHA/9TsH0+qVXxHWAhKTldiH9vKo78wiAzMyvGGd3MzIpygjAzs6KcIMzMrCgnCDMzK6rFTLrVu3fvGDhwYLnDMDOrKM8888zbEVF0GpQWkyAGDhxITU1NucMwM6sokl7d1D53MZmZWVFOEGZmVpQThJmZFdVixiDMzLbFmjVrqK2tZdWqVQ0XrkBVVVX069eP9u3bl3yME4SZGVBbW0vXrl0ZOHAgJb48sGJEBIsWLaK2tpZBgwaVfFxuXUySJiq99nHGJvZL0o1Kr3x8vu5Vhdm+05Re9fiipNPyitHMrM6qVavo1atXi0sOAJLo1avXFreO8hyDuIP0gvRNORYYkn3GArcASOoJXE564coBwOWSeuQYp5kZQItMDnW2pm65dTFFxBP1Xt9Y3xjgZ5Gmk31a0vaSdgIOBx6OiMUAkh4mJZp7NnmmbbF8OfzgB6WVleDLX4bdd88lFDOz5qScYxB9+fDrJmuzbZvavhFJY0mtDwYMGLB1UaxYAVddVVrZCFi8GC67DN58E956a8OnTx/40pe2LgYzM6Bt27aMGDGCtWvXMmjQIO666y623357AGbOnMn555/PggULWL9+PaeeeiqXXHLJBy2DBx98kEsvvZQVK1bQsWNHjjjiCK677rptiqeib3ONiNsiojoiqvv0KfqkeMP69IH160v79O4NN90EO+wAe+0FRx2VksI3v5laFmedBa+/nhLJ22+npZlZiTp16sS0adOYMWMGPXv2ZMKECQCsXLmSE044gYsuuog5c+Ywffp0/vKXv3DzzTcDMGPGDMaNG8fPf/5zZs2aRU1NDbvttts2x1POFsQC0gvb6/TLti0gdTMVbn+syaLanMsvhxdfhF13hZ13Tolihx3gySfh7LPhv/87fTp3Ti2T7beH006DY4+FoUPh5Zdh+PB0jJnZZhx00EE8//zzAPziF7/gkEMOYdSoUQB07tyZm266icMPP5zzzjuPa6+9lu9+97sMHToUSC2Rc889d5tjKGeCmAyMkzSJNCD9TkS8Iekh4HsFA9OjgIvLFeSHjBtXfPuwYSkRfP7zaX3wYLjhBli6FP7zP9OnUO/eKdFkTUcza2a++U2YNq1xzzlyZPpdKMG6det45JFHOPPMM4HUvbTffvt9qMzgwYNZtmwZ7777LjNmzOBb3/pW48ZLjglC0j2klkBvSbWkO5PaA0TEj4EHgONI7+ddAZyR7Vss6Upganaq8XUD1s1ax44wefKG9fHjYeVKeOqp9Bk8GO65B6ZMSd1PY8bA44+XL14za3ZWrlzJyJEjWbBgAcOGDePoo48uazx53sV0cgP7AzhvE/smAhPziKvJdO2aPmPGpA+kMYq1a6F9e3jiCbj2WnjjDejbF+qyfwu+zc6sYpT4L/3GVjcGsWLFCo455hgmTJjA17/+dYYPH84TTzzxobLz5s2jS5cudOvWjT322INnnnmGvffeu3EDiogW8dlvv/2iYnztaxFpCHvjz/DhET/7WbkjNGt1Zs2aVe4QYrvttvvg+7PPPhsDBgyINWvWxIoVK2LQoEHx8MMPR0TEihUr4pOf/GTceOONERExffr0GDx4cMyZMyciItatWxe33HLLRucvVkegJjbxu1rRdzFVrAkT4NFHYfp0ePpp2Gkn6NYt7Zs1C049Fer1N5pZ67LPPvuw1157cc8999CpUyf+93//l6uuuoqPfvSjjBgxgv33359x2bjoXnvtxQ033MDJJ5/MsGHD2HPPPZk3b942x6BoIbdiVldXR8W/MOj99+Gxx+CYY9L6rrvCiSemW2hHjEjdT+6CMsvF7NmzGTZsWLnDyFWxOkp6JiKqi5V3C6I56dABRo2Cn/wkrc+bBz/8Ybr7oW1baNMmJYjp08sappm1Dk4QzdFXvwpr1qTb7MaPT91Pe+21Yf/IkR9OEi2kFWhmzYsTRHPVrh3svTdceim8805KCOvXw6GHpv0jR6bE0bv3hpbFV78KV1+dWiF13VHZgzZm1rCW0uVezNbUzQmikkjp9tizz07r772Xntru3Tut3347XHIJvPLKhmP23jsdd8QR6cG8z38+DYBffDEcd1y65bZTJ/jOd5q6NmbNSlVVFYsWLWqRSSKy90FUVVVt0XEepK5UdTfGtsly/DPPpAkEDz4YemQPoZ9zDtx6a+nnPOww+NGPUuvEg+HWyrTWN8ptbpDaCaKli0h3R7VtC//8J/TsmVoY/ftDly6pzGOPwSc+8eHjvv3tVPY3v0n72rSBl16CF16AZctg9Oh0i+6gQbBkCTzyCOy/PyxaBLNnwze+Ab16pWXdLbxm1uw4QVjDXnkFJk6EK6/c+nP07JmmQy/mK19J3V3r18Nf/pK6x3r1gj//Od3Cu8sucOONxVsuEWlK9TffTGXbuGfUrLE4QVjpVq9Oc0etXZvunJozJ/1A9+sHAwakH/h//hM+8pE0eN65c/pEbPhxnz07zWZ76KFpPqotNWYMLFgADf19duqUnhU54YQUU+/eaWbdvfaCM8+EPfeEqip3l5lthhOElde8eWmMpHv31Ipo1y51b3XokBLRggWpq6pO587pR/7NN9P6ccfBkCFwwQVbd/22bVNC69oV9t0X7r572+tk1kI4QVjlWL++9C6kJUvSuEiPHimp7LBD6oqaOBHuv7/hFkidSy5Jz5u4pWGtkBOEGcCrr8LAgSmh7L8//OEPG5cZNCi92AlSoho+HGbMSOtf+EK6S6xtWzj99A2D/GYVzAnCbFP+9jc45ZT0Aqd+/aC2dsO+AQPSHVubGniHlGyGDk2J5a230sD7fvulQff27dN07mvWwNixaczErJlxgjDbVrW18NxzafD9iSdSq6Jufv527dJYSkP69EnPmKxdm7536JDOucMO6SHGIUPgk59M33v2TC2Yd95Jg+/bb58Sjlkjc4IwawoRaWB9+vR0G+/gwend5ZdfDrfdlp49adMmdXVtjd69U8vk8cdhu+0aN3ZrtZwgzJqT9es3nrp9+XL43e9SN9X8+amVMWhQSjhvvpnuBJsy5cPnqapKSWi//WC33WDu3NTyaNcuvda2W7c0blLXOunQoUmraZXBCcKsJVi7Nt3qW1MDu++exk2eemrjclVVsLnpIjp0gE9/Os3L1a9fep96v36p28tanbIlCEmjgf8E2gL/HRHX1Nu/C+nd032AxcApEVGb7VsH/D0r+lpEnLC5azlBWKv0/vup1dG9expM7949fZYvhzvuSM9+zJwJf/pT6t6aPXvzg+4dO6Y7tUaMSAPwAwemZ1e6dEktmna5vcbeyqQsCUJSW+AF4GigFpgKnBwRswrK/BL4bUTcKekI4IyI+Nds37KIKPk+QicIsxJFpOlOZs1KT7q/9Va65Xfq1IaP/Zd/SWMhy5fDRz+axlV69ICDDkrJxLf+VpxyJYiDgCsi4phs/WKAiPh+QZmZwOiImC9JwDsR0S3b5wRhVg4rV8LChWmgfcqUlAx+9rOUUBqyzz7pTqxDDoGPfzw9wGjN2uYSRJ7txb7A/IL1WuDAemWmA58ldUN9BugqqVdELAKqJNUAa4FrIuL++heQNBYYCzBgwIBGr4BZq9SpU3oGBGCPPdLy29/+cJmFC9Mtv8uWpRl/X3wxPcH+3HPpU9/OO8MXv5gGzpctS62QwYPTQ4cHH5ymQrFmJ88WxOdIrYOvZuv/ChwYEeMKyuwM3AQMAp4ATgT2jIilkvpGxAJJuwKPAkdGxEubup5bEGZlFpEmTZw8Ob3J8LXX4B//SMmjFF27psHyPfdMCaNfPzj22LTuaVByU64WxAKgf8F6v2zbByLidVILAkldgBMjYmm2b0G2nCfpMWAfYJMJwszKTEo/7GPHFt//9tvpTqxXX03Ty7dvn94jMmlSGjh/7700iD579oZj6louvXql95IMGpSmjh882A8ONoE8WxDtSIPUR5ISw1TgSxExs6BMb2BxRKyXdDWwLiIuk9QDWBERq7MyTwFjCge463MLwqyFWLUqPQsyY0aaRHHatE2XHTIkJY133km37w4cmKaLP+CA1AJxy6NB5bzN9TjgBtJtrhMj4mpJ44GaiJicdUN9HwhSF9N5WVI4GLgVWE96b/YNEXH75q7lBGHWgq1bl1ogkyalu63uvjs9Td6pU9q+KV26pPexn3BCaoF06JAeHPTdVh/wg3Jm1vK9/356Gn3KFPjlL9O8V7///eaP2WknuPNOOPLIVvumQicIM2udItKT53fdlR4g7NAB7rsvzZdVX9++aXbfnXdu+jjLyAnCzKyYKVNS8vjpTz+8fcCAdAvuiSfCuHFpksQWygnCzKwhF1yQ3kS4cmW6XbfQUUelZHHqqS3u4b/NJYjW2elmZlbfj36U3ib45pupa+q99+Daa9O+P/4Rzj03DYzXzcTbs2ca81i9urxx58gJwsysmC5d4MILN7zn4//8n/Q0+PHHp/mnlixJ36uq0rQipcxlVWE8NaOZWUN23BF+8IMPb/vDH9I0I7Nnpwf+Djggbe/bF/7rv+Azn2n6OBuZWxBmZltj1KiUCP74x/Sypk99Km1fsAA++1k48MDU+qhgThBmZttq8OA0B1UEvPFGmhrkb39Ldz/dckt6RqMCOUGYmTWmj3wEXn89vbFv/nz42tfSi5hOOSXNgltBnCDMzBpbhw5w771pUsJvfCNtu/tu2GGHdAfU8ceniQubOScIM7O87LIL3HBDmkvqllvSG/gg3R7bvj389rdlDa8hThBmZnlr0wbOOSe9I2P9+vS6VkgD21dfXd7YNsMJwsysKUkffpHSJZfAV79a3pg2wQnCzKwcdtsNHnwwfb/99pQ4br65vDHV4wRhZlYuo0eneZ/q3sl93nlpCo9mMoDtBGFmVk477JCenVi0KE0EuGRJGsBes6bckTlBmJk1Cz17wrJlG6bsuOii8saDE4SZWfMhwdNPpy6n66+HY44pazhOEGZmzYmUXmQEaULAX/+6bKHkmiAkjZY0R9JcSRu1lyTtIukRSc9LekxSv4J9p0l6MfuclmecZmbNytCh8MAD6fuJJ8JNN5UljNwShKS2wATgWGA4cLKk4fWK/V/gZxGxFzAe+H52bE/gcuBA4ADgckk98orVzKzZOfZYeOih9P3889P8Tk0szxbEAcDciJgXEe8Dk4Ax9coMBx7Nvk8p2H8M8HBELI6IJcDDwOgcYzUza35GjUpTdABceWWTXz7PBNEXmF+wXpttKzQd+Gz2/TNAV0m9SjwWSWMl1UiqWVhhsySamZXk7LPT8sc/bvJLl3uQ+t+BwyQ9BxwGLADWlXpwRNwWEdURUd2nT5+8YjQzKx8JTsuGYS+7rEkvnWeCWAD0L1jvl237QES8HhGfjYh9gO9m25aWcqyZWatRNwXHlVc26Vvq8kwQU4EhkgZJ6gCcBEwuLCCpt6S6GC4GJmbfHwJGSeqRDU6PyraZmbU+nTvDd7+bvs+c2WSXzS1BRMRaYBzph302cG9EzJQ0XtIJWbHDgTmSXgB2BK7Ojl0MXElKMlOB8dk2M7PW6bDD0vKOO5rskooKf6l2nerq6qipqSl3GGZm+Vi9GqqqoHfvRn11qaRnIqK62L5yD1KbmVkpOnaEvn3h7bebbBzCCcLMrFKccUZaXnddk1zOCcLMrFJceGFaPtQ09+w4QZiZVYpu3WDHHeHPf26SyzlBmJlVkiFDYOVKWL8+90s5QZiZVZJDD03LJmhFOEGYmVWSz30uLZ9+OvdLOUGYmVWSPfdMyyeeyP1SThBmZpWkQwc4+mj4xz9yv5QThJlZpdl1V5g7N/fLOEGYmVWa995Ly0WLcr2ME4SZWaU57ri0fPbZXC/jBGFmVmlGjEjLRx7J9TJOEGZmlWbYsLSsqsr1Mk4QZmaVpn172G67NLNrjpwgzMwq0Y47epDazMyK6NQJpkzJ9RJOEGZmlejtt2GnnXK9REkJQtIhkh6W9IKkeZJeljSvhONGS5ojaa6ki4rsHyBpiqTnJD0v6bhs+0BJKyVNyz4/3vKqmZm1YAcfDMuX53qJdiWWux24AHgGWFfKAZLaAhOAo4FaYKqkyRExq6DYJcC9EXGLpOHAA8DAbN9LETGyxPjMzFqX9u3hxRdzvUSpCeKdiHhwC899ADA3IuYBSJoEjAEKE0QA3bLv3YHXt/AaZmatU93T1GvWpGSRg1LHIKZI+qGkgyTtW/dp4Ji+wPyC9dpsW6ErgFMk1ZJaD+cX7BuUdT09LunQEuM0M2sdjjoqLVesyO0SpbYgDsyW1QXbAjhiG69/MnBHRFwn6SDgLkl7Am8AAyJikaT9gPsl7RER7xYeLGksMBZgwIAB2xiKmVkF6dw5LRcvhu7dc7lESQkiIj6xFedeAPQvWO+XbSt0JjA6u8ZTkqqA3hHxFrA62/6MpJeA3YGaenHdBtwGUF1dHVsRo5lZZerQIS1nzYJBg3K5RKl3MXWXdL2kmuxznaSGUtZUYIikQZI6ACcBk+uVeQ04MrvGMKAKWCipTzbIjaRdgSFAg3dNmZm1GnvvnZarV+d2iVLHICYC7wFfyD7vAj/d3AERsRYYBzwEzCbdrTRT0nhJJ2TFvgWcJWk6cA9wekQE8HHgeUnTgF8B50TE4i2qmZlZS9azZ1o+uKX3D5Wu1DGIwRFxYsH6f2Q/3psVEQ+QBp8Lt11W8H0WcEiR4+4D7isxNjOz1mfgwLSsG4vIQaktiJWS/qVuRdIhwMp8QjIzswZJ0LVrGqTOSaktiHOBO7NxBwGLgdPzCsrMzEqwdi28/HJupy/1LqZpwN6SumXr727+CDMzy12vXrD99rmdfrMJQtIpEfFzSf9WbzsAEXF9bpGZmdnm9e4NTz6Z2+kbakFsly275haBmZltnZUrYdddczv9ZhNERNyaLf8jtwjMzGzrfPSj8OqruZ2+1AflrpXUTVJ7SY9IWijplNyiMjOzhnXokCbry0mpt7mOygamjwdeAXYDLswrKDMzK0G7dmmqjZyUmiDquqI+CfwyIt7JKR4zMyvVgvrT2zWuUhPEbyX9A9gPeERSH2BVfmGZmVmDDjoIqqpyO31JCSIiLgIOBqojYg2wnPTyHzMzK5eOHWHVKoh8JrNu6DmIIyLiUUmfLdhWWOTXuURlZmYNq5tmY/ly6NKl0U/f0HMQhwGPAp8qsi9wgjAzK58hQ9Jy1aqmTxARcXm2PKPRr2xmZtumU6e0zOmdEKU+B/E9SdsXrPeQdFUuEZmZWWk6dkzLd/OZHq/Uu5iOjYildSsRsQQ4LpeIzMysNGvXpuXSpbmcvtQE0VZSx7oVSZ2Ajpspb2Zmeat7F/XKfF7PU+r7IO4mPf9Q95rRM4A7c4nIzMy2zNtv53LaUt8H8YPsvdFHZZuujIiHconIzMxK0717Wub0sFypXUwAs4HfR8S/A09KanAKcEmjJc2RNFfSRUX2D5A0RdJzkp6XdFzBvouz4+ZIOmYL4jQzax3atk3LdetyOX2pdzGdBfwKuDXb1Be4v4Fj2gITgGOB4cDJkobXK3YJcG9E7AOcBNycHTs8W98DGA3cnJ3PzMzqNIcEAZwHHAK8CxARLwI7NHDMAcDciJgXEe8Dk9h4eo4AumXfuwOvZ9/HAJMiYnVEvAzMzc5nZmZ1mkmCWJ39yAMgqR3px31z+gLzC9Zrs22FrgBOkVQLPACcvwXHImmspBpJNQsXLiylHmZmLUczSRCPS/oO0EnS0cAvgd80wvVPBu6IiH6k5yruklTyuEhE3BYR1RFR3adPn0YIx8ysgrTJfi7Xr8/n9CWW+zawEPg7cDbpX/uXNHDMAqB/wXq/bFuhM4F7ASLiKaAK6F3isWZmrVvOLYgGb3PNBodnRsRQ4CdbcO6pwBBJg0g/7icBX6pX5jXgSOAOScNICWIhMBn4haTrgZ2BIcDftuDaZmYtX12CeOWVXE7fYIKIiHXZraYDIuK1Uk8cEWsljQMeAtoCEyNipqTxQE1ETAa+BfxE0gWkMY3TIyKAmZLuBWYBa4HzIiKfFGlmVqnqJuvr1SuX05f6JHUP0o/230gvCwIgIk7Y3EER8QCpO6pw22UF32eR7o4qduzVwNUlxmdm1vq0y37Cy9XFlLk0l6ubmdnWK+cYhKQq4BxgN9IA9e0RsTaXSMzMbMuU+TbXO4FqUnI4FrgulyjMzGzL1SWInG5zbaiLaXhEjACQdDu+k8jMrPmoew6iTC2INXVf3LVkZtbMlPk5iL0l1b3LTqQnqd/NvkdEdNv0oWZmlqu6BJHTK0c3myAiwjOompk1V1Ja/vOfuZx+S94HYWZmzdGOO+ZyWicIM7NKtt12EA1Nrr11nCDMzCqZVPbZXM3MrDmS3IIwM7Mi2rRxgjAzsyLcxWRmZkW5i8nMzIpygjAzs6I8BmFmZkV5DMLMzIpyF5OZmRVVqV1MkkZLmiNprqSLiuz/kaRp2ecFSUsL9q0r2Dc5zzjNzCpWjl1Mpb6TeotJagtMAI4GaoGpkiZHxKy6MhFxQUH584F9Ck6xMiJG5hWfmVmLUKFdTAcAcyNiXkS8D0wCxmym/MnAPTnGY2bW8lRoF1NfYH7Bem22bSOSdgEGAY8WbK6SVCPpaUmf3sRxY7MyNQsXLmyksM3MKkgruIvpJOBXEVH43rxdIqIa+BJwg6TB9Q+KiNsiojoiqvv06dNUsZqZNR8rVsCbb+Zy6jwTxAKgf8F6v2xbMSdRr3spIhZky3nAY3x4fMLMzAAWLYLVq3M5dZ4JYiowRNIgSR1ISWCju5EkDQV6AE8VbOshqWP2vTdwCDCr/rFmZq1e//7Qs2cup87tLqaIWCtpHPAQ0BaYGBEzJY0HaiKiLlmcBEyK+NAoyzDgVknrSUnsmsK7n8zMLLPddmmgOge5JQiAiHgAeKDetsvqrV9R5Li/ACPyjM3MrEWQcjt1cxmkNjOzZsYJwsys0lXgcxBmZpY3dzGZmdkmuQVhZmYbcQvCzMyamhOEmVmlcxeTmZltxF1MZma2SW5BmJnZRtyCMDOzpuYEYWZW6dzFZGZmG3EXk5mZbZJbEGZmthG3IMzMrKk5QZiZVTp3MZmZ2UbcxWRmZptUiS0ISaMlzZE0V9JFRfb/SNK07POCpKUF+06T9GL2OS3POM3MKlaOLYh2eZ1YUltgAnA0UAtMlTQ5ImbVlYmICwrKnw/sk33vCVwOVAMBPJMduySveM3MKlYFtiAOAOZGxLyIeB+YBIzZTPmTgXuy78cAD0fE4iwpPAyMzjFWM7PKVKFjEH2B+QXrtdm2jUjaBRgEPLolx0oaK6lGUs3ChQsbJWgzM0uayyD1ScCvImLdlhwUEbdFRHVEVPfp0yen0MzMmrkK7GJaAPQvWO+XbSvmJDZ0L23psWZmrVeFdjFNBYZIGiSpAykJTK5fSNJQoAfwVMHmh4BRknpI6gGMyraZmVl9ObUgcruLKSLWShpH+mFvC0yMiJmSxgM1EVGXLE4CJkVsqGFELJZ0JSnJAIyPiMV5xWpmVrEq8TZXgIh4AHig3rbL6q1fsYljJwITcwvOzMw2q7kMUpuZ2daqwEFqMzPLW4UOUpuZWVNwC8LMzDbiFoSZmTU1Jwgzs0rnLiYzM9uIu5jMzGyT3IIwM7ONuAVhZmZNzQnCzKzSuYvJzMw24i4mMzPbJLcgzMxsI25BmJnZJrkFYWZmG3ELwszMmpoThJlZpavELiZJoyXNkTRX0kWbKPMFSbMkzZT0i4Lt6yRNyz6Tix1rZtbqVeI7qSW1BSYARwO1wFRJkyNiVkGZIcDFwCERsUTSDgWnWBkRI/OKz8ysxajAFsQBwNyImBcR7wOTgDH1ypwFTIiIJQAR8VaO8ZiZtTwVOkjdF5hfsF6bbSu0O7C7pD9LelrS6IJ9VZJqsu2fzjFOMzMrIrcupi24/hDgcKAf8ISkERGxFNglIhZI2hV4VNLfI+KlwoMljQXGAgwYMKBJAzczazYqsItpAdC/YL1ftq1QLTA5ItZExMvAC6SEQUQsyJbzgMeAfepfICJui4jqiKju06dP49fAzKy5k2DdulxOnWeCmAoMkTRIUgfgJKD+3Uj3k1oPSOpN6nKaJ6mHpI4F2w8BZmFmZh+2fDk8+WQup86tiyki1koaBzwEtAUmRsRMSeOBmoiYnO0bJWkWsA64MCIWSToYuFXSelISu6bw7iczM8tccgnMmJG6mRp5wFqRU99VU6uuro6amppyh2FmVlEkPRMR1cX2+UlqMzMrygnCzMyKcoIwM7OinCDMzKwoJwgzMyvKCcLMzIpygjAzs6KcIMzMrKgW86CcpIXAq9twit7A240UTqVobXVubfUF17m12JY67xIRRSezazEJYltJqtnU04QtVWurc2urL7jOrUVedXYXk5mZFeUEYWZmRTlBbHBbuQMog9ZW59ZWX3CdW4tc6uwxCDMzK8otCDMzK8oJwszMimpVCULSaElzJM2VdFGR/R0l/U+2/6+SBpYhzEZVQp3/TdIsSc9LekTSLuWIszE1VOeCcidKCkkVf0tkKXWW9IXs73qmpF80dYyNrYT/tgdImiLpuey/7+PKEWdjkTRR0luSZmxivyTdmP15PC9p322+aES0ig/ptacvAbsCHYDpwPB6Zb4G/Dj7fhLwP+WOuwnq/Amgc/b93NZQ56xcV+AJ4GmgutxxN8Hf8xDgOaBHtr5DueNugjrfBpybfR8OvFLuuLexzh8H9gVmbGL/ccCDgICPAX/d1mu2phbEAcDciJgXEe8Dk4Ax9cqMAe7Mvv8KOFJq5Je8Nq0G6xwRUyJiRbb6NNCviWNsbKX8PQNcCfwAWNWUweWklDqfBUyIiCUAEfFWE8fY2EqpcwDdsu/dgdebML5GFxFPAIs3U2QM8LNInga2l7TTtlyzNSWIvsD8gvXabFvRMhGxFngH6NUk0eWjlDoXOpP0L5BK1mCds6Z3/4j4XVMGlqNS/p53B3aX9GdJT0sa3WTR5aOUOl8BnCKpFngAOL9pQiubLf3/vUHttikcazEknQJUA4eVO5Y8SWoDXA+cXuZQmlo7UjfT4aRW4hOSRkTE0nIGlbOTgTsi4jpJBwF3SdozItaXO7BK0ZpaEAuA/gXr/bJtRctIakdqli5qkujyUUqdkXQU8F3ghIhY3USx5aWhOncF9gQek/QKqa92coUPVJfy91wLTI6INRHxMvACKWFUqlLqfCZwL0BEPAVUkSa1a6lK+v99S7SmBDEVGCJpkKQOpEHoyfXKTAZOy75/Dng0stGfCtVgnSXtA9xKSg6V3i8NDdQ5It6JiN4RMTAiBpLGXU6IiJryhNsoSvlv+35S6wFJvUldTvOaMMbGVkqdXwOOBJA0jJQgFjZplE1rMnBqdjfTx4B3IuKNbTlhq+liioi1ksYBD5HugJgYETMljQdqImIycDupGTqXNBh0Uvki3nYl1vmHQBfgl9l4/GsRcULZgt5GJda5RSmxzg8BoyTNAtYBF0ZExbaOS6zzt4CfSLqANGB9eiX/g0/SPaQk3zsbV7kcaA8QET8mjbMcB8wFVgBnbPM1K/jPy8zMctSaupjMzGwLOEGYmVlRThBmZlaUE4SZmRXlBGFmZkU5QZhtAUnrJE2TNEPSbyRt38jnfyV7TgFJyxrz3GZbygnCbMusjIiREbEn6VmZ88odkFlenCDMtt5TZJOhSRos6feSnpH0pKSh2fYdJf0/SdOzz8HZ9vuzsjMljS1jHcw2qdU8SW3WmCS1JU3jcHu26TbgnIh4UdKBwM3AEcCNwOMR8ZnsmC5Z+a9ExGJJnYCpku6r5CebrWVygjDbMp0kTSO1HGYDD0vqAhzMhulKADpmyyOAUwEiYh1pCnmAr0v6TPa9P2niPCcIa1acIMy2zMqIGCmpM2keoPOAO4ClETGylBNIOhw4CjgoIlZIeow0kZxZs+IxCLOtkL2F7+ukCeFWAC9L+jx88G7gvbOij5Be5YqktpK6k6aRX5Ilh6GkKcfNmh0nCLOtFBHPAc+TXkzzZeBMSdOBmWx4/eU3gE9I+jvwDOndyL8H2kmaDVxDmnLcrNnxbK5mZlaUWxBmZlaUE4SZmRXlBGFmZkU5QZiZWVFOEGZmVpQThJmZFeUEYWZmRf1/EucGOjw+9CMAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis AUPR score: 0.9583349634019925\n"
     ]
    }
   ],
   "source": [
    "precision, recall, aupr_score = aupr(test_y, ood_scores)\n",
    "\n",
    "plot_precision_recall_curve(precision, recall)\n",
    "print('Mahalanobis AUPR score:', aupr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mahalanobis fpr95 score: 0.0894\n"
     ]
    }
   ],
   "source": [
    "fpr95 = fpr_at_95_tpr(test_y, ood_scores)\n",
    "print('Mahalanobis fpr95 score:', fpr95)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OOD detection by SML (Standardized Max Logits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using cache found in /home/simc/.cache/torch/hub/facebookresearch_deit_main\n",
      "100%|██████████| 98/98 [00:25<00:00,  3.80it/s]\n"
     ]
    }
   ],
   "source": [
    "from ood_with_vit.metrics.sml import SML\n",
    "\n",
    "model = initialize_vit_model(config)\n",
    "sml = SML(config, model, id_train_dataloader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing in-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:06<00:00, 15.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing out-of-distribution samples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 261/261 [00:13<00:00, 19.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id mean score: 0.1767480533869523 id variance: 1.4942061282549084\n",
      "ood mean score: 5.2902633455482615 ood variance: 1.8708931763079546\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "test_y, ood_scores, id_ood_scores, ood_ood_scores = compute_ood_scores(\n",
    "    metric=sml,\n",
    "    in_dist_dataloader=id_test_dataloader,\n",
    "    out_of_dist_dataloader=ood_test_dataloader,    \n",
    ")\n",
    "print('id mean score:', np.mean(id_ood_scores), 'id variance:', np.var(id_ood_scores))\n",
    "print('ood mean score:', np.mean(ood_ood_scores), 'ood variance:', np.var(ood_ood_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA7CUlEQVR4nO3dd5gUVdbA4d9hhpyTq5KGDEMQdcCAJFGSgPq5KorusjvKAiK4pkUxsixrwoASFcWIrrjqKCi6KmYlZySIZBAYcmZmzvfHrdZ2nNADU1PT0+d9nn6qu+t21anq7jp1b1XdElXFGGNM7CoWdADGGGOCZYnAGGNinCUCY4yJcZYIjDEmxlkiMMaYGGeJwBhjYpwlgiJORJaJSMeg4ygsRORuEXkuoHlPEZGRQcw7v4lIXxH56AQ/a7/JQsYSQQESkXUiclhEDojINm/DUM7PeapqM1Wd5ec8QkSkpIj8W0Q2eMu5WkTuEBEpiPlnEU9HEdkU/p6qjlLVG3yan4jIEBFZKiIHRWSTiLwpIi38mN+JEpEHROSVk5mGqr6qql0imNfvkt+J/iZFpIQX+2pv/a4TkedFJCGv0zK/ZYmg4PVS1XJAK+BM4K5gw8k7EYnPZtSbQGegB1AeuB7oDzzlQwwiIoXt9/sUMBQYAlQBGgHvAJfk94xy+A58F+C8pwG9gWuBisAZwDzcby5Pglx/hZKq2qOAHsA64KKw148A08Nenwt8A+wBFgEdw8ZVAV4AtgC7gXfCxvUEFnqf+wZomXmewOnAYaBK2LgzgZ1Ace/1X4EV3vRnAnXCyipwE7Aa+CmLZesMHAFqZXr/HCAdaOC9ngX8G5gN7APezRRTTutgFvAv4GtvWRoAf/Fi3g+sBf7mlS3rlckADniP04EHgFe8Mgnecv0Z2OCti+Fh8ysNvOitjxXAncCmbL7bht5ytsnh+58CjAWme/F+D9QPG/8UsNFbL/OAdmHjHsBtCF/xxt8AtAG+9dbVVuAZoETYZ5oBHwO7gJ+Bu4FuwDHguLdOFnllKwKTvelsBkYCcd64ft46fwJI9cb1A77yxos3brsX2xKgOW4n4Lg3vwPAe5n/B0CcF9eP3jqZR6bfkFfuIu/7/N24HP5fWX3Xyd53/QXwATA40zQWAf/nPW8Stv5WAlcFvQ3xbdsUdACx9Mj0B6jp/WGe8l7X8P5kPXA1tYu919W98dOBN4DKQHGgg/f+md4f8BzvT/Vnbz4ls5jnp8CNYfE8Ckzwnl8KrAGaAvHAPcA3YWXV+1NUAUpnsWwPAZ9ns9zr+XUDPcvb0DTHbazfCvuz5rYOZnl/4mZejMVxe9v1cRujDsAh4CyvfEcybbiz2Tg8i9vonwEcBZqGL5O3zmsCizNPL2y6A4D1uXz/U7zlaePF/yrwetj464Cq3rjbgG1AqbC4jwOXeeumNHA2LnHGe8uyArjFK18et1G/DSjlvT4n8zoIm/fbwETvOzkFl6hD31k/IA242ZtXaX6bCLriNuCVvO+hKXBa2DKPzOF/cAfuf9DY++wZQNW8/L6ymm4O3/VL3jKWBv4EfB1WPhGXVEt6ZTbidjTi+XWnKTHo7Ygfj8JWtY4F74jIftyPbDtwv/f+dcAMVZ2hqhmq+jEwF+ghIqcB3YEBqrpbVY+r6ufe5/oDE1X1e1VNV9UXcRuzc7OY92vANeCaVoA+3nvgNmT/VtUVqpoGjAJaiUidsM//W1V3qerhLKZdDbfhycpWb3zIy6q6VFUPAvcCV4lIXE7rIOyzU1R1maqmeethuqr+qM7nwEdAu2ziyM6DqnpYVRfh9gjP8N6/ChjlrfNNwJgcplE1h+UP97aqzvbW8au4JkIAVPUVVU31lm00boPUOOyz36rqO966Oayq81T1O6/8OtyGvINXtiewTVVHq+oRVd2vqt9nFZCI/AG3jm9R1YOquh23h98nrNgWVX3am1fm7/84LtE0AcT7DUWyLsDVbO5R1ZXed7hIVVOzKBfp+s3NA94yHsYlv/DfeF/gv6p6FLf+1qnqC94yL8DttFyZDzEUOpYICt5lqloet7fahF83kHWAK0VkT+gBXACcBtQCdqnq7iymVwe4LdPnauGaQTJ7CzjPSyztcc0mX4ZN56mwaezC7aHVCPv8xhyWa6cXa1ZO88ZnNZ31uD37auS8DrKMQUS6i8h3IrLLK9+D3yadSGwLe34ICB3APz3T/HJa/lSyX/5I5oWI3C4iK0Rkr7csFfntsmRe9kYi8r534sE+XPIOla+Fa26JRB3cd7A1bL1PxNUMspx3OFX9FNcsNRbYLiKTRKRChPOONM5I129uflkOVd2Pq2mHEt41uOQMbp2ck+m32Bc4NR9iKHQsEQTE23udAjzmvbURt6dcKexRVlUf8sZVEZFKWUxqI/CvTJ8ro6pTs5jnbtwe89W4A26vq6qGTedvmaZTWlW/CZ9EDov0P9wfp1b4myJyDu7P/mnY2+FlauP2KHfmsg5+F4OIlMQlt8eAP6hqJWAGLoHlFm8ktuKahLKKO7NPgJoiknQiMxKRdrhjEFcBlb1l2cuvywK/X57xwA9AQ1WtgGtrD5XfCNTLZnaZp7MRV4usFrbeK6hqsxw+89sJqo5R1bNxzSuNcE0+uX7Om3f9XMqA+321EZGaOZQ5CJQJe53VRjtzPFOBa0TkPFwT2mdhcX2e6bdYTlUHRhBr1LFEEKwngYtF5AzcQcBeItJVROJEpJR3+mNNr5r9ATBORCqLSHERae9N41lggIic451JU1ZELhGR8tnM8zVc2+gf+bVZCGACcJeINAMQkYoiEnE1WFX/h9sYviUizbxlONdbrvGqujqs+HUikigiZYARwDRVTc9pHWQz2xK45pMdQJqIdAfCT2n8GagqIhUjXY5M/oNbJ5VFpAYwOLuC3vKNA6Z6MZfw4u8jIsMimFd5XDv8DiBeRO4DcturLo87OHtARJoA4Rup94HTROQWcaf1lveSMrj1khA668r7fX0EjBaRCiJSTETqi0gHIiAirb3fX3HcxvgIrrYZmld2CQngOeCfItLQ+/22FJGqmQt5v6+PgbdF5GwRifeWaYCI/NUrthDo4/0/knC/8dzMwO39jwDeUNVQ3O8DjUTkem96xb3lbBrBNKOOJYIAqeoO3MGr+1R1I+6A7d24jcFG3F5V6Du6Hrfn/APu2MIt3jTmAjfiqua7cQd8++Uw2xTcGS7bvDbxUCxvAw8Dr3vNDEtxxyXy4grcHtWHuLNEXsGdiXJzpnIv42pD23B7YUO8GHJbB7/hVe2H4DbYu3G1nJSw8T/g9vjWetX7rJrLcjIC2AT8hNsjnYbbc87OEH5tItmDa/K4HHgvgnnNxK23VbjmsiPk3BQFcDtumffjdgjeCI3w1s3FQC/cel4NdPJGv+kNU0Vkvvf8T7jEuhy3LqcReVNMBW/+u73YU3EnIoD7/hO99f9OFp99HPf9fYRLapNxB3Kz8kfchvsNXG1pKZCE+27AHW+q78XxIL/d0cmSdzzgv7izkl4Le38/bqeiD+5MvW24/0fJ3KYZjeTXlgFj/Ccis3BncgRyde/JEJGBQB9VjWhP2ZhoYTUCY7IhIqeJSFuvqaQx7lTMt4OOy5j8ZlfXGZO9ErizZ+rimnpexx0HMKZIsaYhY4yJcdY0ZIwxMS7qmoaqVaumCQkJQYdhjDFRZd68eTtVtXpW46IuESQkJDB37tygwzDGmKgiIuuzG2dNQ8YYE+MsERhjTIyzRGCMMTHOEoExxsQ4SwTGGBPjfEsE4m4qvV1ElmYzXkRkjIisEZHFInKWX7EYY4zJnp81gim4+6NmpzuuF8yGuLtsjfcxFmOMMdnw7ToCVf1CRBJyKHIp8JJ3Y5TvRKSSiJyWh1vcmYKk6h5Hj0J6untkZGQ9PH7clQt9LiMjsmFqKpQs6Z6HzzP0PPMw0vf8mNbPP0Plyr9dP5nXVyTj8lI2v6YTzfPcvh3KloUSJYglBznGDg6R0Ot6aN0636cf5AVlNfhtf+ubvPd+lwhEpD+u1kDt2rULJLhC68gR2LULVq2CLVvgp5/cn+LIEfc6IwOKFYNjx2DDBjeuWDG3cU5Lc49Vq+CUU9zGet8+97mKFUHk1w16+MY99DCmMBHJvUwR8GmCcmMvqHgU5s5tQrEilggipqqTgEkASUlJRaeXvMOHYccO2LkTNm6E9ethxQrYutVtePftg927YdkyKFfOlT92LPfpVqkCZcpA8eJuD6phQ5cQ4uPde02bumTSsqXbAz9yxI2vWtUljbg4N8zq+c6dULv2b8dlNdy7F0491b0WyX0o4pJVuXIultCfPDQu9DzzMNL38ntaqr8m2ZDMG6bw1zmNy0vZ/JpONM8z9Jsp4vYc2cMdH93Bcwueo0GVBjzR6zmKJfhzK4wgE8FmfnsP2Jree0XD0aOwciXMmeM2ij/+CF9/Ddu2uQ3y5lwWtUYNqFMHEhLcBnvvXrcBL1sWqld300hMdBvlihXdhj8+KvK6MSYX6RnpnD/5fFamruTO8+/kgY4PULp4djduO3lBbjlSgMEi8jpwDrA3ao8PqLoN/tixsH8/LF3qNvzhzSnlyrmN9umnQ4UK0LOn23hXqQKNG0PNmm6jX7VqzLV/GmOc1EOpVCldhbhicfzrwn9Rq2Itkk5P8n2+viUCEZkKdASqicgm4H6gOICqTsDde7QH7h67h4C/+BWLLw4ehPHjXQL49FPXZBLSsydcfbVrkqlb1+25V6kSE9VZY0zeqSqvLnmVoR8O5aHOD3Hj2TdyedPLC2z+fp41dE0u4xW4ya/5++bbb2HYMNfMk54OpUpBt27u0akTNGoUdITGmCiyce9GBkwfwIzVMzi35rm0rd22wGOwRuVIffwx3HQTrF7tXtesCePGQY8e7uCoMcbk0dQlU/nb+38jXdN5suuTDG4zmLhiBb89sUSQmx9/hBtugFmzoHRpuPde6NvXtesbY8xJqFy6MufUPIdJPSdRt3LdwOKwRJCTMWPg9tvdufdXXAGTJ7szdIwx5gSkZaTxxLdPcCz9GMPbD6dbg250rd8VCfj4oSWC7EyYAEOHurN9Fi1yp24aY8wJWrRtEckpyczbOo+rml2FqiIigScBsN5Hs/bllzBwINSv75qGLAkYY07Q0bSj3PvpvSQ9m8TGfRt588o3ef2K1wtFAgixGkFmBw/Ctde6qxenT3ddMRhjzAlavWs1D3/9MNe2uJbHuzxO1TJVgw7pdywRZDZyJGzaBCkpdkDYGHNCDhw7wLs/vEvfln1pfkpzfhj8A/Uq1ws6rGxZ01C4gwfhySfhoougV6+gozHGRKGPf/yYFuNbcP3b17NixwqAQp0EwBLBb02d6jpgGzgw6EiMMVFm9+HdJL+bTJdXulAirgSf9/ucptWj4/iiNQ2FHD8ODz3keue02oAxJg/SM9Jp+3xbVqWu4q4L7uK+DvdRKr5U0GFFzBJByFNPuTOEJkxwycAYY3Kx89DOXzqJG9V5FLUr1uas06LvrrvWNBTy5ptu+Ne/BhuHMabQU1VeWvQSjZ5uxHPznwPgsiaXRWUSAKsROAcOwOzZ7hZwVhswxuRg/Z71/O39vzHzx5mcX+t82tdpH3RIJ80SAcBbb7mh1QaMMTl4ZfErDJw+EFXl6e5PM6j1IIpJ9DesWCIAd1+BYsXguuuCjsQYU4hVL1OdtrXaMrHnROpUqhN0OPnGEsGuXfD99zBggOtXyBhjPMfTjzP629EcTz/OvR3upWuDrnSp36VQdQ+RHywR/PijG7ZpE2wcxphCZcHWBSSnJLNg2wL6NO9TqDqJy2/R37h1slJS3LB162DjMMYUCkfSjnD3J3fT+tnWbNm/hbeueoupV0wtkgkgxGoEI0e6m8Y3bx50JMaYQmDNrjU89s1j/OmMPzG6y2gql64cdEi+i+1EcPCgGyYkBBqGMSZYB44d4O0Vb3P9GdfT/JTmrBy8MtA7hhW02G4aWr/eDS+7LNAwjDHBmblmJs3GNePP7/z5l07iYikJQKwngunT3bBhw2DjMMYUuNRDqfz5nT/T7dVulClehi//8mXUdBKX32K7aWjJEjfs1i3YOIwxBSrUSdyaXWsY3m4497S/J6o6ictvsZ0IvvkGunZ1F5MZY4q8HQd3ULVMVeKKxfHwRQ9Tp1IdWp3aKuiwAhe7W8Bjx9w1BImJQUdijPGZqvLCghdo9Ewjnp33LACXNrnUkoAndmsEc+a4YfXqwcZhjPHVuj3r6P9efz5e+zHtarejU91OQYdU6MRuIli82A2vuCLYOIwxvnl50csMnD4QEWFcj3H8LelvRaKTuPwWu4ngq6/csEGDYOMwxvjmD+X+QPs67ZnQcwK1K9YOOpxCK3YTwfr1EB9vB4qNKUKOpx/nka8fIV3Tua/DfXSp34Uu9bsEHVahF7tbwa1b4fTTg47CGJNP5m+dT+tnW3PPZ/ewMnUlqhp0SFEjdhNBaqolAmOKgMPHDzPsf8No82wbfj74M29f/Tav/t+rRbqTuPzmayIQkW4islJE1ojIsCzG1xaRz0RkgYgsFpEefsbzC1XYu9euKDamCFi7ey2Pf/s4/Vr1Y/mg5VzW5LKgQ4o6viUCEYkDxgLdgUTgGhHJfNL+PcB/VPVMoA8wzq94fuOnn9ywVq0CmZ0xJn/tO7qPKQunANDslGasvnk1z/V+LiZ6CvWDnzWCNsAaVV2rqseA14FLM5VRoIL3vCKwxcd4frV1qxs2a1YgszPG5J8Zq2fQfFxzklOSf+kkrijdNjIIfiaCGsDGsNebvPfCPQBcJyKbgBnAzVlNSET6i8hcEZm7Y8eOk48s1MdQkyYnPy1jTIHYeWgn1799PZe8dgnlS5bn679+HbOdxOW3oA8WXwNMUdWaQA/gZZHfX+2hqpNUNUlVk6rnx5XAoRpBo0YnPy1jjO9CncS9vvR17mt/H/P7z+fcmucGHVaR4ed1BJuB8Eb4mt574ZKBbgCq+q2IlAKqAdt9jAs2ehUVu1m9MYXazwd+pnrZ6sQVi+Oxix+jTqU6tPxDy6DDKnL8rBHMARqKSF0RKYE7GJySqcwGoDOAiDQFSgH50PaTi7Q032dhjDlxqsrk+ZNp/ExjJs2bBECvxr0sCfjEtxqBqqaJyGBgJhAHPK+qy0RkBDBXVVOA24BnReTvuAPH/bQgrgLZtw8aN/Z9NsaYvFu7ey03vncjn/70KR3qdOCiehcFHVKR52sXE6o6A3cQOPy9+8KeLwfa+hlDlrZvh1NPLfDZGmNy9uLCFxk0YxBxEseESyZw49k3WidxBSA2+xpatgw6WVe0xhQ2p5c/nQvrXsj4S8ZTs0LNoMOJGbGZCMqVg927g47CmJh3LP0YD331EBmawQMdH+Di+hdzcf2Lgw4r5sRmnevYMbuYzJiAzdk8h7Mnnc39s+5n7e611klcgGIzEezcCaVi90bVxgTp0PFD3P7R7Zw7+Vx2H95NSp8UXrr8JeskLkCx1zR09KgbHjkSbBzGxKifdv/E07Of5sazbuThix6mYqmKQYcU82IvEaxf74a17W5FxhSUvUf28t8V/+UvZ/6FZqc0Y83Na6hV0Tp9LCxir2kodFWxJQJjCsT0VdNpNq4ZN7x3Az/s/AHAkkAhE3uJ4PhxN6xbN9g4jCnidhzcQd//9qXn1J5ULl2Zb5O/pUk16+ixMIq9pqFQjcAOFhvjm/SMdC544QJ+2v0TD3Z8kGEXDKNEXImgwzLZiL1EUML7MZYtG2wcxhRB2w5s45SypxBXLI7RXUaTUCmB5qc0Dzosk4uIm4ZEpIyfgRSYUNNQ6dLBxmFMEZKhGUycO5FGTzdi4tyJAPRs1NOSQJTINRGIyPkishz4wXt9hogUzC0l/RBKBMWLBxuHMUXEml1r6PxSZwZMH0DrGq3p2qBr0CGZPIqkaegJoCteF9KqukhE2vsalZ8sERiTb15Y8AKDZgyiRFwJnu31LMlnJtuFYVEoomMEqrox05eb7k84BWDtWjcsYQeujDlZtSvWpmv9roztMZYaFTLfidZEi0gSwUYROR9QESkODAVW+BuWjyp6VzHa3cmMybOjaUf591f/JkMzGNFpBJ3rdaZzvc5Bh2VOUiQHiwcAN+FuPL8ZaAUM8jEmfx05AiVLglVfjcmT7zd9z9mTzubBzx9kw94N1klcERJJjaCxqvYNf0NE2gJf+xOSz374AeLigo7CmKhx8NhB7v3sXp787klqVKjB+9e8zyWNLgk6LJOPIqkRPB3he9GhUiU4dCjoKIyJGuv3rmfcnHEMSBrAskHLLAkUQdnWCETkPOB8oLqI3Bo2qgLuHsTR6cABaGKXuRuTkz1H9jBt+TRuOOsGEqsnsmbIGrtjWBGWU9NQCaCcV6Z82Pv7gD/6GZSvdu60A8XG5ODdH95l4PSBbD+4nQtqX0CTak0sCRRx2SYCVf0c+FxEpqjq+gKMyV+HD0O1akFHYUyhs/3gdoZ8MIQ3lr1Byz+0JOWaFOskLkZEcrD4kIg8CjQDfumpTVUv9C0qP/38M9Sw852NCZeekU7b59uyYe8GRnYayZ1t76R4nF10GSsiSQSvAm8APXGnkv4Z2OFnUL7avdvVCowxbNm/hVPLnUpcsTie6vYUCZUSSKyeGHRYpoBFctZQVVWdDBxX1c9V9a9AdNYGAPbvt5vSmJiXoRmMnzOeJs80YcLcCQD0aNjDkkCMiqRG4HXOw1YRuQTYAlTxLyQfhfoZsovJTAxblbqKG9+7kS/Wf8FF9S6ie4PuQYdkAhZJIhgpIhWB23DXD1QAbvEzKN+Erh9o3DjYOIwJyOT5kxn8wWBKxZfi+d7P069VP+skzuSeCFT1fe/pXqAT/HJlcfTZssUN7dJ4E6MSKiXQvUF3xvYYy2nlTws6HFNI5HRBWRxwFa6PoQ9VdamI9ATuBkoDZxZMiPlo2zY3tBqBiRFH047yzy/+CcDIC0daJ3EmSznVCCYDtYDZwBgR2QIkAcNU9Z0CiC3/HT3qhtbXkIkB32z8huSUZH7Y+QN/bfVXVNWagUyWckoESUBLVc0QkVLANqC+qqYWTGg+SEtzQ7ugzBRhB44dYPgnw3l69tPUqliLD/t+aHcNMznK6fTRY6qaAaCqR4C1eU0CItJNRFaKyBoRGZZNmatEZLmILBOR1/Iy/Tyzu5OZGLBh7wYmzpvITa1vYunApZYETK5yqhE0EZHF3nMB6nuvBVBVbZnThL1jDGOBi4FNwBwRSVHV5WFlGgJ3AW1VdbeInHISy5K7UI0gPqIbsxkTNXYf3s2by9+k/9n9SayeyNqhazm9/OlBh2WiRE5bxKYnOe02wBpVXQsgIq8DlwLLw8rcCIxV1d0Aqrr9JOeZs59/dkOrEZgi5O0VbzNoxiB2HNxBhzodaFytsSUBkyc5dTp3sh3N1QA2hr3eBJyTqUwjABH5Gte19QOq+mHmCYlIf6A/QO2TuSo4dJDY7ldsioBtB7Zx8wc3M235NFqd2orp106ncTU7I87kXdBtJPFAQ6AjUBP4QkRaqOqe8EKqOgmYBJCUlHTiFwGkp7th+fI5lzOmkEvPSKfdC+3YuHcjoy4cxe3n326dxJkT5mci2Iw7/TSkpvdeuE3A96p6HPhJRFbhEsMcXyI6dswNrUZgotSmfZs4vfzpxBWLY0y3MdStXNe6ijYnLZJO5xCR0iKS1zrnHKChiNQVkRJAHyAlU5l3cLUBRKQarqlobR7nE7nt3iEIO0ZgokyGZvD090/T5JkmjJ8zHoDuDbtbEjD5ItdEICK9gIXAh97rViKSeYP+O6qaBgwGZgIrgP+o6jIRGSEivb1iM4FUEVkOfAbc4et1CqHTR0uW9G0WxuS3H3b+QPsX2jPkwyFcUPsCejbqGXRIpoiJpGnoAdwZQLMAVHWhiNSNZOKqOgOYkem9+8KeK3Cr9/Bf8eLugHGxiCpCxgTuufnPMXjGYMoUL8OLl73I9S2vt6uDTb6LqBtqVd2b6ccXnb22paVBqVK5lzOmkKhfuT69Gvfime7P8Idyfwg6HFNERZIIlonItUCcdwHYEOAbf8PySVqaXUxmCrUjaUcY8fkIAEZ1HkWnup3oVLdTwFGZoi6SNpKbcfcrPgq8huuO+hYfY/KPJQJTiH294WtaTWjFv7/6NzsO7kCtu3RTQCLZKjZR1eHAcL+D8d2xY3Z3MlPo7D+6n7s/uZuxc8ZSp1IdZl43ky71uwQdlokhkSSC0SJyKjANeENVl/ock39Wr/61vyFjColN+zbx3ILnuLnNzfyr878oV6Jc0CGZGJNr05CqdsLdmWwHMFFElojIPb5H5odTT7W7k5lCIfVQ6i/XAzSt3pS1Q9byVPenLAmYQER0HqWqblPVMcAA3DUF9+X8iUIqPR1Os9vzmeCoKtOWTyNxXCJDPhzCyp0rAey2kSZQkVxQ1lREHhCRJbib13+D6y4i+qSn293JTGC27t/KFf+5givfvJJaFWox98a51kmcKRQiOUbwPPAG0FVVt/gcj7/S0+1iMhOIUCdxm/dv5pGLHuHv5/2d+GJ2BpspHHL9JarqeQURSIGwGoEpYBv3bqRGhRrEFYtjbI+x1K1cl0ZVGwUdljG/ke3usYj8xxsuEZHFYY8lYXcuiy4ZGZYITIFIz0hnzPdjaDL2107iujboaknAFEo51QiGesOi08PVli3WBbXx3YodK0hOSebbTd/SvUF3ejXuFXRIxuQo2xqBqm71ng5S1fXhD2BQwYSXz0Rgx46gozBF2KR5k2g1sRWrUlfx8uUvM/3a6dSueBJ31TOmAERy5PTiLN7rnt+BFAhVaGxnaRj/NKzSkMubXM7ym5ZzXcvrrKdQExWybRoSkYG4Pf96mY4JlAe+9jswXyxdCgkJQUdhipDDxw/zwKwHEBEeuugh6yTORKWcjhG8BnwA/BsYFvb+flXd5WtUfqleHQ4fDjoKU0R8sf4Lbki5gdW7VjPg7AGoqtUATFTKqWlIVXUdcBOwP+yBiFTxPzQfqEJta681J2ff0X0Mmj6IDlM6kK7pfPKnTxjfc7wlARO1cqsR9ATm4W5EE/4rV6Cej3H5w64jMPlgy/4tTFk4hVvPvZURnUZQtkTZoEMy5qRkmwhUtac3jOi2lFHBEoE5QTsP7eQ/y/7DoNaDaFKtCT8N/cnuGGaKjEj6GmorImW959eJyOMiEp3tK5YITB6pKm8sfYPEsYnc8uEtrEpdBWBJwBQpkZw+Oh44JCJnALcBPwIv+xqVXywRmDzYsn8Ll71xGX3e6kOdSnWY13+eXRlsiqRIer1KU1UVkUuBZ1R1sogk+x2YLw4csERgIpKekU77F9qzef9mHrv4MYaeO9Q6iTNFViS/7P0ichdwPdBORIoBxf0NywehG9LYlcUmB+v3rKdmhZrEFYtj3CXjqFe5Hg2qNAg6LGN8FUnT0NW4G9f/VVW34e5F8KivUfkhI8MN7YIyk4X0jHQe//Zxmo5tyvi5rpO4LvW7WBIwMSGSW1VuA14FKopIT+CIqr7ke2T5LZQIikdfZcb4a+n2pZz//Pnc9tFtdK7XmcuaXBZ0SMYUqEjOGroKmA1cCVwFfC8if/Q7sHwXSgR2YxoTZsLcCZw18SzW7l7La//3Gil9UqhZITpvwGfMiYrkGMFwoLWqbgcQkerA/4BpfgaW79LT3dASgYFfuoNoWq0pVza7kie7Pkn1stWDDsuYQESSCIqFkoAnlQhvel+ohGoEdtZQTDt0/BD3fXYfcRLHwxc/TIeEDnRI6BB0WMYEKpIN+ociMlNE+olIP2A6MMPfsHxgTUMxb9a6WbQc35LR347mwLEDaOhMMmNiXCT3LL5DRP4PuMB7a5Kqvu1vWD6wRBCz9h7Zy50f38mk+ZOoX7k+n/7pU+sq2pgwOd2PoCHwGFAfWALcrqqbCyqwfGfHCGLW1gNbeWXJK9x+3u082OlByhQvE3RIxhQqOW0VnwfeB67A9UD6dF4nLiLdRGSliKwRkWE5lLtCRFREkvI6j4jZMYKYsuPgDp7+3v1km1Rrwrqh63i0y6OWBIzJQk5NQ+VV9Vnv+UoRmZ+XCYtIHDAWd6vLTcAcEUlR1eWZypUHhgLf52X6eXbkiBuGagamSFJVpi6dypAPhrDv6D66NuhKo6qN7IwgY3KQU42glIicKSJnichZQOlMr3PTBlijqmtV9RjwOnBpFuX+CTwMHMlz9Hlx/LgbWiIosjbu3Uivqb3o+9++NKjSgAV/W2CdxBkTgZxqBFuBx8Nebwt7rcCFuUy7BrAx7PUm4JzwAl5CqaWq00XkjuwmJCL9gf4AtU/0DmOhBHDKKSf2eVOopWWk0fHFjmw7sI0nuj7BzW1uJq6YNQMaE4mcbkzj62kVXud1jwP9ciurqpOASQBJSUknds5fWpob2jGCImXdnnXUqlCL+GLxTOw5kXqV61GvcvTdPM+YIPl5Cs1moFbY65reeyHlgebALBFZB5wLpPh2wDhUI7BEUCSkZaTx2DeP0XRsU8bNGQfARfUusiRgzAnws4P1OUBDEamLSwB9gGtDI1V1L1At9FpEZuFOUZ3rSzSWCIqMxT8vJjklmblb5nJp40u5IvGKoEMyJqr5lghUNU1EBgMzgTjgeVVdJiIjgLmqmuLXvLNkiaBIGDdnHEM/HErlUpV5449vcGXilYhI0GEZE9VyTQTi/mV9gXqqOsK7X/Gpqjo7t8+q6gwydUehqvdlU7ZjRBGfqNBZQ5YIolKok7jmpzSnT/M+PNH1CaqVqZb7B40xuYqkRjAOyMCdJTQC2A+8BbT2Ma78t3evGx4+HGwcJk8OHjvIPZ/eQ3yxeB7t8ijt67SnfZ32QYdlTJESycHic1T1Jrzz/FV1N1DC16j8UMILuZrtRUaLT9Z+QovxLXjy+yc5mn7UOokzxieR1AiOe1cJK/xyP4IMX6PyQ2gjYu3Jhd6eI3u4/aPbmbxgMg2rNOSLfl/Qrk67oMMypsiKpEYwBngbOEVE/gV8BYzyNSo/WCKIGj8f+JnXl77OP9r+g0UDFlkSMMZnkXRD/aqIzAM6AwJcpqorfI8sv1kiKNRCG/+h5w6lcbXGrLtlnR0MNqaARHLWUG3gEPBe+HuqusHPwPKdJYJCSVV5dcmrDP1wKAeOHaBHwx40rNrQkoAxBSiSYwTTcccHBCgF1AVWAs18jCv/WSIodDbs3cCA9wfwwZoPOK/meUzuPZmGVRsGHZYxMSeSpqEW4a+9juIG+RaRXywRFCppGWl0nNKR7Qe3M6bbGAa1HmSdxBkTkDxfWayq80XknNxLFjKWCAqFtbvXUqdiHeKLxfNsr2epX6U+CZUSgg7LmJgWyTGCW8NeFgPOArb4FpFfLBEEKi0jjdHfjOb+WffzyMWPMOScIXSu1znosIwxRFYjKB/2PA13zOAtf8LxkSWCwCzctpDklGTmb53P5U0u58rEK4MOyRgTJsdE4F1IVl5Vby+gePxjiSAQz8x+hr/P/DtVS1dl2pXTrKdQYwqhbBOBiMR7PYi2LciAfGOJoECFOolr+YeW9G3Rl8e7Pk6V0lWCDssYk4WcagSzcccDFopICvAmcDA0UlX/63Ns+csSQYE4cOwAwz8ZTvG44jzW5THrJM6YKBBJFxOlgFRc76M9gV7eMLpYIvDdRz9+RPNxzXl69tMcTz9uncQZEyVyqhGc4p0xtJRfLygLib5/uCUC3+w+vJtbP7qVKQun0LhqY774yxdcUPuCoMMyxkQop0QQB5TjtwkgxBKB+cX2g9uZtnwad11wF/d1uI9S8aWCDskYkwc5JYKtqjqiwCLxmyWCfLXtwDamLpnK38/7u+skbug6qpapGnRYxpgTkNMxgqK1xbREkC9UlRcXvkji2ETu+uQuVqeuBrAkYEwUyykRFK3LPi0RnLR1e9bR7dVu9Hu3H4nVE1k4YKF1EmdMEZBt05Cq7irIQHxnieCkpGWk0enFTuw8tJOxPcYyIGkAxSSSk86MMYVdnjudi1qWCE7Iml1rqFupLvHF4nm+9/PUq1yPOpXqBB2WMSYfxc4unSWCPDmefpxRX46i2bhmjJ0zFoBOdTtZEjCmCIqdGsGePW5oiSBX87fOJzklmYXbFnJl4pVc3ezqoEMyxvgodhJBMa/yY4kgR2O+H8OtM2+letnq/Peq/3J508uDDskY47PYSQTx3qKWLh1sHIVUqJO4M089kz+d8SdGdxlN5dKVgw7LGFMAYicRWL83Wdp/dD93fXIXJeNKMrrraNrVaUe7Ou2CDssYU4Bi52BxiDUN/eLDNR/SfHxzxs0Zh6LWSZwxMcpqBDEo9VAqt350Ky8teomm1Zry9V+/5rxa5wUdljEmILGTCEKsRkDq4VTeXvE297a/l+HthlMyvmTQIRljAuRr05CIdBORlSKyRkSGZTH+VhFZLiKLReQTEfHvJPUYrxFs3b+Vx755DFWlUdVGrL9lPSM6jbAkYIzxLxF49zseC3QHEoFrRCQxU7EFQJKqtgSmAY/4FU9YYL7PojBRVZ5f8DxNxzbl3s/uZc2uNQB2RpAx5hd+1gjaAGtUda2qHgNeBy4NL6Cqn6nqIe/ld0BN36KJwRrBT7t/ossrXUhOSeaMU89g0YBF1kmcMeZ3/DxGUAPYGPZ6E3BODuWTgQ+yGiEi/YH+ALVr1z65qGKkRpCWkcaFL11I6qFUxl8ynv5n97dO4owxWSoUB4tF5DogCeiQ1XhVnQRMAkhKSjqxXfsYqRGsTl1Nvcr1iC8WzwuXvkD9yvWpVbFW0GEZYwoxP3cRNwPhW6Ca3nu/ISIXAcOB3qp61Md4QjP0fRZBOJ5+nJFfjKT5+OY8M/sZADomdLQkYIzJlZ81gjlAQxGpi0sAfYBrwwuIyJnARKCbqm73MZYibe6WuSSnJLP458X0ad6Ha1pcE3RIxpgo4lsiUNU0ERkMzATigOdVdZmIjADmqmoK8ChQDnhT3J76BlXt7VNAvkw2aE999xS3fnQrp5Y7lXf7vEvvxv6sPmNM0eXrMQJVnQHMyPTefWHPL/Jz/lkqIk1DoU7ikk5PIvnMZB65+BEqlaoUdFjGmChUKA4WF4giUiPYd3Qf//j4H5SKL8UT3Z6gbe22tK3dNuiwjDFRLPbOJ4ziGsGM1TNoNq4Zk+ZPIr5YvHUSZ4zJF1YjiAI7D+3klg9v4dUlr9KsejOmXTmNc2rmdEmGMcZELnYSQUgU1gh2H97Ne6ve4/4O93N3u7spEVci6JCMMUVI7CSCKKsRbN63mVeXvMod599Bw6oNWX/LejsYbIzxhR0jKGRUlWfnPUviuEQemPUAP+7+EcCSgDHGN7GTCKKgRvDjrh/p/FJn+r/fn7NOO4vFAxfToEqDoMMyxhRxsdM0FFJIawRpGWl0fqkzuw7vYmLPidxw1g3WSZwxpkDETiIopDWClTtXUr9KfeKLxfPiZS9Sv0p9albwrzduY4zJLPZ2OQtJjeBY+jEenPUgLca3YOzssQB0SOhgScAYU+Bip0ZQiMzePJvklGSWbl/KtS2upW/LvkGHZIyJYbGTCApJ09CT3z3JbR/dxmnlTuO9a96jZ6OeQYdkjIlxsZMIQgJqGgp1EtemRhtuPOtGHr7oYSqWqhhILMYYEy52EkFANYK9R/Zy58d3Urp4aZ7s9iTn1zqf82udH0gsxhiTFTtY7KP3Vr5H4rhEnlvwHCXjSlonccaYQslqBD7YcXAHQz8cytSlU2lxSgveufodWtdoXWDzN8aYvIidRBBSADWCvUf3MmP1DB7s+CDDLhhmncQZYwq12EkEPtcINu7dyCuLX2HYBcNoUKUB629ZbweDjTFRwY4RnKQMzWDC3Ak0G9eMkV+O/KWTOEsCxphoETuJwIcawerU1Vz44oUMnD6QNjXasGTgEuskzhgTdWKnaSgkn2oEaRlpXPzyxew5sofJvSfzl1Z/QQpJ9xXGGJMXsZMI8qlGsGLHChpWbUh8sXhevvxl6lepz+nlT8+XaRtj/HX8+HE2bdrEkSNHgg7FN6VKlaJmzZoUL1484s/ETiIIOcG99qNpRxn15ShGfTWKRy9+lFvOvYV2ddrlc3DGGD9t2rSJ8uXLk5CQUCRr8KpKamoqmzZtom7duhF/LnYSwUnUCL7b9B3JKcks37Gc61tez/Utr8/HwIwxBeXIkSNFNgkAiAhVq1Zlx44defpc7CSCkDz+AEZ/M5o7Pr6DmhVqMuPaGXRv2N2nwIwxBaGoJoGQE1m+2EsEEcrQDIpJMc6rdR4Dkgbw0EUPUaFkhaDDMsaYfGenj2ay58gekt9NZugHQwE4v9b5jLtknCUBY0y+iIuLo1WrVjRv3pxevXqxZ8+eX8YtW7aMCy+8kMaNG9OwYUP++c9//qaPsg8++ICkpCQSExM588wzue222/IlpthJBCE5VJve+eEdEscm8uKiFylfsrx1EmeMyXelS5dm4cKFLF26lCpVqjB2rLtD4eHDh+nduzfDhg1j5cqVLFq0iG+++YZx48YBsHTpUgYPHswrr7zC8uXLmTt3Lg0a5M91S7HTNJTDRn37we0MnjGYN5e/SatTW/H+te9z1mlnFWBwxpgCd8stsHBh/k6zVSt48smIi5933nksXrwYgNdee422bdvSpUsXAMqUKcMzzzxDx44duemmm3jkkUcYPnw4TZo0AVzNYuDAgfkSttUIgH1H9/Hx2o/514X/YvYNsy0JGGN8l56ezieffELv3r0B1yx09tln/6ZM/fr1OXDgAPv27WPp0qW/G59fYrZGsGHvBl5e9DJ3t7ubBlUasOGWDZQvWT6g4IwxBS4Pe+756fDhw7Rq1YrNmzfTtGlTLr744kDiCOdrjUBEuonIShFZIyLDshhfUkTe8MZ/LyIJfsYDkIEybs44mo1rxqivRv3SSZwlAWNMQQgdI1i/fj2q+ssxgsTERObNm/ebsmvXrqVcuXJUqFCBZs2a/W58fvEtEYhIHDAW6A4kAteISGKmYsnAblVtADwBPOxXPKiysip0fKM7N824ifNqnseyQcuskzhjTCDKlCnDmDFjGD16NGlpafTt25evvvqK//3vf4CrOQwZMoQ777wTgDvuuINRo0axatUqADIyMpgwYUK+xOJnjaANsEZV16rqMeB14NJMZS4FXvSeTwM6i09Xe6RpOl2vhyU7l/PCpS8w87qZJFRK8GNWxhgTkTPPPJOWLVsydepUSpcuzbvvvsvIkSNp3LgxLVq0oHXr1gwePBiAli1b8uSTT3LNNdfQtGlTmjdvztq1a/MlDj+PEdQANoa93gSck10ZVU0Tkb1AVWBneCER6Q/0B6hdu/YJBRPfJJFXlnei/vDJnFYt8j44jDEmPx04cOA3r997771fnrdo0YJZs2Zl+9mePXvSs2fPfI8pKs4aUtVJqpqkqknVq1c/sYn07s0FUz61JGCMMZn4mQg2A7XCXtf03suyjIjEAxWBVB9jMsYYk4mfiWAO0FBE6opICaAPkJKpTArwZ+/5H4FP1S7nNcb4qKhvYk5k+XxLBKqaBgwGZgIrgP+o6jIRGSEivb1ik4GqIrIGuBX43SmmxhiTX0qVKkVqamqRTQah+xGUKlUqT5+TaFshSUlJOnfu3KDDMMZEoVi+Q5mIzFPVpKw+EztXFhtjYl7x4sXzdOeuWBEVZw0ZY4zxjyUCY4yJcZYIjDEmxkXdwWIR2QGsP8GPVyPTVcsxwJY5Ntgyx4aTWeY6qprlFblRlwhOhojMze6oeVFlyxwbbJljg1/LbE1DxhgT4ywRGGNMjIu1RDAp6AACYMscG2yZY4MvyxxTxwiMMcb8XqzVCIwxxmRiicAYY2JckUwEItJNRFaKyBoR+V2PpiJSUkTe8MZ/LyIJAYSZryJY5ltFZLmILBaRT0SkThBx5qfcljms3BUioiIS9acaRrLMInKV910vE5HXCjrG/BbBb7u2iHwmIgu833ePIOLMLyLyvIhsF5Gl2YwXERnjrY/FInLWSc9UVYvUA4gDfgTqASWARUBipjKDgAne8z7AG0HHXQDL3Ako4z0fGAvL7JUrD3wBfAckBR13AXzPDYEFQGXv9SlBx10AyzwJGOg9TwTWBR33SS5ze+AsYGk243sAHwACnAt8f7LzLIo1gjbAGlVdq6rHgNeBSzOVuRR40Xs+DegsIlKAMea3XJdZVT9T1UPey+9wd4yLZpF8zwD/BB4GikK/w5Es843AWFXdDaCq2ws4xvwWyTIrUMF7XhHYUoDx5TtV/QLYlUORS4GX1PkOqCQip53MPItiIqgBbAx7vcl7L8sy6m6gsxeoWiDR+SOSZQ6XjNujiGa5LrNXZa6lqtMLMjAfRfI9NwIaicjXIvKdiHQrsOj8EckyPwBcJyKbgBnAzQUTWmDy+n/Pld2PIMaIyHVAEtAh6Fj8JCLFgMeBfgGHUtDicc1DHXG1vi9EpIWq7gkyKJ9dA0xR1dEich7wsog0V9WMoAOLFkWxRrAZqBX2uqb3XpZlRCQeV51MLZDo/BHJMiMiFwHDgd6qerSAYvNLbstcHmgOzBKRdbi21JQoP2Acyfe8CUhR1eOq+hOwCpcYolUky5wM/AdAVb8FSuE6ZyuqIvq/50VRTARzgIYiUldESuAOBqdkKpMC/Nl7/kfgU/WOwkSpXJdZRM4EJuKSQLS3G0Muy6yqe1W1mqomqGoC7rhIb1WN5vucRvLbfgdXG0BEquGaitYWYIz5LZJl3gB0BhCRprhEsKNAoyxYKcCfvLOHzgX2qurWk5lgkWsaUtU0ERkMzMSdcfC8qi4TkRHAXFVNASbjqo9rcAdl+gQX8cmLcJkfBcoBb3rHxTeoau/Agj5JES5zkRLhMs8EuojIciAduENVo7a2G+Ey3wY8KyJ/xx047hfNO3YiMhWXzKt5xz3uB4oDqOoE3HGQHsAa4BDwl5OeZxSvL2OMMfmgKDYNGWOMyQNLBMYYE+MsERhjTIyzRGCMMTHOEoExxsQ4SwSmUBKRdBFZGPZIyKHsgXyY3xQR+cmb13zvCtW8TuM5EUn0nt+dadw3JxujN53QelkqIu+JSKVcyreK9t44jf/s9FFTKInIAVUtl99lc5jGFOB9VZ0mIl2Ax1S15UlM76Rjym26IvIisEpV/5VD+X64XlcH53cspuiwGoGJCiJSzruPwnwRWSIiv+tpVEROE5EvwvaY23nvdxGRb73PvikiuW2gvwAaeJ+91ZvWUhG5xXuvrIhMF5FF3vtXe+/PEpEkEXkIKO3F8ao37oA3fF1ELgmLeYqI/FFE4kTkURGZ4/Ux/7cIVsu3eJ2NiUgbbxkXiMg3ItLYuxJ3BHC1F8vVXuzPi8hsr2xWPbaaWBN039v2sEdWD9xVsQu9x9u4q+AreOOq4a6qDNVoD3jD24Dh3vM4XH9D1XAb9rLe+/8A7stiflOAP3rPrwS+B84GlgBlcVdlLwPOBK4Ang37bEVvOAvvngehmMLKhGK8HHjRe14C14tkaaA/cI/3fklgLlA3izgPhC3fm0A373UFIN57fhHwlve8H/BM2OdHAdd5zyvh+iIqG/T3bY9gH0WuiwlTZBxW1VahFyJSHBglIu2BDNye8B+AbWGfmQM875V9R1UXikgH3M1Kvva61iiB25POyqMicg+un5pkXP81b6vqQS+G/wLtgA+B0SLyMK456cs8LNcHwFMiUhLoBnyhqoe95qiWIvJHr1xFXGdxP2X6fGkRWegt/wrg47DyL4pIQ1w3C8WzmX8XoLeI3O69LgXU9qZlYpQlAhMt+gLVgbNV9bi4HkVLhRdQ1S+8RHEJMEVEHgd2Ax+r6jURzOMOVZ0WeiEinbMqpKqrxN3roAcwUkQ+UdURkSyEqh4RkVlAV+Bq3I1WwN1t6mZVnZnLJA6raisRKYPrf+cmYAzuBjyfqerl3oH1Wdl8XoArVHVlJPGa2GDHCEy0qAhs95JAJ+B391wWdx/mn1X1WeA53O3+vgPaikiozb+siDSKcJ5fApeJSBkRKYtr1vlSRE4HDqnqK7jO/LK6Z+xxr2aSlTdwHYWFahfgNuoDQ58RkUbePLOk7m5zQ4Db5Neu1ENdEfcLK7of10QWMhO4WbzqkbheaU2Ms0RgosWrQJKILAH+BPyQRZmOwCIRWYDb235KVXfgNoxTRWQxrlmoSSQzVNX5uGMHs3HHDJ5T1QVAC2C210RzPzAyi49PAhaHDhZn8hHuxkD/U3f7RXCJazkwX9xNyyeSS43di2Ux7sYsjwD/9pY9/HOfAYmhg8W4mkNxL7Zl3msT4+z0UWOMiXFWIzDGmBhnicAYY2KcJQJjjIlxlgiMMSbGWSIwxpgYZ4nAGGNinCUCY4yJcf8PdwWeeAOcl3kAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SML AUROC score: 0.9900661350000001\n"
     ]
    }
   ],
   "source": [
    "fpr, tpr, auroc_score = auroc(test_y, ood_scores)\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "print('SML AUROC score:', auroc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAhN0lEQVR4nO3de5hdVX3/8fcnk0wmtzGQhFvuksQkhBhgAAFbkJtAJVT91ULh8YdaEEqkVUrF2iIKVbTFKg+hgBVBqKAiP01tLEpAFAUlkQC5GIwRkgkIYSAJIQkkme/vj7UPOZnMJCeT2ecy5/N6nvPss/deZ6/vmknOd9bae6+tiMDMzOpXn0oHYGZmleVEYGZW55wIzMzqnBOBmVmdcyIwM6tzTgRmZnXOicBqhqRzJf24hHI3SfrncsRUDpKekXRy9v4qSXdWOibrXZwIrEdkX1abJG2Q9IKk2yQN7sk6IuK/IuLUEspdFBFX92TdBZJC0mtZO1dL+rKkhjzq6g5JzZK+ImllFuPvs/XhlY7NqpcTgfWkMyNiMHA40AL8U8cCkvqWPaqe9/asnccDfwl8uMLxACCpEZgHHAKcBjQDxwBtwFHdOF5v+F1ZCZwIrMdFxGrgR8A0ePOv6Esk/Q74XbbtPZIWSlor6ZeSphc+L2m0pHslrZHUJumGbPv5kh7O3kvSv0t6UdJ6SU9JKtR3m6Rrio53gaTlkl6WNEfSQUX7QtJFkn6XxTJbkkps53LgF8CMouN1p10HS3og2/aSpP+SNHQPf+wAHwTGAO+NiCUR0R4RL0bE1RExt6i9E4pievNnJekESa2SPinpj8A3JC2V9J6i8n2z+A/P1t+RtXOtpCckndCNuK3CnAisx0kaDZwBPF60+c+Bo4Gpkg4DbgU+CgwDbgbmSOqfDbP8EHgWGAeMBO7upJpTgT8FJgFvAT5A+su3YywnAl/I9h+YHbfj8d4DHAlMz8q9u8R2Tgb+BFierXe3XcpiPAiYAowGriolhg5OBv43IjZ047MFBwD7AmOBC4G7gHOK9r8beCkifiNpJPA/wDXZZ/4e+J6kEXtRv1WAE4H1pO9LWgs8DDwEfL5o3xci4uWI2ET6grk5In4VEdsi4nbgdeAdpCGMg4DLI+K1iNgcEQ93UtcWYAgwGVBELI2I5zspdy5wa0T8JiJeBz4FHCNpXFGZayNibUSsBB6k6C/8LvxG0mvAUuCnwI3Z9m61KyKWR8RPIuL1iFgDfJk07LSnhgGd/Qz2RDvwmSyWTcC3gJmSBmb7/4qUHADOA+ZGxNys9/ETYD7pjwCrIU4E1pP+PCKGRsTYiPib7IukYFXR+7HAZdlwwtoseYwmfVGOBp6NiK27qigiHgBuAGYDL0q6RVJzJ0UPIv0VXvjcBlLPYWRRmT8Wvd8IDAaQtDg74bpB0p8UlTk8K/OXpF7OoL1pl6T9Jd2dnXxeD9wJdOfkbhup17M31kTE5sJKNvy1FDgzSwYzSckBUnv/okN739kDMViZORFYuRRPc7sK+JcsaRReAyPirmzfmFJOVEbE9RFxBDCVNER0eSfFniN9YQEgaRDpL+fVJRz/kIgYnL1+3mFfRMR3gEeAK/eyXZ8n/XwOjYhm0l/aJZ2n6OB+4N1ZG7uyERhYtH5Ah/2dTUdcGB46C1iSJQdIbbqjQ3sHRcS13YjdKsiJwCrha8BFko7OTvoOkvRnkoYAvyYNb1ybbW+SdFzHA0g6Mvt8P+A1YDNpWKOju4APSZohqT/pS/dXEfFMD7XlWuACSQfsRbuGABuAddm4e2cJrRR3kL6cvydpsqQ+koZJ+kdJheGahcBfSWqQdBqlDUHdTTonczHbewOQei5nSnp3drym7ITzqG7GbxXiRGBlFxHzgQtIQzuvkE62np/t2wacCUwAVgKtpCGYjppJX7yvkIZ+2oB/7aSu+4F/Br5H+iI+GDi7B9vyFPAz0th/d9v1WdJw0zrSydd7uxnL66QTxr8FfgKsJyWg4cCvsmJ/m8WxlnT+5PslHPd5Us/nWODbRdtXkXoJ/wisISWhy/H3Ss2RH0xjZlbfnLnNzOqcE4GZWZ1zIjAzq3NOBGZmda7mJpUaPnx4jBs3rtJhmJnVlAULFrwUEZ1O/1FziWDcuHHMnz+/0mGYmdUUSc92tc9DQ2Zmdc6JwMyszjkRmJnVOScCM7M650RgZlbncksEkm5Veozgoi72S9L1So8QfLLw6DszMyuvPHsEt5EeoN2V04GJ2etC4D9yjMXMzLqQ230EEfGzDo8D7Ogs4JuRpj99VNJQSQd28bjBvffww/DjH+dyaGbMgPe9b8dtEbBtG7zxBmzZAkOGQB+PxJlZ9ankDWUj2fHxha3Ztp0SgaQLSb0GxowZ073aHnkErrmme5/dlcI03sOHpy/84ldHH/jA9sRQvCy8Vq6ED38YGho6L7d1a0ouHV/t7Z1v35Myxfs3bUp1HXFE2l7Yt24d7L8/jB69fXvxq3Ccrl4d92/bBps3p5/hpEldf+6FF1IsL78Mhx7a+XFXroRTT00Jd8AAaGyEgw5K73ensRFGjEiJuqEhLRsbS/usWS+Q6/MIsh7BDyNiWif7fkh6aPjD2fo84JPZwz261NLSElV1Z/GCBXDzzdC3L/Trt/OrsRHuuAM2bICmprTe2Lh9X+H9U09Ba2s65oAB2/cXL/v2TV9Unb0KX2Ld3V9cZsmS9OXav//27Rs3wqOPwqhRKY4+fbp+FT6zu32Qempjx3b9mV//Gg44AH7/+1RvU9POZV56Kb/fb79+cNhh2xN+xM7vI6CtDaZOTYnpyCPTz2vChLSvvX3HsoXX1q3p+KNHp3bsvz9I6dXeDvvtl7YXtvXpk34W/fundbM9IGlBRLR0tq+SPYLVpAd6F4yihOfIVp0jjoBbbtl1mU9+srRjRfg/eHetWZO+fCH1aJYuLW0obv369Nmmph17Ro89Bs88A83N27+IYfv74vUXX0yf//GPU0L7yU9yaeIOmptTYpRSvGvXpl5VQXt7ateECTB4cPqZHHwwTJwIw4bBW9+aPtvUBFOmpKTTt+ZmnLEeUsnf/BxglqS7gaOBdbmdH6gVTgLdN6LDXFqTJ1cmDkgJfc2a9L74L/qOr40bUyJ64QV4/fUdewsrVqRhruIeyG9/mz7z6qswcOD27UuXpvYX//t58cX0hT94cOpVvfpqWu5KY2Maghw9Oh2ruRkOPBCOPnrHYb1t27YPUx5wAOyzT+opdhyy27ABRo5M5fbZJ70K5a2q5JYIJN0FnAAMl9QKfAboBxARNwFzgTNIz3XdCHwor1jMykpKf2HvTnNz+mIs/ks+bytXpiQRkZLQU0/Bc8+lfUuXpnNdkPbdey8sWgTz5u08rPfaa3sXR0tLSpYHHQTjxsHQoWmI78ADU5LbtCklkWnTUpIrDJeOHZuGxqxH1dwzi6vuHIFZb9bVcGVE6mW0taUeTf/+O563aW9PFxc0NMDzz6eyGzbAJz4BJ5+cEkxjIyxfnnoIGzakiyKk7T2drjQ1paGtgQPTeZkJE9JFBCNHpqGvoUNz+VHUumo9R2Bm1a6r4crC0FFzM4wfX/rxLrlk92Xa21Ov5emnYfXqlGS2bYMnnkjDY888A6tWpW27+qNwypSUGPr3Tyf83/GOdCJ/8ODS460T7hGYWe3asiVdubdxIzz+eEoQq1bBoEEpmdx3X+efmzQJzjgDjjkGTjopnU/p5XbVI3AiMLPe7fXX08n3X/wCfvCDdF5k3bp0pVXBwIEpKTQ2pgsNTj89DWH1ogs4nAjMzDpauRK++1246aZ0tdTzz+98I+hRR6UbQSdPTr2IiRMrE2sPcCIwMytFRBpqOvfcdI6io+nT4dJLYebMnS9ZrnK7SgSe/MbMrEBKl7YuW5aSwrp16dzDnXfC8cfDk0/CX/91ujx4+HA4//y0rcb+oO7IicDMrCvNzWlSyXPPhZ/+NPUSvvrVdIlqWxvcfju8/e3pktmLL05zZ9UgJwIzs1JNnJiGhl55JV2+et998P73p3033ZTuifjgB2H27FSmRjgRmJl1R58+acbbe+5JSeGOO1IiuOMOmDUL9t03DR09+2ylI90tJwIzs73Vpw+cd16aruO55+A//zNNl3H77WkKjbPPTtNmVCknAjOznnTggfCRj6SEMG9e2vbtb6d7Fe6/v7KxdcGJwMwsLyeemO5NeOc70/opp8BnP1vZmDrhRGBmlqe+feHnP08PfBoyBK66as/mZyoDJwIzs3KYMiXNtgpp4rxSJuArEycCM7Ny2W+/NOU2wI03pruYq4ATgZlZOQ0alCbAA7jssvS0twpzIjAzK7djj4ULLoCHHoIvfKHS0TgRmJlVxA03pOWVV24/d1AhTgRmZpXQ2Aj/8R/p/QknVDQUJwIzs0q56KI0gd3q1Ts/C6GMnAjMzCrpvPPS8qMfrVgITgRmZpV0/fVp+Y1vwB//WJEQnAjMzCpJSrOVAlx7bUVCcCIwM6u0r341PQTnq1+tSPVOBGZmldanT3oUJsBLL5W/+rLXaGZmO/vYx9KyArOTOhGYmVWDU05Jy8KNZmXkRGBmVi322y8tV60qa7VOBGZm1eJ730vLO+8sa7VOBGZm1eLoo9Py3nvLWq0TgZlZtejXDw4/HObPL2u1TgRmZtVk2rS03Ly5bFU6EZiZVZPp09Ny4cKyVelEYGZWTd797rT85jfLVqUTgZlZNTnkkLQs4x3GTgRmZtVEgokTYd68slWZayKQdJqkZZKWS7qik/1jJc2T9KSkn0oalWc8ZmY1YcwYePll2LatLNXllggkNQCzgdOBqcA5kqZ2KPZvwDcjYjrwOaDyT3E2M6u0k05Ky2efLUt1efYIjgKWR8SKiHgDuBs4q0OZqcAD2fsHO9lvZlZ/DjooLZ9/vizV5ZkIRgLFE2a0ZtuKPQG8L3v/XmCIpGEdDyTpQknzJc1fs2ZNLsGamVWNCRPS8je/KUt1lT5Z/PfA8ZIeB44HVgM7DYpFxC0R0RIRLSNGjCh3jGZm5TVxYlq++mpZquub47FXA6OL1kdl294UEc+R9QgkDQbeHxFrc4zJzKz6DRsGDQ3Q1laW6vLsETwGTJQ0XlIjcDYwp7iApOGSCjF8Crg1x3jMzGpDQwMccAA89FBZqsstEUTEVmAWcB+wFPhORCyW9DlJM7NiJwDLJD0N7A/8S17xmJnVlFdfTT2DMshzaIiImAvM7bDtyqL39wD35BmDmVlNOvLIXnH5qJmZdVdDAyxbVpaqnAjMzKrR5MlpWYZnEzgRmJlVozPPTMvVq3ddrgc4EZiZVaMDD0zL5ctzr8qJwMysGo0bl5a//GXuVTkRmJlVo0GD0vK113KvyonAzKxajRgBP/957tXkeh+BmZnthX33hcGDc6/GPQIzs2p17LHwhz/kXo0TgZlZtdqwIS0jcq3GicDMrFqNziZwXr8+12qcCMzMqtWkSWmZ83MJnAjMzKrVwIFpmfPkc04EZmbV6uCD0/Lxx3OtxonAzKxaTZ+elvPm5VqNE4GZWbUq3EPwwgu5VuNEYGZWzcaOhVdeybUKJwIzs2o2fjwMGJBrFU4EZmbVbNgweOqpXKtwIjAzq2Zr1kBjY65VOBGYmVWzlhbYuDHXaSacCMzMqllhvqE33sitCicCM7NqVphm4vXXc6vCicDMrJo1NaVljk8qcyIwM6tmhQTgRGBmVqfGjk3LLVtyq8KJwMysmhUuHfXJYjOzOlVIBG1tuVXhRGBmVgu2bcvt0H1LKSTpOOAqYGz2GQEREW/NLTIzM4P990/LTZtyq6KkRAB8Hfg4sADILy2ZmdmOCk8pW7QIZs7MpYpSE8G6iPhRLhGYmVnX9tsvLZubc6ui1ETwoKR/Be4F3ry9LSJ+k0tUZmaWlOGqoVITwdHZsqVoWwAn9mw4Zma2g0IiWLcutypKSgQR8a7cIjAzs64VEkFra25VlHT5qKS3SPqypPnZ6zpJbynhc6dJWiZpuaQrOtk/RtKDkh6X9KSkM7rTCDOzXqtP9jU9fHh+VZRY7lbgVeAD2Ws98I1dfUBSAzAbOB2YCpwjaWqHYv8EfCciDgPOBm4sPXQzszoxaBBs3Zrb4Us9R3BwRLy/aP2zkhbu5jNHAcsjYgWApLuBs4AlRWUCKJwKfwvwXInxmJnVj4aGXG8oK7VHsEnSOwsr2Q1mu7u7YSSwqmi9NdtW7CrgPEmtwFzgYyXGY2ZWP/r2rfydxcDFwO3ZeQEBLwPn90D95wC3RcR1ko4B7pA0LSLaiwtJuhC4EGDMmDE9UK2ZWQ1paKj80FBELATeLqk5W19fwsdWA6OL1kdl24p9BDgtO+YjkpqA4cCLHeq/BbgFoKWlJb8Hd5qZVSOpcolA0nkRcaekT3TYDkBEfHkXH38MmChpPCkBnA38VYcyK4GTgNskTQGagDV71AIzs95u40ZYuDC3w++uRzAoWw7Z0wNHxFZJs4D7gAbg1ohYLOlzwPyImANcBnxN0sdJJ47Pjwj/xW9mVmzTJnhrfnN87jIRRMTN2fKz3Tl4RMwlnQQu3nZl0fslwHHdObaZWd2YNAna23dfrptKvaHsS5KaJfWTNE/SGknn5RaVmZltVyWXj56anSB+D/AMMAG4PK+gzMysSJUkgsIQ0p8B342I/GY/MjOzHeWcCEq9j+CHkn5LuonsYkkjgM25RWVmZttVQ48gIq4AjgVaImIL8BppuggzM8tbJXsEkk6MiAckva9oW3GRe/MKzMzMMgMGVPSZxccDDwBndrIvcCIwM8vfoEGwuuPEDD1nd/cRfCZbfii3CMzMbNeamuDxx3M7fKn3EXxe0tCi9X0kXZNbVGZmtt2qVTB0aG6HL/Xy0dMjYm1hJSJeAfw0MTOzcpg+PZ0nyEmpiaBBUv/CiqQBQP9dlDczs57Sp0+uU0yUeh/BfwHzJBUeT/kh4PZ8QjIzsx306VP5G8oi4ouSngBOzjZdHRH35RaVmZlt19BQFT0CgKXA1oi4X9JASUMi4tW8AjMzs0zOQ0OlXjV0AXAPcHO2aSTw/ZxiMjOzYtWQCIBLSM8NWA8QEb8D9ssrKDMzK1IlieD1iHijsCKpL+nOYjMzy1uVJIKHJP0jMEDSKcB3gf/OLSozM9uuShLBJ0kPlX8K+Cjp8ZP/lFdQZmZWpE8f2JzfzP+7vWpIUgOwOCImA1/LLRIzM+tcYcK5zZvTvEM9bLc9gojYBiyTNKbHazczs9079NC03Lo1l8OXeh/BPsBiSb8mPZQGgIiYmUtUZma2XeE5MJHPNTqlJoJ/zqV2MzPbvUomAklNwEXABNKJ4q9HRD59EzMz61zOiWB35whuB1pISeB04LpcojAzs65VeGhoakQcmuLQ14Ff5xKFmZl1rcI9gi2FNx4SMjOrkD7ZV3WFegRvl7Q+ey/SncXrs/cREc25RGVmZtsVegQ53V28u4fXN+RSq5mZla7CQ0NmZlZpTgRmZnXOicDMrM45EZiZ1TknAjOzOudEYGZW55wIzMzqXC0nAkmnSVomabmkKzrZ/++SFmavpyWtzTMeM7OaVCXTUO+x7Mlms4FTgFbgMUlzImJJoUxEfLyo/MeAw/KKx8ysZtVwj+AoYHlErIiIN4C7gbN2Uf4c4K4c4zEzq001nAhGAquK1luzbTuRNBYYDzzQxf4LJc2XNH/NmjU9HqiZWVWr4USwJ84G7smej7yTiLglIloiomXEiBFlDs3MrMJqOBGsBkYXrY/KtnXmbDwsZGbWubVr03LDhlwOn2cieAyYKGm8pEbSl/2cjoUkTQb2AR7JMRYzs9p10EFpWegZ9LDcEkH2IJtZwH3AUuA7EbFY0uckzSwqejZwd0ROfR4zs1rXN7vAsxLPI9hbETEXmNth25Ud1q/KMwYzs5pXeEJZTomgWk4Wm5lZV5wIzMzqnBOBmVmdcyIwM6tzTgRmZnXOicDMrM45EZiZ1TknAjOzOle4o7itLZfDOxGYmdWKpqZcDutEYGZW7QYOzPXwTgRmZrWiBqehNjOznpDTrKMFTgRmZrXCPQIzszrlHoGZmQHuEZiZ1S33CMzMDHCPwMysbrlHYGZmgHsEZmZ1yz0CMzMD3CMwM6tb7hGYmVmenAjMzGqFh4bMzOqUh4bMzAxwj8DMrG65R2BmZoB7BGZmdcs9AjMzA9wjMDOrW+4RmJkZ4B6BmVndco/AzMwA9wjMzOqWewRmZpanXBOBpNMkLZO0XNIVXZT5gKQlkhZL+lae8ZiZ1bSchob65nJUQFIDMBs4BWgFHpM0JyKWFJWZCHwKOC4iXpG0X17xmJnVrBoeGjoKWB4RKyLiDeBu4KwOZS4AZkfEKwAR8WKO8ZiZ1bYaPFk8ElhVtN6abSs2CZgk6ReSHpV0WmcHknShpPmS5q9ZsyancM3MqlQN9whK0ReYCJwAnAN8TdLQjoUi4paIaImIlhEjRpQ3QjOzalGDPYLVwOii9VHZtmKtwJyI2BIRfwCeJiUGMzMrqOEewWPAREnjJTUCZwNzOpT5Pqk3gKThpKGiFTnGZGZWu2qtRxARW4FZwH3AUuA7EbFY0uckzcyK3Qe0SVoCPAhcHhFtecVkZlaTcu4R5Hb5KEBEzAXmdth2ZdH7AD6RvczMbFdq7T6CctqyZQutra1s3ry50qHkpqmpiVGjRtGvX79Kh2Jm5VbLPYJyaW1tZciQIYwbNw7l/AOrhIigra2N1tZWxo8fX+lwzKxSau0cQTlt3ryZYcOG9cokACCJYcOG9eoej5ntQg1fNVRWvTUJFPT29plZ5fSaRGBm1ut5aKi6NTQ0MGPGDKZNm8aZZ57J2rVr39y3ePFiTjzxRN72trcxceJErr76aqLoF/qjH/2IlpYWpk6dymGHHcZll11WgRaYWdXy0FBtGDBgAAsXLmTRokXsu+++zJ49G4BNmzYxc+ZMrrjiCpYtW8YTTzzBL3/5S2688UYAFi1axKxZs7jzzjtZsmQJ8+fPZ8KECZVsiplVK18+WqK/+ztYuLBnjzljBnzlKyUXP+aYY3jyyScB+Na3vsVxxx3HqaeeCsDAgQO54YYbOOGEE7jkkkv40pe+xKc//WkmT54MpJ7FxRdf3LPxm1ltc4+gtmzbto158+Yxc2a6eXrx4sUcccQRO5Q5+OCD2bBhA+vXr2fRokU77Tcz65R7BCXag7/ce9KmTZuYMWMGq1evZsqUKZxyyikVicPMeiH3CGpD4RzBs88+S0S8eY5g6tSpLFiwYIeyK1asYPDgwTQ3N3PIIYfstN/MrFO+aqg2DBw4kOuvv57rrruOrVu3cu655/Lwww9z//33A6nncOmll/IP//APAFx++eV8/vOf5+mnnwagvb2dm266qWLxm1kVKvQI2ttzObwTQQ4OO+wwpk+fzl133cWAAQP4wQ9+wDXXXMPb3vY2Dj30UI488khmzZoFwPTp0/nKV77COeecw5QpU5g2bRorVngmbjMrUkgEjz6az+Ejp65GXlpaWmL+/Pk7bFu6dClTpkypUETlUy/tNLMO2tvhU5+CU06Bk0/u1iEkLYiIls729b6TxWZmvU2fPvDFL+Z3+NyObGZmNaHXJIJaG+LaU729fWZWOb0iETQ1NdHW1tZrvywLzyNoamqqdChm1gv1inMEo0aNorW1lTVr1lQ6lNwUnlBmZtbTekUi6Nevn5/cZWbWTb1iaMjMzLrPicDMrM45EZiZ1bmau7NY0hrg2W5+fDjwUg+GUwvc5vrgNteHvWnz2IgY0dmOmksEe0PS/K5use6t3Ob64DbXh7za7KEhM7M650RgZlbn6i0R3FLpACrAba4PbnN9yKXNdXWOwMzMdlZvPQIzM+vAicDMrM71ykQg6TRJyyQtl3RFJ/v7S/p2tv9XksZVIMweVUKbPyFpiaQnJc2TNLYScfak3bW5qNz7JYWkmr/UsJQ2S/pA9rteLOlb5Y6xp5Xwb3uMpAclPZ79+z6jEnH2FEm3SnpR0qIu9kvS9dnP40lJh+91pRHRq15AA/B74K1AI/AEMLVDmb8Bbsrenw18u9Jxl6HN7wIGZu8vroc2Z+WGAD8DHgVaKh13GX7PE4HHgX2y9f0qHXcZ2nwLcHH2firwTKXj3ss2/ylwOLCoi/1nAD8CBLwD+NXe1tkbewRHAcsjYkVEvAHcDZzVocxZwO3Z+3uAk6TC06Fr0m7bHBEPRsTGbPVRoNbntC7l9wxwNfBFYHM5g8tJKW2+AJgdEa8ARMSLZY6xp5XS5gCas/dvAZ4rY3w9LiJ+Bry8iyJnAd+M5FFgqKQD96bO3pgIRgKritZbs22dlomIrcA6YFhZostHKW0u9hHSXxS1bLdtzrrMoyPif8oZWI5K+T1PAiZJ+oWkRyWdVrbo8lFKm68CzpPUCswFPlae0CpmT/+/71aveB6BlU7SeUALcHylY8mTpD7Al4HzKxxKufUlDQ+dQOr1/UzSoRGxtpJB5ewc4LaIuE7SMcAdkqZFRHulA6sVvbFHsBoYXbQ+KtvWaRlJfUndybayRJePUtqMpJOBTwMzI+L1MsWWl921eQgwDfippGdIY6lzavyEcSm/51ZgTkRsiYg/AE+TEkOtKqXNHwG+AxARjwBNpMnZequS/r/vid6YCB4DJkoaL6mRdDJ4Tocyc4D/m73/P8ADkZ2FqVG7bbOkw4CbSUmg1seNYTdtjoh1ETE8IsZFxDjSeZGZETG/MuH2iFL+bX+f1BtA0nDSUNGKMsbY00pp80rgJABJU0iJoPc+tza1/4PZ1UPvANZFxPN7c8BeNzQUEVslzQLuI11xcGtELJb0OWB+RMwBvk7qPi4nnZQ5u3IR770S2/yvwGDgu9l58ZURMbNiQe+lEtvcq5TY5vuAUyUtAbYBl0dEzfZ2S2zzZcDXJH2cdOL4/Fr+w07SXaRkPjw77/EZoB9ARNxEOg9yBrAc2Ah8aK/rrOGfl5mZ9YDeODRkZmZ7wInAzKzOORGYmdU5JwIzszrnRGBmVuecCMw6IWmbpIWSFkn6b0lDe/j4z2TX+SNpQ08e22xPORGYdW5TRMyIiGmke00uqXRAZnlxIjDbvUfIJvWSdLCk/5W0QNLPJU3Otu8v6f9JeiJ7HZtt/35WdrGkCyvYBrMu9bo7i816kqQG0vQFX8823QJcFBG/k3Q0cCNwInA98FBEvDf7zOCs/Icj4mVJA4DHJH2vlu/0td7JicCscwMkLST1BJYCP5E0GDiW7dN0APTPlicCHwSIiG2kqc0BLpX03uz9aNIEcE4EVlWcCMw6tykiZkgaSJrn5hLgNmBtRMwo5QCSTgBOBo6JiI2SfkqaEM2sqvgcgdkuZE91u5Q0sdlG4A+S/gLefHbs27Oi80iPAEVSg6S3kKY3fyVLApNJU2GbVR0nArPdiIjHgSdJD0A5F/iIpCeAxWx/bOLfAu+S9BSwgPTs3P8F+kpaClxLmgrbrOp49lEzszrnHoGZWZ1zIjAzq3NOBGZmdc6JwMyszjkRmJnVOScCM7M650RgZlbn/j+3dz/CuUa8jAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SML AUPR score: 0.9890504516877849\n"
     ]
    }
   ],
   "source": [
    "precision, recall, aupr_score = aupr(test_y, ood_scores)\n",
    "\n",
    "plot_precision_recall_curve(precision, recall)\n",
    "print('SML AUPR score:', aupr_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SML fpr95 score: 0.0384\n"
     ]
    }
   ],
   "source": [
    "fpr95 = fpr_at_95_tpr(test_y, ood_scores)\n",
    "print('SML fpr95 score:', fpr95)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}