cloud-rn50-unet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_kg_hide-output": true,
    "execution": {
     "iopub.execute_input": "2020-11-03T09:11:56.119273Z",
     "iopub.status.busy": "2020-11-03T09:11:56.117476Z",
     "iopub.status.idle": "2020-11-03T09:16:35.421259Z",
     "shell.execute_reply": "2020-11-03T09:16:35.420294Z"
    },
    "papermill": {
     "duration": 279.336911,
     "end_time": "2020-11-03T09:16:35.421427",
     "exception": false,
     "start_time": "2020-11-03T09:11:56.084516",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/pip/_internal/commands/install.py:235: UserWarning: Disabling all use of wheels due to the use of --build-option / --global-option / --install-option.\r\n",
      "  cmdoptions.check_install_build_global(options)\r\n",
      "Processing /kaggle/input/apex-325f5a0/apex-master\r\n",
      "Skipping wheel build for apex, due to binaries being disabled for it.\r\n",
      "Installing collected packages: apex\r\n",
      "    Running setup.py install for apex ... \u001b[?25l-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \bdone\r\n",
      "\u001b[?25hSuccessfully installed apex-0.1\r\n"
     ]
    }
   ],
   "source": [
    "!pip install -q git+https://github.com/qubvel/segmentation_models.pytorch\n",
    "!pip install -q -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ../input/apex-325f5a0/apex-master/\n",
    "from apex import amp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_kg_hide-input": true,
    "_kg_hide-output": true,
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:35.509836Z",
     "iopub.status.busy": "2020-11-03T09:16:35.509104Z",
     "iopub.status.idle": "2020-11-03T09:16:39.785815Z",
     "shell.execute_reply": "2020-11-03T09:16:39.784473Z"
    },
    "papermill": {
     "duration": 4.329399,
     "end_time": "2020-11-03T09:16:39.785974",
     "exception": false,
     "start_time": "2020-11-03T09:16:35.456575",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os, gc, time, json, copy\n",
    "import pickle, random, itertools, collections\n",
    "import tqdm\n",
    "from functools import partial\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from PIL import Image\n",
    "import cv2\n",
    "\n",
    "from sklearn.model_selection import KFold, StratifiedKFold, train_test_split\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "from torch.utils.data import TensorDataset, DataLoader, Dataset\n",
    "from torch.utils.data.sampler import SubsetRandomSampler\n",
    "from torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\n",
    "\n",
    "import albumentations as albu\n",
    "from albumentations import pytorch as AT\n",
    "\n",
    "import segmentation_models_pytorch as smp\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.034698,
     "end_time": "2020-11-03T09:16:39.855975",
     "exception": false,
     "start_time": "2020-11-03T09:16:39.821277",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:40.293162Z",
     "iopub.status.busy": "2020-11-03T09:16:40.292304Z",
     "iopub.status.idle": "2020-11-03T09:16:40.297869Z",
     "shell.execute_reply": "2020-11-03T09:16:40.297186Z"
    },
    "papermill": {
     "duration": 0.407183,
     "end_time": "2020-11-03T09:16:40.298014",
     "exception": false,
     "start_time": "2020-11-03T09:16:39.890831",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "SEED = 2019\n",
    "\n",
    "# Data parameters\n",
    "path = '../input/understanding_cloud_organization'\n",
    "img_paths = '../input/understanding-clouds-resized'\n",
    "output_prefix = \"best_model_optim.pt\"\n",
    "n_splits = 5\n",
    "which_fold = 0  # should be int in [0, n_splits-1]\n",
    "\n",
    "# Learning parameters\n",
    "batch_size = 32\n",
    "lr = 1.3e-2\n",
    "epochs_for_sched = 14\n",
    "checkpoint_iter = 200\n",
    "warmup_proportion = 0.25\n",
    "grad_accumulation_steps = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.035265,
     "end_time": "2020-11-03T09:16:40.368110",
     "exception": false,
     "start_time": "2020-11-03T09:16:40.332845",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_kg_hide-input": true,
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:40.585028Z",
     "iopub.status.busy": "2020-11-03T09:16:40.579747Z",
     "iopub.status.idle": "2020-11-03T09:16:40.619380Z",
     "shell.execute_reply": "2020-11-03T09:16:40.618712Z"
    },
    "papermill": {
     "duration": 0.216055,
     "end_time": "2020-11-03T09:16:40.619518",
     "exception": false,
     "start_time": "2020-11-03T09:16:40.403463",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sigmoid = lambda x: 1 / (1 + np.exp(-x))\n",
    "\n",
    "def seed_torch(seed=SEED):\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "\n",
    "def get_img(x, folder: str='train_images_525/train_images_525'):\n",
    "    \"\"\"\n",
    "    Return image based on image name and folder.\n",
    "    \"\"\"\n",
    "    data_folder = f\"{img_paths}/{folder}\"\n",
    "    image_path = os.path.join(data_folder, x)\n",
    "    img = cv2.imread(image_path)\n",
    "    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "    return img\n",
    "\n",
    "def rle_decode(mask_rle: str = '', shape: tuple = (1400, 2100)):\n",
    "    \"\"\"\n",
    "    Decode run-length encoded mask.\n",
    "    \n",
    "    :param mask_rle: run-length as string formatted (start length)\n",
    "    :param shape: (height, width) of array to return \n",
    "    Returns numpy array, 1 - mask, 0 - background\n",
    "    \"\"\"\n",
    "    s = mask_rle.split()\n",
    "    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n",
    "    starts -= 1\n",
    "    ends = starts + lengths\n",
    "    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n",
    "    for lo, hi in zip(starts, ends):\n",
    "        img[lo:hi] = 1\n",
    "    return img.reshape(shape, order='F')\n",
    "\n",
    "def make_mask(image_name: str='img.jpg', shape: tuple=(350, 525)):\n",
    "    \"Create mask (ndarray, values between 0-1) based on image name and shape.\"\n",
    "    masks = np.zeros((shape[0], shape[1], 4), dtype=np.float32)\n",
    "    for classidx, classid in enumerate([\"Fish\", \"Flower\", \"Gravel\", \"Sugar\"]):\n",
    "        mask = cv2.imread(\"../input/understanding-clouds-resized/train_masks_525/train_masks_525/\" + classid + image_name)\n",
    "        if mask is None:\n",
    "            continue\n",
    "        if mask[:,:,0].shape != (350,525):\n",
    "            mask = cv2.resize(mask, (525,350))\n",
    "        masks[:, :, classidx] = mask[:,:,0]\n",
    "    masks = masks / 255\n",
    "    return masks\n",
    "\n",
    "def mask2rle(img):\n",
    "    \"\"\"\n",
    "    Convert mask to run-length encoding.\n",
    "    img: numpy array, 1 - mask, 0 - background\n",
    "    Returns run length as string formated\n",
    "    \"\"\"\n",
    "    pixels= img.T.flatten()\n",
    "    pixels = np.concatenate([[0], pixels, [0]])\n",
    "    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n",
    "    runs[1::2] -= runs[::2]\n",
    "    return ' '.join(str(x) for x in runs)\n",
    "\n",
    "\n",
    "def visualize(image, mask, original_image=None, original_mask=None):\n",
    "    \"\"\"\n",
    "    Plot images and masks.\n",
    "    \n",
    "    Input\n",
    "    - image: ndarray image loaded from cv2\n",
    "    - mask: ndarray output from make_mask\n",
    "    - original ones: If given, show them all\n",
    "    \"\"\"\n",
    "    fontsize = 14\n",
    "    class_dict = {0: 'Fish', 1: 'Flower', 2: 'Gravel', 3: 'Sugar'}\n",
    "    \n",
    "    if original_image is None and original_mask is None:\n",
    "        f, ax = plt.subplots(1, 5, figsize=(24, 24))\n",
    "\n",
    "        ax[0].imshow(image)\n",
    "        for i in range(4):\n",
    "            ax[i + 1].imshow(mask[:, :, i])\n",
    "            ax[i + 1].set_title(f'Mask {class_dict[i]}', fontsize=fontsize)\n",
    "    else:\n",
    "        f, ax = plt.subplots(2, 5, figsize=(24, 12))\n",
    "\n",
    "        ax[0, 0].imshow(original_image)\n",
    "        ax[0, 0].set_title('Original image', fontsize=fontsize)\n",
    "                \n",
    "        for i in range(4):\n",
    "            ax[0, i + 1].imshow(original_mask[:, :, i])\n",
    "            ax[0, i + 1].set_title(f'Original mask {class_dict[i]}', fontsize=fontsize)\n",
    "        \n",
    "        ax[1, 0].imshow(image)\n",
    "        ax[1, 0].set_title('Transformed image', fontsize=fontsize)        \n",
    "        \n",
    "        for i in range(4):\n",
    "            ax[1, i + 1].imshow(mask[:, :, i])\n",
    "            ax[1, i + 1].set_title(f'Transformed mask {class_dict[i]}', fontsize=fontsize)\n",
    "            \n",
    "            \n",
    "def visualize_with_raw(image, mask, original_image=None, original_mask=None, raw_image=None, raw_mak=None):\n",
    "    \"\"\"\n",
    "    Plot images and masks.\n",
    "    \n",
    "    Input\n",
    "    - image: ndarray image loaded from cv2\n",
    "    - mask: ndarray output from make_mask\n",
    "    - original ones / raw ones: If given, show them all\n",
    "    \"\"\"\n",
    "    fontsize = 14\n",
    "    class_dict = {0: 'Fish', 1: 'Flower', 2: 'Gravel', 3: 'Sugar'}\n",
    "\n",
    "    f, ax = plt.subplots(3, 5, figsize=(24, 12))\n",
    "\n",
    "    ax[0, 0].imshow(original_image)\n",
    "    ax[0, 0].set_title('Original image', fontsize=fontsize)\n",
    "\n",
    "    for i in range(4):\n",
    "        ax[0, i + 1].imshow(original_mask[:, :, i])\n",
    "        ax[0, i + 1].set_title(f'Original mask {class_dict[i]}', fontsize=fontsize)\n",
    "\n",
    "    ax[1, 0].imshow(raw_image)\n",
    "    ax[1, 0].set_title('Original image', fontsize=fontsize)\n",
    "\n",
    "    for i in range(4):\n",
    "        ax[1, i + 1].imshow(raw_mak[:, :, i])\n",
    "        ax[1, i + 1].set_title(f'Raw predicted mask {class_dict[i]}', fontsize=fontsize)\n",
    "        \n",
    "    ax[2, 0].imshow(image)\n",
    "    ax[2, 0].set_title('Transformed image', fontsize=fontsize)\n",
    "\n",
    "    for i in range(4):\n",
    "        ax[2, i + 1].imshow(mask[:, :, i])\n",
    "        ax[2, i + 1].set_title(f'Predicted mask with processing {class_dict[i]}', fontsize=fontsize)\n",
    "            \n",
    "            \n",
    "def plot_with_augmentation(image, mask, augment):\n",
    "    \"\"\"\n",
    "    Plot images, masks and augmentation results.\n",
    "    \n",
    "    Input\n",
    "    - image: ndarray image loaded from cv2\n",
    "    - mask: ndarray output from make_mask\n",
    "    - augment: transformation from albumentations\n",
    "    \"\"\"\n",
    "    augmented = augment(image=image, mask=mask)\n",
    "    image_flipped = augmented['image']\n",
    "    mask_flipped = augmented['mask']\n",
    "    visualize(image_flipped, mask_flipped, original_image=image, original_mask=mask)\n",
    "\n",
    "\n",
    "def post_process(probability, threshold, min_size):\n",
    "    \"\"\"\n",
    "    Post processing of each predicted mask, components with lesser number of pixels\n",
    "    than `min_size` are ignored\n",
    "    \n",
    "    Input\n",
    "    - probability: predicted probability mask, ndarray (350, 525)\n",
    "    - threshold: value to binarize probability mask\n",
    "    - min_size: min size to filter false positive\n",
    "    \"\"\"\n",
    "    mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]\n",
    "    num_component, component = cv2.connectedComponents(mask.astype(np.uint8))\n",
    "    predictions = np.zeros((350, 525), np.float32)\n",
    "    num = 0\n",
    "    for c in range(1, num_component):\n",
    "        p = (component == c)\n",
    "        if p.sum() > min_size:\n",
    "            predictions[p] = 1\n",
    "            num += 1\n",
    "    return predictions, num\n",
    "\n",
    "def post_process_val(probability, threshold, min_size):\n",
    "    \"\"\"\n",
    "    Post processing of each predicted mask, components with lesser number of pixels\n",
    "    than `min_size` are ignored\n",
    "    \n",
    "    Input\n",
    "    - probability: predicted probability mask, ndarray (256, 384)\n",
    "    - threshold: value to binarize probability mask\n",
    "    - min_size: min size to filter false positive\n",
    "    \"\"\"\n",
    "    mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]\n",
    "    num_component, component = cv2.connectedComponents(mask.astype(np.uint8))\n",
    "    predictions = np.zeros((256, 384), np.float32)\n",
    "    num = 0\n",
    "    for c in range(1, num_component):\n",
    "        p = (component == c)\n",
    "        if p.sum() > min_size:\n",
    "            predictions[p] = 1\n",
    "            num += 1\n",
    "    return predictions\n",
    "\n",
    "\n",
    "def get_training_augmentation():\n",
    "    train_transform = [\n",
    "        albu.Resize(256, 384),\n",
    "        albu.HorizontalFlip(),\n",
    "        albu.VerticalFlip(),\n",
    "        albu.ShiftScaleRotate(scale_limit=0.35, rotate_limit=10, shift_limit=0.1, border_mode=0),\n",
    "    ]\n",
    "    return albu.Compose(train_transform)\n",
    "\n",
    "def get_validation_augmentation():\n",
    "    \"\"\"Add paddings to make image shape divisible by 32\"\"\"\n",
    "    test_transform = [\n",
    "        albu.Resize(256, 384)\n",
    "    ]\n",
    "    return albu.Compose(test_transform)\n",
    "\n",
    "def to_tensor(x, **kwargs):\n",
    "    \"\"\"\n",
    "    Convert image or mask.\n",
    "    \"\"\"\n",
    "    return x.transpose(2, 0, 1).astype('float32')\n",
    "\n",
    "def get_preprocessing(preprocessing_fn):\n",
    "    \"\"\"Construct preprocessing transform\n",
    "    \n",
    "    Args:\n",
    "        preprocessing_fn (callbale): data normalization function \n",
    "            (can be specific for each pretrained neural network)\n",
    "    Return:\n",
    "        transform: albumentations.Compose\n",
    "    \n",
    "    \"\"\"\n",
    "    _transform = [\n",
    "        albu.Lambda(image=preprocessing_fn),\n",
    "        albu.Lambda(image=to_tensor, mask=to_tensor),\n",
    "    ]\n",
    "    return albu.Compose(_transform)\n",
    "\n",
    "\n",
    "def compute_dice(img1, img2):\n",
    "    img1 = np.asarray(img1).astype(np.bool)\n",
    "    img2 = np.asarray(img2).astype(np.bool)\n",
    "    intersection = np.logical_and(img1, img2)\n",
    "    return 2. * intersection.sum() / (img1.sum() + img2.sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.038595,
     "end_time": "2020-11-03T09:16:40.695339",
     "exception": false,
     "start_time": "2020-11-03T09:16:40.656744",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Data overview\n",
    "\n",
    "Let's have a look at the data first.\n",
    "We have folders with train and test images, file with train image ids and masks and sample submission."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:40.781343Z",
     "iopub.status.busy": "2020-11-03T09:16:40.780433Z",
     "iopub.status.idle": "2020-11-03T09:16:43.554923Z",
     "shell.execute_reply": "2020-11-03T09:16:43.556104Z"
    },
    "papermill": {
     "duration": 2.822569,
     "end_time": "2020-11-03T09:16:43.556271",
     "exception": false,
     "start_time": "2020-11-03T09:16:40.733702",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(f'{path}/train.csv')\n",
    "sub = pd.read_csv(f'{path}/sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:43.681034Z",
     "iopub.status.busy": "2020-11-03T09:16:43.679752Z",
     "iopub.status.idle": "2020-11-03T09:16:43.836921Z",
     "shell.execute_reply": "2020-11-03T09:16:43.838613Z"
    },
    "papermill": {
     "duration": 0.223508,
     "end_time": "2020-11-03T09:16:43.838819",
     "exception": false,
     "start_time": "2020-11-03T09:16:43.615311",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 5546 images in train dataset\n",
      "There are 3698 images in test dataset\n"
     ]
    }
   ],
   "source": [
    "n_train = len(os.listdir(f'{img_paths}/train_images_525/train_images_525'))\n",
    "n_test = len(os.listdir(f'{img_paths}/test_images_525/test_images_525'))\n",
    "print(f'There are {n_train} images in train dataset')\n",
    "print(f'There are {n_test} images in test dataset')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.055657,
     "end_time": "2020-11-03T09:16:43.951793",
     "exception": false,
     "start_time": "2020-11-03T09:16:43.896136",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Preparing data for modelling\n",
    "\n",
    "At first, let's create a list of unique image_ids and the count of masks for each image. This will allow us to make a stratified split based on this count."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:44.073599Z",
     "iopub.status.busy": "2020-11-03T09:16:44.072586Z",
     "iopub.status.idle": "2020-11-03T09:16:44.080269Z",
     "shell.execute_reply": "2020-11-03T09:16:44.079451Z"
    },
    "papermill": {
     "duration": 0.072211,
     "end_time": "2020-11-03T09:16:44.080454",
     "exception": false,
     "start_time": "2020-11-03T09:16:44.008243",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train_val_split(id_mask_count, n_splits=None, which_fold=None):\n",
    "    kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n",
    "    cv_indices = [(tr_idx, val_idx) for tr_idx, val_idx in kf.split(id_mask_count['img_id'], id_mask_count['count'])]\n",
    "    if which_fold is not None:\n",
    "        return cv_indices[which_fold]\n",
    "    return cv_indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:44.207078Z",
     "iopub.status.busy": "2020-11-03T09:16:44.205930Z",
     "iopub.status.idle": "2020-11-03T09:16:44.267629Z",
     "shell.execute_reply": "2020-11-03T09:16:44.266406Z"
    },
    "papermill": {
     "duration": 0.132395,
     "end_time": "2020-11-03T09:16:44.267762",
     "exception": false,
     "start_time": "2020-11-03T09:16:44.135367",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "id_mask_count = (train.loc[~train['EncodedPixels'].isnull(), 'Image_Label']\n",
    "                    .apply(lambda x: x.split('_')[0])\n",
    "                    .value_counts()\n",
    "                    .reset_index()\n",
    "                    .rename(columns={'index': 'img_id', 'Image_Label': 'count'}))\n",
    "\n",
    "trn_idx, val_idx = train_val_split(id_mask_count, n_splits, which_fold)\n",
    "train_ids, valid_ids = id_mask_count['img_id'].values[trn_idx], id_mask_count['img_id'].values[val_idx]\n",
    "test_ids = sub['Image_Label'].apply(lambda x: x.split('_')[0]).drop_duplicates().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:44.348337Z",
     "iopub.status.busy": "2020-11-03T09:16:44.347217Z",
     "iopub.status.idle": "2020-11-03T09:16:44.352171Z",
     "shell.execute_reply": "2020-11-03T09:16:44.352781Z"
    },
    "papermill": {
     "duration": 0.049134,
     "end_time": "2020-11-03T09:16:44.352924",
     "exception": false,
     "start_time": "2020-11-03T09:16:44.303790",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4436, 1110)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_ids), len(valid_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:44.438263Z",
     "iopub.status.busy": "2020-11-03T09:16:44.437391Z",
     "iopub.status.idle": "2020-11-03T09:16:44.448316Z",
     "shell.execute_reply": "2020-11-03T09:16:44.448935Z"
    },
    "papermill": {
     "duration": 0.059404,
     "end_time": "2020-11-03T09:16:44.449086",
     "exception": false,
     "start_time": "2020-11-03T09:16:44.389682",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>img_id</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1d20d37.jpg</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0d56b5d.jpg</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1fa5dc9.jpg</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>98cfcc2.jpg</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0c1d048.jpg</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5541</th>\n",
       "      <td>589c00e.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5542</th>\n",
       "      <td>3555fe9.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5543</th>\n",
       "      <td>da0d544.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5544</th>\n",
       "      <td>0d24e80.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5545</th>\n",
       "      <td>725f986.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5546 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           img_id  count\n",
       "0     1d20d37.jpg      4\n",
       "1     0d56b5d.jpg      4\n",
       "2     1fa5dc9.jpg      4\n",
       "3     98cfcc2.jpg      4\n",
       "4     0c1d048.jpg      4\n",
       "...           ...    ...\n",
       "5541  589c00e.jpg      1\n",
       "5542  3555fe9.jpg      1\n",
       "5543  da0d544.jpg      1\n",
       "5544  0d24e80.jpg      1\n",
       "5545  725f986.jpg      1\n",
       "\n",
       "[5546 rows x 2 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id_mask_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:44.535351Z",
     "iopub.status.busy": "2020-11-03T09:16:44.531717Z",
     "iopub.status.idle": "2020-11-03T09:16:44.538707Z",
     "shell.execute_reply": "2020-11-03T09:16:44.539234Z"
    },
    "papermill": {
     "duration": 0.052273,
     "end_time": "2020-11-03T09:16:44.539372",
     "exception": false,
     "start_time": "2020-11-03T09:16:44.487099",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2    2372\n",
       "3    1560\n",
       "1    1348\n",
       "4     266\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id_mask_count['count'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.038224,
     "end_time": "2020-11-03T09:16:44.615866",
     "exception": false,
     "start_time": "2020-11-03T09:16:44.577642",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Only 266 images have all four masks."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.037863,
     "end_time": "2020-11-03T09:16:44.692188",
     "exception": false,
     "start_time": "2020-11-03T09:16:44.654325",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Setting up data loader, model, solver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:44.787874Z",
     "iopub.status.busy": "2020-11-03T09:16:44.785733Z",
     "iopub.status.idle": "2020-11-03T09:16:44.788890Z",
     "shell.execute_reply": "2020-11-03T09:16:44.789893Z"
    },
    "papermill": {
     "duration": 0.059579,
     "end_time": "2020-11-03T09:16:44.790109",
     "exception": false,
     "start_time": "2020-11-03T09:16:44.730530",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class CloudDataset(Dataset):\n",
    "    def __init__(self, dataset_type: str='train', img_ids: np.array=None,\n",
    "                 transforms=albu.Compose([albu.HorizontalFlip(), AT.ToTensor()]),\n",
    "                 preprocessing=None, label_smooth=0):\n",
    "        if dataset_type != 'test':\n",
    "            self.data_folder = f\"{img_paths}/train_images_525/train_images_525\"\n",
    "        else:\n",
    "            self.data_folder = f\"{img_paths}/test_images_525/test_images_525\"\n",
    "        self.img_ids = img_ids\n",
    "        self.transforms = transforms\n",
    "        self.preprocessing = preprocessing\n",
    "        self.label_smooth = label_smooth\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        image_name = self.img_ids[idx]\n",
    "        mask = make_mask(image_name)\n",
    "        img = cv2.imread(os.path.join(self.data_folder, image_name))\n",
    "        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
    "        augmented = self.transforms(image=img, mask=mask)\n",
    "        img = augmented['image']\n",
    "        mask = augmented['mask']\n",
    "        if self.label_smooth > 0:\n",
    "            mask = np.where(mask==1, 1-label_smooth, label_smooth).astype('float32')\n",
    "        if self.preprocessing:\n",
    "            preprocessed = self.preprocessing(image=img, mask=mask)\n",
    "            img = preprocessed['image']\n",
    "            mask = preprocessed['mask']\n",
    "        return img, mask\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.img_ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.039134,
     "end_time": "2020-11-03T09:16:44.868880",
     "exception": false,
     "start_time": "2020-11-03T09:16:44.829746",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "In the optimizer initialization, we split model parameters to 3 parts for differential learning rates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:44.959551Z",
     "iopub.status.busy": "2020-11-03T09:16:44.958467Z",
     "iopub.status.idle": "2020-11-03T09:16:44.961799Z",
     "shell.execute_reply": "2020-11-03T09:16:44.961204Z"
    },
    "papermill": {
     "duration": 0.054347,
     "end_time": "2020-11-03T09:16:44.961930",
     "exception": false,
     "start_time": "2020-11-03T09:16:44.907583",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "ENCODER = 'resnet50'\n",
    "ENCODER_WEIGHTS = 'imagenet'\n",
    "ACTIVATION = None\n",
    "\n",
    "preprocessing_fn = smp.encoders.get_preprocessing_fn(ENCODER, ENCODER_WEIGHTS)\n",
    "\n",
    "def model_optimizer_init(ft_lrs, num_labels=4):\n",
    "    print(\"Building model and optimizer...\")\n",
    "    model = smp.Unet(\n",
    "        encoder_name=ENCODER, \n",
    "        encoder_weights=ENCODER_WEIGHTS, \n",
    "        classes=4, \n",
    "        activation=ACTIVATION\n",
    "    )\n",
    "    param_g1 = [p for c in list(model.encoder.children())[:5] for p in c.parameters()]\n",
    "    param_g2 = [p for c in list(model.encoder.children())[5:] for p in c.parameters()]\n",
    "    optimizer = torch.optim.Adam([\n",
    "        {'params': param_g1, 'lr': ft_lrs[0]},\n",
    "        {'params': param_g2, 'lr': ft_lrs[1]}, \n",
    "        {'params': model.decoder.parameters(), 'lr': ft_lrs[2]},\n",
    "    ])\n",
    "    return model, optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "_kg_hide-input": true,
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:45.048154Z",
     "iopub.status.busy": "2020-11-03T09:16:45.046504Z",
     "iopub.status.idle": "2020-11-03T09:16:45.048912Z",
     "shell.execute_reply": "2020-11-03T09:16:45.049475Z"
    },
    "papermill": {
     "duration": 0.048615,
     "end_time": "2020-11-03T09:16:45.049629",
     "exception": false,
     "start_time": "2020-11-03T09:16:45.001014",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# def dice_loss(input, target):\n",
    "#     input = torch.sigmoid(input)\n",
    "#     smooth = 1.0\n",
    "#     iflat = input.view(-1)\n",
    "#     tflat = target.view(-1)\n",
    "#     intersection = (iflat * tflat).sum()\n",
    "#     return ((2.0 * intersection + smooth) / (iflat.sum() + tflat.sum() + smooth))\n",
    "\n",
    "\n",
    "# class FocalLoss(nn.Module):\n",
    "#     def __init__(self, gamma):\n",
    "#         super().__init__()\n",
    "#         self.gamma = gamma\n",
    "\n",
    "#     def forward(self, input, target):\n",
    "#         if not (target.size() == input.size()):\n",
    "#             raise ValueError(\"Target size ({}) must be the same as input size ({})\"\n",
    "#                              .format(target.size(), input.size()))\n",
    "#         max_val = (-input).clamp(min=0)\n",
    "#         loss = input - input * target + max_val + \\\n",
    "#             ((-max_val).exp() + (-input - max_val).exp()).log()\n",
    "#         invprobs = F.logsigmoid(-input * (target * 2.0 - 1.0))\n",
    "#         loss = (invprobs * self.gamma).exp() * loss\n",
    "#         return loss.mean()\n",
    "\n",
    "\n",
    "# class MixedLoss(nn.Module):\n",
    "#     def __init__(self, alpha, gamma):\n",
    "#         super().__init__()\n",
    "#         self.alpha = alpha\n",
    "#         self.focal = FocalLoss(gamma)\n",
    "\n",
    "#     def forward(self, input, target):\n",
    "#         loss = self.alpha * self.focal(input, target) - torch.log(dice_loss(input, target))\n",
    "#         return loss.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:45.173310Z",
     "iopub.status.busy": "2020-11-03T09:16:45.146587Z",
     "iopub.status.idle": "2020-11-03T09:16:45.175771Z",
     "shell.execute_reply": "2020-11-03T09:16:45.176346Z"
    },
    "papermill": {
     "duration": 0.088253,
     "end_time": "2020-11-03T09:16:45.176538",
     "exception": false,
     "start_time": "2020-11-03T09:16:45.088285",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Functions for the training process\n",
    "\n",
    "class NetSolver(object):\n",
    "\n",
    "    def __init__(self, model, criterion, optimizer, scheduler, print_freq, filepath):\n",
    "        self.model = model\n",
    "        self.criterion = criterion\n",
    "        self.optimizer = optimizer\n",
    "        self.scheduler = scheduler\n",
    "        self.print_freq = print_freq\n",
    "        self.filepath = filepath\n",
    "\n",
    "        self.model = self.model.to(device)\n",
    "        self.criterion = self.criterion.to(device)\n",
    "        self._reset()\n",
    "\n",
    "    def _reset(self):\n",
    "        \"\"\"Set up some book-keeping variables for optimization.\n",
    "        \"\"\"\n",
    "        self.best_val_loss = 1e4\n",
    "        self.best_val_dice = 0.\n",
    "        self.loss_history = []\n",
    "        self.val_loss_history = []\n",
    "        self.dice_history = []\n",
    "        self.val_dice_history = []\n",
    "        self.models = {}\n",
    "\n",
    "    def save_checkpoint(self, iteration):\n",
    "        \"\"\"Save model checkpoint.\n",
    "        \"\"\"\n",
    "#         self.models[f'ckpt_{iteration}'] = self.model.state_dict()\n",
    "        self.model_state = self.model.state_dict()\n",
    "    \n",
    "    def save_final_state(self):\n",
    "        \"\"\"Save final states.\n",
    "        \"\"\"\n",
    "        state = {'model': self.model_state,\n",
    "                 'optimizer': self.optimizer.state_dict()\n",
    "                 }\n",
    "        torch.save(state, self.filepath)\n",
    "\n",
    "    def forward_pass(self, im, gt):\n",
    "        \"\"\"Forward pass through the network.\n",
    "        \"\"\"\n",
    "        im, gt = im.to(device=device, dtype=torch.float), gt.to(device=device, dtype=torch.float)\n",
    "        scores = self.model(im)\n",
    "        loss = self.criterion(scores, gt)\n",
    "        return loss, torch.sigmoid(scores)\n",
    "\n",
    "    def train(self, loaders, iterations, start_time):\n",
    "        \"\"\"Weight of network updated by apex, grad accumulation, model checkpoint.\n",
    "        \"\"\"\n",
    "        train_loader, val_loader = loaders\n",
    "        loader = iter(train_loader)\n",
    "        running_loss = 0.\n",
    "        self.optimizer.zero_grad()\n",
    "\n",
    "        # start training for iterations\n",
    "        for i in range(iterations):\n",
    "            self.model.train()\n",
    "\n",
    "            try:\n",
    "                im, gt = next(loader)\n",
    "            except:\n",
    "                loader = iter(train_loader)\n",
    "                im, gt = next(loader)\n",
    "            loss, _ = self.forward_pass(im, gt)\n",
    "            \n",
    "#             loss.backward()\n",
    "            with amp.scale_loss(loss, self.optimizer) as scaled_loss:\n",
    "                scaled_loss.backward()\n",
    "\n",
    "            # gradient accumulation for larger batch size effect with smaller memory usage\n",
    "            if (i+1) % grad_accumulation_steps == 0:   # Wait for several backward steps\n",
    "                self.optimizer.step()                  # Now we can do an optimizer step\n",
    "                self.optimizer.zero_grad()\n",
    "                self.scheduler.step()\n",
    "\n",
    "            running_loss += loss.item()\n",
    "            \n",
    "            # verbose and checkpoint\n",
    "            if (i+1) % self.print_freq == 0 or (i+1) == iterations:\n",
    "                print(f'Iteration {i+1}:')\n",
    "                train_dice, train_rdice, _ = self.check_metric(train_loader, num_batches=50)\n",
    "                print('{\"metric\": \"Loss\", \"value\": %.4f}' % (running_loss/(i+1),))\n",
    "                print('{\"metric\": \"DICE\", \"value\": %.4f}' % (train_dice,))\n",
    "                print('{\"metric\": \"RDICE\", \"value\": %.4f}' % (train_rdice,))\n",
    "                \n",
    "                is_print = False if (i+1) < iterations else True\n",
    "                val_dice, val_rdice, val_loss = self.check_metric(val_loader)\n",
    "                print('{\"metric\": \"Val. Loss\", \"value\": %.4f}' % (val_loss,))\n",
    "                print('{\"metric\": \"Val. DICE\", \"value\": %.4f}' % (val_dice,))\n",
    "                print('{\"metric\": \"Val. RDICE\", \"value\": %.4f}' % (val_rdice,))\n",
    "                \n",
    "                self.loss_history.append(running_loss/(i+1))\n",
    "                self.val_loss_history.append(val_loss)\n",
    "                self.dice_history.append(train_dice)\n",
    "                self.val_dice_history.append(val_dice)\n",
    "                \n",
    "                if val_loss < self.best_val_loss:\n",
    "                    print('updating best val loss...')\n",
    "                    self.best_val_loss = val_loss\n",
    "                    self.save_checkpoint(i+1)\n",
    "                if val_dice > self.best_val_dice:\n",
    "                    print('updating best val auc...')\n",
    "                    self.best_val_dice = val_dice\n",
    "                \n",
    "                torch.cuda.empty_cache()\n",
    "                print()\n",
    "                \n",
    "            if (time.time() - start_time) > 29000:\n",
    "                break\n",
    "        \n",
    "        self.save_final_state()\n",
    "\n",
    "    def check_metric(self, loader, num_batches=None):\n",
    "        \"\"\"Calculate metrics for validation\n",
    "        \"\"\"\n",
    "        self.model.eval()\n",
    "        losses, dices, rdices = [], [], []\n",
    "        with torch.no_grad():\n",
    "            for t, (im, gt) in enumerate(loader):\n",
    "                l, probs = self.forward_pass(im, gt)\n",
    "                losses.append(l.item())\n",
    "                dices.append(compute_dice((probs.cpu().numpy()>=0.5),\n",
    "                                          gt.cpu().numpy()))\n",
    "                rdices.append(compute_dice(post_process_val(probs.cpu().numpy(), 0.5, 5000),\n",
    "                                           gt.cpu().numpy()))\n",
    "                if num_batches is not None and (t+1) == num_batches:\n",
    "                    break\n",
    "\n",
    "        loss = np.mean(losses)\n",
    "        dice = np.mean(dices)\n",
    "        rdice = np.mean(rdices)\n",
    "\n",
    "        return dice, rdice, loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:45.296363Z",
     "iopub.status.busy": "2020-11-03T09:16:45.280624Z",
     "iopub.status.idle": "2020-11-03T09:16:45.312756Z",
     "shell.execute_reply": "2020-11-03T09:16:45.313311Z"
    },
    "papermill": {
     "duration": 0.097558,
     "end_time": "2020-11-03T09:16:45.313495",
     "exception": false,
     "start_time": "2020-11-03T09:16:45.215937",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class OneCycleScheduler(object):\n",
    "    # one-cycle scheduler\n",
    "    SCHEDULES = set(['cosine', 'linear', 'linear_cosine'])\n",
    "\n",
    "    def __init__(self, optimizer, iterations, sched_profile='cosine', max_lr=3e-3,\n",
    "                 moms=(.95, .85), div_factor=25, warmup=0.3, final_div=None):\n",
    "\n",
    "        self.optimizer = optimizer\n",
    "        assert sched_profile in self.SCHEDULES\n",
    "        self.sched_profile = sched_profile\n",
    "\n",
    "        if isinstance(max_lr, list) or isinstance(max_lr, tuple):\n",
    "            if len(max_lr) != len(optimizer.param_groups):\n",
    "                raise ValueError(\"expected {} max_lr, got {}\".format(\n",
    "                    len(optimizer.param_groups), len(max_lr)))\n",
    "            self.max_lrs = list(max_lr)\n",
    "            self.init_lrs = [lr/div_factor for lr in self.max_lrs]\n",
    "        else:\n",
    "            self.max_lrs = [max_lr] * len(optimizer.param_groups)\n",
    "            self.init_lrs = [max_lr/div_factor] * len(optimizer.param_groups)\n",
    "\n",
    "        self.final_div = final_div\n",
    "        if self.final_div is None: self.final_div = div_factor*1e4\n",
    "        self.final_lrs = [lr/self.final_div for lr in self.max_lrs]\n",
    "        self.moms = moms\n",
    "\n",
    "        self.total_iteration = iterations\n",
    "        self.up_iteration = int(self.total_iteration * warmup)\n",
    "        self.down_iteration = self.total_iteration - self.up_iteration\n",
    "\n",
    "        self.curr_iter = 0\n",
    "        self._assign_lr_mom(self.init_lrs, [moms[0]]*len(optimizer.param_groups))\n",
    "\n",
    "    def _assign_lr_mom(self, lrs, moms):\n",
    "        for param_group, lr, mom in zip(self.optimizer.param_groups, lrs, moms):\n",
    "            param_group['lr'] = lr\n",
    "            param_group['betas'] = (mom, 0.999)\n",
    "\n",
    "    def _annealing_cos(self, start, end, pct):\n",
    "        cos_out = np.cos(np.pi * pct) + 1\n",
    "        return end + (start-end)/2 * cos_out\n",
    "\n",
    "    def _annealing_linear(self, start, end, pct):\n",
    "        return start + pct * (end-start)\n",
    "    \n",
    "    def _annealing_function(self, curr_iter):\n",
    "        if self.sched_profile == 'cosine':\n",
    "            return self._annealing_cos\n",
    "        if self.sched_profile == 'linear':\n",
    "            return self._annealing_linear\n",
    "        if self.sched_profile == 'linear_cosine':\n",
    "            if curr_iter <= self.up_iteration:\n",
    "                return self._annealing_linear\n",
    "            else:\n",
    "                return self._annealing_cos\n",
    "    \n",
    "    def step(self):\n",
    "        self.curr_iter += 1\n",
    "        anneal = self._annealing_function(self.curr_iter)\n",
    "\n",
    "        if self.curr_iter <= self.up_iteration:\n",
    "            pct = self.curr_iter / self.up_iteration\n",
    "            curr_lrs = [anneal(min_lr, max_lr, pct) \\\n",
    "                            for min_lr, max_lr in zip(self.init_lrs, self.max_lrs)]\n",
    "            curr_moms = [anneal(self.moms[0], self.moms[1], pct) \\\n",
    "                            for _ in range(len(self.optimizer.param_groups))]\n",
    "        else:\n",
    "            pct = (self.curr_iter-self.up_iteration) / self.down_iteration\n",
    "            curr_lrs = [anneal(max_lr, final_lr, pct) \\\n",
    "                            for max_lr, final_lr in zip(self.max_lrs, self.final_lrs)]\n",
    "            curr_moms = [anneal(self.moms[1], self.moms[0], pct) \\\n",
    "                            for _ in range(len(self.optimizer.param_groups))]\n",
    "\n",
    "        self._assign_lr_mom(curr_lrs, curr_moms)\n",
    "\n",
    "def lr_range_test(train_loader, model, optimizer, criterion, start_lr=1e-7,\n",
    "                  end_lr=10, num_it=100, stop_div=True):\n",
    "    epochs = int(np.ceil(num_it/len(train_loader)))\n",
    "    n_groups = len(optimizer.param_groups)\n",
    "\n",
    "    if isinstance(start_lr, list) or isinstance(start_lr, tuple):\n",
    "        if len(start_lr) != n_groups:\n",
    "            raise ValueError(\"expected {} max_lr, got {}\".format(n_groups, len(start_lr)))\n",
    "        start_lrs = list(start_lr)\n",
    "    else:\n",
    "        start_lrs = [start_lr] * n_groups\n",
    "\n",
    "    if isinstance(end_lr, list) or isinstance(end_lr, tuple):\n",
    "        if len(end_lr) != n_groups:\n",
    "            raise ValueError(\"expected {} max_lr, got {}\".format(n_groups, len(end_lr)))\n",
    "        end_lrs = list(end_lr)\n",
    "    else:\n",
    "        end_lrs = [end_lr] * n_groups\n",
    "\n",
    "    curr_lrs = start_lrs*1\n",
    "    for param_group, lr in zip(optimizer.param_groups, curr_lrs):\n",
    "        param_group['lr'] = lr\n",
    "\n",
    "    n, lrs_logs, loss_log = 0, [], []\n",
    "\n",
    "    for e in range(epochs):\n",
    "        model.train()\n",
    "        for x, y in train_loader:\n",
    "            x, y = x.to(device=device, dtype=torch.float), y.to(device=device, dtype=torch.float)\n",
    "            scores = model(x)\n",
    "            loss = criterion(scores, y)\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "#             loss.backward()\n",
    "            with amp.scale_loss(loss, optimizer) as scaled_loss:\n",
    "                scaled_loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            lrs_logs.append(curr_lrs)\n",
    "            loss_log.append(loss.item())\n",
    "\n",
    "            # update best loss\n",
    "            if n == 0:\n",
    "                best_loss, n_best = loss.item(), n\n",
    "            else:\n",
    "                if loss.item() < best_loss:\n",
    "                    best_loss, n_best = loss.item(), n\n",
    "\n",
    "            # update lr per iter with exponential schedule\n",
    "            n += 1\n",
    "            curr_lrs = [lr * (end_lr/lr) ** (n/num_it) for lr, end_lr in zip(start_lrs, end_lrs)]\n",
    "            for param_group, lr in zip(optimizer.param_groups, curr_lrs):\n",
    "                param_group['lr'] = lr\n",
    "\n",
    "            # stopping condition\n",
    "            if n == num_it or (stop_div and (loss.item() > 4*best_loss or torch.isnan(loss))):\n",
    "                break\n",
    "\n",
    "    print('minimum loss {}, at lr {}'.format(best_loss, lrs_logs[n_best]))\n",
    "    return lrs_logs, loss_log"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.038947,
     "end_time": "2020-11-03T09:16:45.392164",
     "exception": false,
     "start_time": "2020-11-03T09:16:45.353217",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Model training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:45.496613Z",
     "iopub.status.busy": "2020-11-03T09:16:45.495337Z",
     "iopub.status.idle": "2020-11-03T09:16:45.503347Z",
     "shell.execute_reply": "2020-11-03T09:16:45.502602Z"
    },
    "papermill": {
     "duration": 0.07165,
     "end_time": "2020-11-03T09:16:45.503491",
     "exception": false,
     "start_time": "2020-11-03T09:16:45.431841",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_dataset = CloudDataset(dataset_type='train', img_ids=train_ids,\n",
    "                             transforms=get_training_augmentation(),\n",
    "                             preprocessing=get_preprocessing(preprocessing_fn))\n",
    "\n",
    "valid_dataset = CloudDataset(dataset_type='valid', img_ids=valid_ids,\n",
    "                             transforms=get_validation_augmentation(),\n",
    "                             preprocessing=get_preprocessing(preprocessing_fn))\n",
    "\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True,\n",
    "                          num_workers=4, pin_memory=True)\n",
    "valid_loader = DataLoader(valid_dataset, batch_size=batch_size, shuffle=False,\n",
    "                          num_workers=4, pin_memory=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "_kg_hide-output": true,
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:45.621008Z",
     "iopub.status.busy": "2020-11-03T09:16:45.620126Z",
     "iopub.status.idle": "2020-11-03T09:16:53.646874Z",
     "shell.execute_reply": "2020-11-03T09:16:53.646159Z"
    },
    "papermill": {
     "duration": 8.085907,
     "end_time": "2020-11-03T09:16:53.647015",
     "exception": false,
     "start_time": "2020-11-03T09:16:45.561108",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building model and optimizer...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/resnet50-19c8e357.pth\" to /root/.cache/torch/hub/checkpoints/resnet50-19c8e357.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e84e611225a44b148af31f868b88c0d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=102502400.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Each allocated and cached memory in cuda (will clean cache):\n",
      "131.667968M\n",
      "155.189248M\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/torch/cuda/memory.py:346: FutureWarning: torch.cuda.memory_cached has been renamed to torch.cuda.memory_reserved\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# model setup\n",
    "seed_torch(SEED)\n",
    "torch.cuda.empty_cache()\n",
    "\n",
    "num_train_steps = int(epochs_for_sched * len(train_loader) / grad_accumulation_steps)\n",
    "ft_lrs = [0.0025*lr, 0.05*lr, lr]\n",
    "\n",
    "model, optimizer = model_optimizer_init(ft_lrs)\n",
    "scheduler = OneCycleScheduler(optimizer, num_train_steps, sched_profile='cosine', max_lr=ft_lrs,\n",
    "                              warmup=warmup_proportion)\n",
    "\n",
    "model = model.to(device)\n",
    "criterion = smp.utils.losses.DiceLoss(eps=1., activation='sigmoid').to(device)\n",
    "model, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\", verbosity=0)\n",
    "solver = NetSolver(model, criterion, optimizer, scheduler, checkpoint_iter, output_prefix)\n",
    "\n",
    "print()\n",
    "print('Each allocated and cached memory in cuda (will clean cache):')\n",
    "print(str(torch.cuda.memory_allocated(device)/1e6 ) + 'M')\n",
    "print(str(torch.cuda.memory_cached(device)/1e6 ) + 'M')\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T09:16:53.750348Z",
     "iopub.status.busy": "2020-11-03T09:16:53.742117Z",
     "iopub.status.idle": "2020-11-03T09:48:35.662114Z",
     "shell.execute_reply": "2020-11-03T09:48:35.661389Z"
    },
    "papermill": {
     "duration": 1901.972575,
     "end_time": "2020-11-03T09:48:35.662251",
     "exception": false,
     "start_time": "2020-11-03T09:16:53.689676",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start training.\n",
      "Iteration 200:\n",
      "{\"metric\": \"Loss\", \"value\": 0.6280}\n",
      "{\"metric\": \"DICE\", \"value\": 0.5139}\n",
      "{\"metric\": \"RDICE\", \"value\": 0.0000}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 0.5765}\n",
      "{\"metric\": \"Val. DICE\", \"value\": 0.5069}\n",
      "{\"metric\": \"Val. RDICE\", \"value\": 0.0000}\n",
      "updating best val loss...\n",
      "updating best val auc...\n",
      "\n",
      "Iteration 400:\n",
      "{\"metric\": \"Loss\", \"value\": 0.5819}\n",
      "{\"metric\": \"DICE\", \"value\": 0.4879}\n",
      "{\"metric\": \"RDICE\", \"value\": 0.0000}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 0.5280}\n",
      "{\"metric\": \"Val. DICE\", \"value\": 0.4941}\n",
      "{\"metric\": \"Val. RDICE\", \"value\": 0.0000}\n",
      "updating best val loss...\n",
      "\n",
      "Iteration 600:\n",
      "{\"metric\": \"Loss\", \"value\": 0.5436}\n",
      "{\"metric\": \"DICE\", \"value\": 0.5494}\n",
      "{\"metric\": \"RDICE\", \"value\": 0.0000}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 0.4547}\n",
      "{\"metric\": \"Val. DICE\", \"value\": 0.5560}\n",
      "{\"metric\": \"Val. RDICE\", \"value\": 0.0000}\n",
      "updating best val loss...\n",
      "updating best val auc...\n",
      "\n",
      "Iteration 800:\n",
      "{\"metric\": \"Loss\", \"value\": 0.5174}\n",
      "{\"metric\": \"DICE\", \"value\": 0.5790}\n",
      "{\"metric\": \"RDICE\", \"value\": 0.0000}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 0.4250}\n",
      "{\"metric\": \"Val. DICE\", \"value\": 0.5806}\n",
      "{\"metric\": \"Val. RDICE\", \"value\": 0.0000}\n",
      "updating best val loss...\n",
      "updating best val auc...\n",
      "\n",
      "Iteration 1000:\n",
      "{\"metric\": \"Loss\", \"value\": 0.4993}\n",
      "{\"metric\": \"DICE\", \"value\": 0.5844}\n",
      "{\"metric\": \"RDICE\", \"value\": 0.0000}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 0.4263}\n",
      "{\"metric\": \"Val. DICE\", \"value\": 0.5772}\n",
      "{\"metric\": \"Val. RDICE\", \"value\": 0.0000}\n",
      "\n",
      "Iteration 1200:\n",
      "{\"metric\": \"Loss\", \"value\": 0.4853}\n",
      "{\"metric\": \"DICE\", \"value\": 0.6020}\n",
      "{\"metric\": \"RDICE\", \"value\": 0.0000}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 0.4124}\n",
      "{\"metric\": \"Val. DICE\", \"value\": 0.5899}\n",
      "{\"metric\": \"Val. RDICE\", \"value\": 0.0000}\n",
      "updating best val loss...\n",
      "updating best val auc...\n",
      "\n",
      "Iteration 1400:\n",
      "{\"metric\": \"Loss\", \"value\": 0.4743}\n",
      "{\"metric\": \"DICE\", \"value\": 0.6020}\n",
      "{\"metric\": \"RDICE\", \"value\": 0.0000}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 0.4054}\n",
      "{\"metric\": \"Val. DICE\", \"value\": 0.5966}\n",
      "{\"metric\": \"Val. RDICE\", \"value\": 0.0000}\n",
      "updating best val loss...\n",
      "updating best val auc...\n",
      "\n",
      "Iteration 1600:\n",
      "{\"metric\": \"Loss\", \"value\": 0.4646}\n",
      "{\"metric\": \"DICE\", \"value\": 0.6201}\n",
      "{\"metric\": \"RDICE\", \"value\": 0.0000}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 0.3937}\n",
      "{\"metric\": \"Val. DICE\", \"value\": 0.6082}\n",
      "{\"metric\": \"Val. RDICE\", \"value\": 0.0000}\n",
      "updating best val loss...\n",
      "updating best val auc...\n",
      "\n",
      "Iteration 1800:\n",
      "{\"metric\": \"Loss\", \"value\": 0.4564}\n",
      "{\"metric\": \"DICE\", \"value\": 0.6211}\n",
      "{\"metric\": \"RDICE\", \"value\": 0.0000}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 0.3906}\n",
      "{\"metric\": \"Val. DICE\", \"value\": 0.6111}\n",
      "{\"metric\": \"Val. RDICE\", \"value\": 0.0000}\n",
      "updating best val loss...\n",
      "updating best val auc...\n",
      "\n",
      "Iteration 1946:\n",
      "{\"metric\": \"Loss\", \"value\": 0.4512}\n",
      "{\"metric\": \"DICE\", \"value\": 0.6135}\n",
      "{\"metric\": \"RDICE\", \"value\": 0.0000}\n",
      "{\"metric\": \"Val. Loss\", \"value\": 0.3908}\n",
      "{\"metric\": \"Val. DICE\", \"value\": 0.6109}\n",
      "{\"metric\": \"Val. RDICE\", \"value\": 0.0000}\n",
      "\n",
      "Training complete in 1902 seconds.\n",
      "\n",
      "Each allocated and cached memory in cuda (will clean cache):\n",
      "600.086016M\n",
      "5800.722432M\n"
     ]
    }
   ],
   "source": [
    "n_iter = num_train_steps * grad_accumulation_steps\n",
    "print('Start training.')\n",
    "t0 = time.time()\n",
    "solver.train((train_loader, valid_loader), n_iter, t0)\n",
    "print('Training complete in {:.0f} seconds.'.format(time.time()-t0))\n",
    "\n",
    "print()\n",
    "print('Each allocated and cached memory in cuda (will clean cache):')\n",
    "print(str(torch.cuda.memory_allocated(device)/1e6 ) + 'M')\n",
    "print(str(torch.cuda.memory_cached(device)/1e6 ) + 'M')\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T09:48:35.781281Z",
     "iopub.status.busy": "2020-11-03T09:48:35.780513Z",
     "iopub.status.idle": "2020-11-03T09:48:36.018096Z",
     "shell.execute_reply": "2020-11-03T09:48:36.018757Z"
    },
    "papermill": {
     "duration": 0.300444,
     "end_time": "2020-11-03T09:48:36.018927",
     "exception": false,
     "start_time": "2020-11-03T09:48:35.718483",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmcAAAFlCAYAAACukk+HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd1yX9f7/8cebDxsZ4gZ3uRVByZHHlalpy6zM1d5ZnTrlKc/3l40zqtPeew/NY2pDzZXmSMW9cS/ArSAgm+v3x4WoCIoyrg/wvN9u3IA31/XhRUOevsf1MpZlISIiIiLuwcPpAkRERETkFIUzERERETeicCYiIiLiRhTORERERNyIwpmIiIiIG1E4ExEREXEjnk4XUJpq1qxpNW7c2OkyRERERM5rxYoVhy3LqlVwvFKFs8aNG7N8+XKnyxARERE5L2PM7sLGtawpIiIi4kYUzkRERETciMKZiIiIiBupVHvOREREqoqsrCzi4uJIT093uhQ5D19fX+rXr4+Xl1exrlc4ExERqYDi4uIIDAykcePGGGOcLkeKYFkWR44cIS4ujiZNmhTrHi1rioiIVEDp6enUqFFDwczNGWOoUaPGBc1wKpyJiIhUUApmFcOF/ntSOBMREZELlpiYyPvvv39R9w4cOJDExMRzXjN27Fhmz559Ua9fUOPGjTl8+HCpvFZ50J4zERGRKmDKqnhembGZhMQ0wkL8GN2/BYOiwi/69U6Gs4ceeuisr+Xk5OByuYq8d9q0aed9/RdeeOGia6voNHNWTFNWxdPtpd9p8vRUur30O1NWxTtdkoiISLFMWRXPmEnriE9MwwLiE9MYM2ldiX6XPf3002zfvp3IyEhGjx7NvHnz6N27N8OHD6ddu3YADBo0iI4dO9KmTRs+/vjj/HtPzmTt2rWLVq1ace+999KmTRv69etHWloaAHfccQcTJ07Mv/7ZZ5+lQ4cOtGvXjtjYWAAOHTpE37596dChA/fffz+NGjU67wzZ66+/Ttu2bWnbti1vvvkmAKmpqVx99dW0b9+etm3b8sMPP+T/jK1btyYiIoInn3zyov9ZXSjNnBXDyf+o07JygFP/UQMl+luHiIhIaXj+lw1sTDhe5NdX7UkkMyf3jLG0rBz+PnEt42L2FHpP67Agnr22TZGv+dJLL7F+/XpWr14NwLx584iJiWH9+vX5pxI///xzQkNDSUtL47LLLuPGG2+kRo0aZ7zO1q1bGTduHJ988glDhgzhxx9/ZOTIkWd9v5o1a7Jy5Uref/99Xn31VT799FOef/55rrjiCsaMGcNvv/12RgAszIoVK/jiiy9YunQplmXRuXNnevbsyY4dOwgLC2Pq1KkAJCUlcfToUSZPnkxsbCzGmPMuw5YmzZwVwyszNucHs5PSsnJ4ZcZmhyoSEREpvoLB7HzjF6tTp05nPC7i7bffpn379nTp0oW9e/eydevWs+5p0qQJkZGRAHTs2JFdu3YV+tqDBw8+65qFCxcydOhQAK666iqqV69+zvoWLlzIDTfcQEBAANWqVWPw4MEsWLCAdu3aMXv2bJ566ikWLFhAcHAwQUFB+Pr6cs899zBp0iT8/f0v9B/HRdPMWTEkJKZd0LiIiEh5OtcMF0C3l34nvpDfWeEhfvxwf9dSqyMgICD/43nz5jF79mwWL16Mv78/vXr1KvRxEj4+Pvkfu1yu/GXNoq5zuVxkZ2cD9jPELkRR1zdv3pwVK1Ywbdo0xowZQ79+/Rg7diwxMTHMmTOH8ePH8+677/L7779f0Pe7WJo5K4awEL8LGhcREXEno/u3wM/rzA36fl4uRvdvcdGvGRgYSHJycpFfT0pKonr16vj7+xMbG8uSJUsu+nsV5S9/+QsTJkwAYObMmRw7duyc1/fo0YMpU6Zw4sQJUlNTmTx5Mt27dychIQF/f39GjhzJk08+ycqVK0lJSSEpKYmBAwfy5ptv5i/flgfNnBXD6P4tzthzBmCAe7oX70m/IiIiTjq5P7o0T2vWqFGDbt260bZtWwYMGMDVV199xtevuuoqPvzwQyIiImjRogVdunQp0c9QmGeffZZhw4bxww8/0LNnT+rVq0dgYGCR13fo0IE77riDTp06AXDPPfcQFRXFjBkzGD16NB4eHnh5efHBBx+QnJzM9ddfT3p6OpZl8cYbb5R6/UUxFzol6M6io6Ot5cuXl8lrn34EuWagD8dPZFIvxI/x93WlbrBvmXxPERGRomzatIlWrVo5XYajMjIycLlceHp6snjxYh588MFyneG6EIX9+zLGrLAsK7rgtZo5K6ZBUeFn/A1j5Z5j3PZZDMM/XcL4+7pQO1ABTUREpDzt2bOHIUOGkJubi7e3N5988onTJZUKhbOL1KFhdb688zJu+zyGEZ8sZdx9XahZzef8N4qIiEipaNasGatWrXK6jFKnAwElEN04lM/vuIy9x04w8tOlHE3NdLokERERqeAUzkqoS9MafHb7Zew8nMrIT5eSeEIBTURERC6ewlkp6HZpTT6+LZptB1O47fMYktKynC5JREREKiiFs1LSs3ktPry1A5v2Hef2z2NITldAExERkQuncFaKrmhZh/eGd2B9fBJ3frGM1Ixsp0sSERFxG9WqVQMgISGBm266qdBrevXqxfkei/Xmm29y4sSJ/M8HDhxYKr0vn3vuOV599dUSv05JKZyVsn5t6vLOsChW7U3kri+XcSJTAU1ERNzA2gnwRlt4LsR+v3aCY6WEhYUxceLEi76/YDibNm0aISEhpVGaW1A4KwMD2tXjjVsiWbbrKPd+vZz0Ak3TRUREytXaCfDLo5C0F7Ds9788WqKA9tRTT/H+++/nf/7cc8/x2muvkZKSQp8+fejQoQPt2rXjp59+OuveXbt20bZtWwDS0tIYOnQoERER3HLLLWf01nzwwQeJjo6mTZs2PPvss4DdTD0hIYHevXvTu3dvABo3bszhw4cBeP3112nbti1t27blzTffzP9+rVq14t5776VNmzb069evyB6eJ61evZouXboQERHBDTfckN8a6u2336Z169ZERETkN13/448/iIyMJDIykqioqHO2tSoOPeesjFzXPoyc3Fz+NmEN932zgo9v7Yhvgb5mIiIipWL607B/XdFfj1sGORlnjmWlwU8Pw4qvCr+nbjsY8FKRLzl06FAee+wxHnroIQAmTJjAb7/9hq+vL5MnTyYoKIjDhw/TpUsXrrvuOowxhb7OBx98gL+/P2vXrmXt2rV06NAh/2v//ve/CQ0NJScnhz59+rB27VoeffRRXn/9debOnUvNmjXPeK0VK1bwxRdfsHTpUizLonPnzvTs2ZPq1auzdetWxo0bxyeffMKQIUP48ccfGTlyZJE/32233cY777xDz549GTt2LM8//zxvvvkmL730Ejt37sTHxyd/KfXVV1/lvffeo1u3bqSkpODrW7IH02vmrAzdEFWfl2+MYP6WQzz03UoysjWDJiIiDigYzM43XgxRUVEcPHiQhIQE1qxZQ/Xq1WnYsCGWZfGPf/yDiIgIrrzySuLj4zlw4ECRrzN//vz8kBQREUFERET+1yZMmECHDh2Iiopiw4YNbNy48Zw1LVy4kBtuuIGAgACqVavG4MGDWbBgAQBNmjQhMjISgI4dO7Jr164iXycpKYnExER69uwJwO233878+fPzaxwxYgTffvstnp72HFe3bt3429/+xttvv01iYmL++MXSzFkZGxLdgOwci39MXsfD36/i/REd8HIpE4uISCk6xwwXYO8xS9p79nhwA7hz6kV/25tuuomJEyeyf//+/CW+7777jkOHDrFixQq8vLxo3Lgx6enp53ydwmbVdu7cyauvvsqyZcuoXr06d9xxx3lf51z9wn18TnXxcblc513WLMrUqVOZP38+P//8M//85z/ZsGEDTz/9NFdffTXTpk2jS5cuzJ49m5YtW17U64NmzsrF8M4NeeH6NszaeIC/jl9Fdk6u0yWJiEhV0mcsePmdOeblZ4+XwNChQxk/fjwTJ07MP32ZlJRE7dq18fLyYu7cuezevfucr9GjRw++++47ANavX8/atWsBOH78OAEBAQQHB3PgwAGmT5+ef09gYGCh+7p69OjBlClTOHHiBKmpqUyePJnu3btf8M8VHBxM9erV82fdvvnmG3r27Elubi579+6ld+/e/Pe//yUxMZGUlBS2b99Ou3bteOqpp4iOjiY2NvaCv+fpNHNWTm7r2pisHIt//roRl8ca3hjSHk/NoImISHmIGGK/n/MCJMVBcH07mJ0cv0ht2rQhOTmZ8PBw6tWrB8CIESO49tpriY6OJjIy8rwzSA8++CB33nknERERREZG0qlTJwDat29PVFQUbdq0oWnTpnTr1i3/nvvuu48BAwZQr1495s6dmz/eoUMH7rjjjvzXuOeee4iKijrnEmZRvvrqKx544AFOnDhB06ZN+eKLL8jJyWHkyJEkJSVhWRaPP/44ISEhPPPMM8ydOxeXy0Xr1q0ZMGDABX+/05lzTQFWNNHR0db5no3itI/+2M6L02MZHBXOKze3x+VR+AZJERGRc9m0aROtWrVyugwppsL+fRljVliWFV3wWs2clbP7e15Cdq7FKzM24/IwvHxjBB4KaCIiIpJH4cwBo3pfSmZ2Lm/N2Yqny4N/D2qrgCYiIiKAwpljHruyGdm5ubw3dzteLsPz17Up8hkwIiIiUnUonDnEGMOT/VqQlWPx8fwdeHp48Mw1rRTQRESk2CzL0u+NCuBC9/crnDnIGMOYAS3Jysnl80U78XIZnh7QUv+jiYjIefn6+nLkyBFq1Kih3xtuzLIsjhw5ckFdAxTOHGaMYew1rcnOsfho/g68XB480a+5/kcTEZFzql+/PnFxcRw6dMjpUuQ8fH19qV+/frGvVzhzA8bYe86yc3N5d+42PF2Gx65s7nRZIiLixry8vGjSpInTZUgZUDhzEx4ehn8Pakd2jsWbs7fi5fJgVO9LnS5LREREypnCmRvx8DC8dGNE/nPQvFyG+3pc4nRZIiIiUo4UztyMy8Pwyk12QPvPtFhcHh7c/RdNW4uIiFQVCmduyNPlwetD2pOdk8s/f92Il8twW9fGTpclIiIi5UCdt92Ul8uDt4ZGcWWrOoz9aQPfL93jdEkiIiJSDhTO3Ji3pwfvjYiid4ta/GPyOiYs3+t0SSIiIlLGyjScGWOuMsZsNsZsM8Y8XcQ1vYwxq40xG4wxf1zIvVWBj6eLD0Z2pHuzmjz141omr4pzuiQREREpQ2UWzowxLuA9YADQGhhmjGld4JoQ4H3gOsuy2gA3F/feqsTXy8Unt0XTtWkNnpiwhp/XJDhdkoiIiJSRspw56wRssyxrh2VZmcB44PoC1wwHJlmWtQfAsqyDF3BvleLr5eLT26OJbhzK4z+sZvq6fU6XJCIiImWgLMNZOHD6Jqm4vLHTNQeqG2PmGWNWGGNuu4B7ATDG3GeMWW6MWV7ZW1j4e3vy+R2XEdkghEfGrWLmhv1OlyQiIiKlrCzDWWHNIQu2ZfcEOgJXA/2BZ4wxzYt5rz1oWR9blhVtWVZ0rVq1SlJvhVDNx5Mv77yMtuHBjPp+Jb/HHnC6JBERESlFZRnO4oAGp31eHyi4WSoO+M2yrFTLsg4D84H2xby3ygr09eKruzrRsm4QD3yzkj+2VO4ZQxERkaqkLMPZMqCZMaaJMcYbGAr8XOCan4DuxhhPY4w/0BnYVMx7q7RgPy++ubsTl9auxn1fL2fRtsNOlyQiIiKloMzCmWVZ2cDDwAzswDXBsqwNxpgHjDEP5F2zCfgNWAvEAJ9alrW+qHvLqtaKKsTfm2/v6UzjGgHc/dUyluw44nRJIiIiUkLGsgrdylUhRUdHW8uXL3e6jHJ3OCWDoR8vISExja/v6kR041CnSxIREZHzMMassCwruuC4OgRUAjWr+fD9PZ2pG+TLHV8sY+WeY06XJCIiIhdJ4aySqB3ky/f3dqFGNW9u/zyGtXGJTpckIiIiF0HhrBKpG2wHtGA/L279LIb18UlOlyQiIiIXSOGskgkP8WPcvV2o5uPJrZ8tJXb/cadLEhERkQugcFYJNQj15/t7O+Pj6WLEJ0vZeiDZ6ZJERESkmBTOKqlGNQL4/t7OuDwMwz5ZyvZDKU6XJCIiIsWgcFaJNa1Vje/v7QxYDP9kCbsOpzpdkoiIiJyHwlkld2ntQL67pwtZOXZA23v0hNMliYiIyDkonFUBLeoG8u3dnTmRlcPQj5cQd0wBTURExF0pnFURrcOC+PbuziSnZzH8k6XsS0pzuiQREREphMJZFdI2PJiv7+7MsdRMhn+ylAPH050uSURERApQOKtiIhuE8OVdl3HweDrDP1nCoeQMp0sSERGR0yicVUEdG4XyxZ2dSEhMZ8SnSziSooAmIiLiLhTOqqhOTUL57I5o9hw9wYhPl3IsNdPpkkRERAQwlmU5XUOpiY6OtpYvX+50GRXKgq2HuPur5dSq5k2uBfuT0gkL8WN0/xYMigp3ujwREZFKyxizwrKs6ILjmjmr4ro3q8UdlzciPjGdfUnpWEB8YhpjJq1jyqp4p8sTERGpchTOhKlr9581lpaVwyszNjtQjYiISNWmcCYkJBb+zLOixkVERKTsKJwJYSF+hY7XqOZdzpWIiIiIwpkwun8L/LxcZ4wZ4EhKJj8s2+NMUSIiIlWUwpkwKCqcFwe3IzzEDwOEh/jxn8Ft+Uuzmjz14zpenLaJ3NzKc6pXRETEnXk6XUCFsXYCzHkBkuIguD70GQsRQ5yuqtQMigo/69EZN3dswPO/bOSj+TvYeTiVN4dG4u+t/2RERETKkmbOimPtBPjlUUjaC1j2+18etccrMU+XBy9c34Znr23N7E0HGPLRYvYnqR+niIhIWVI4K445L0BWgZOLWWn2eCVnjOHObk349PZodh5K5fr3FrI+PsnpskRERCothbPiSIq7sPFK6IqWdZj44OW4jOHmDxczc8PZz0YTERGRklM4K47g+hc2Xkm1qhfElIe70bxuIPd/u4KP52+nMrX/EhERcQcKZ8XRZyx4FfIssLCo8q/FYbUDffnhvi4MbFuP/0yLZcykdWTl5DpdloiISKWhcFYcEUPg2rchuAFg7BmzhpfDpp9hyQdOV1fufL1cvDMsiod7X8r4ZXu5/fMYkk5kOV2WiIhIpWAq07JUdHS0tXz58vL5ZjnZ8L/bIfZXGPQhRA4rn+/rZn5cEcfTk9bSINSfz2+/jMY1A5wuSUREpEIwxqywLCu64Lhmzi6WyxNu/Aya9ISfRkHsVKcrcsSNHevz3T1dOJaayaD3F7F0xxGnSxIREanQFM5KwssXhn4HYZHwvzth53ynK3JEpyahTH6oG6EB3oz8bCk/rqg6p1hFRERKm8JZSfkEwoiJENoUxg2D+BVOV+SIxjUDmPxgNy5rHMoT/1vDKzNi1fJJRETkIiiclQb/ULh1MvjXgG9vhIOxTlfkiGB/L766qxPDOjXgvbnbeXjcStIyc5wuS0REpEJROCstQfXgting8oZvBsGx3U5X5Agvlwf/uaEd/zewFdPX72fox4s5mKyWTyIiIsWlcFaaQpvaM2hZJ+yAlnzA6YocYYzh3h5N+WhkR7YcSGHQu4vYtO+402WJiIhUCApnpa1OG3sPWvJ++HYwpB1zuiLH9GtTl/890JVcC2764E9+j62aYVVERORCKJyVhQad7FOchzbD97dAZqrTFTmmbXgwU0Z1o0mtAO75ajmfL9yplk8iIiLnoHBWVi65Am76DOKWwQ+3Qnam0xU5pm6wLxPu78qVrerwwq8beean9WSr5ZOIiEihFM7KUuvr7bZP2+fApHsht+qeXPT39uTDkR25v2dTvl2yhzu/XMbxdLV8EhERKUjhrKx1uBX6/Qs2ToFfH4cqvKTn4WEYM6AVL9/YjsXbj3Dj+3+y9+gJp8sSERFxKwpn5eHyR6D7E7DyK5j9rNPVOO6Wyxry9d2dOJicwaD3FrFi91GnSxIREXEbCmfl5YpnIPpuWPQWLHzD6Wocd/klNZn80OUE+noy7JOl/LQ63umSRERE3ILCWXkxBga+Cm1vgtnPwfLPna7IcU1rVWPyQ92IbBDCX8ev5o1ZW3SSU0REqjyFs/Lk4QE3fAjN+sOvf4P1PzpdkeOqB3jz7d2dualjfd6as5W/jl9NelbVPTghIiKicFbeXF5w85fQsCtMug+2zna6Isd5e3rwyk0R/P2qFvy8JoHhnyzhcEqG02WJiIg4QuHMCd7+MHw81G4NP4yE3Yudrshxxhge6nUpH4zowMZ9xxn03iK2HEh2uiwREZFyp3DmFN9gGDkJgsPtLgL71jpdkVsY0K4eP9zXlYzsXG58/0/+2HLI6ZJERETKlcKZk6rVglungE+g3Yfz8DanK3IL7RuE8NOobtQP9eeuL5fxzeJdTpckIiJSbhTOnBbSAG6bAlYufDMIkvRICYCwED/+90BXejWvxTM/beC5nzeQk6uTnCIiUvkpnLmDms3sJc60RDugpR5xuiK3UM3Hk49vi+buvzThyz93cc9Xy0jJyHa6LBERkTKlcOYuwiJh+A+QuMde4kw/7nRFbsHlYXjmmtb8a1Bb5m89zE0f/El8YprTZYmIiJQZhTN30rgbDPkaDqyHccMgSyHkpJFdGvHlnZcRn5jG9e8uYvXeRKdLEhERKRMKZ+6meX8Y9CHsXgT/uxNyspyuyG10b1aLSQ9ejp+3B7d8tJipa/c5XZKIiEipK9NwZoy5yhiz2RizzRjzdCFf72WMSTLGrM57G3va13YZY9bljS8vyzrdTsTNcPWrsGU6/DQKcnOdrshtNKsTyJSHutEuPJhR36/k3d+3quWTiIhUKp5l9cLGGBfwHtAXiAOWGWN+tixrY4FLF1iWdU0RL9PbsqzDZVWjW7vsHkg7Br//C3xDYMDLdn9OoUY1H769pzNP/7iWV2duYcfhVF4c3A4fT5fTpYmIiJRYmYUzoBOwzbKsHQDGmPHA9UDBcCZF6f6kfYJz8bvgFwK9/+F0RW7D18vFG7dE0rRWNV6ftYW4o2l8eGtHQgO8nS5NRESkRMpyWTMc2Hva53F5YwV1NcasMcZMN8a0OW3cAmYaY1YYY+4rwzrdlzHQ718QNRL+eBkWv+90RW7FGMOjfZrx9rAoVsclcsP7i9h2MMXpskREREqkLMNZYWtwBTcHrQQaWZbVHngHmHLa17pZltUBGACMMsb0KPSbGHOfMWa5MWb5oUOVsNWPMXDNW9DqWpgxBlZ/73RFbue69mGMv68LqRnZDH5/EYu2Vc2VcBERqRzKMpzFAQ1O+7w+kHD6BZZlHbcsKyXv42mAlzGmZt7nCXnvDwKTsZdJz2JZ1seWZUVblhVdq1at0v8p3IHLE278DJr2gp8ehk2/Ol2R2+nQsDqTH+pG3WBfbv88hnExe5wuSURE5KKUZThbBjQzxjQxxngDQ4GfT7/AGFPXGHuXuzGmU149R4wxAcaYwLzxAKAfsL4Ma3V/nj5wy3cQFgUT74Qd85yuyO00CPXnxwcvp9ulNRkzaR3/nrpRLZ9ERKTCKbNwZllWNvAwMAPYBEywLGuDMeYBY8wDeZfdBKw3xqwB3gaGWvZzEeoAC/PGY4CplmX9Vla1Vhg+1WDE/6DGpTBuOMStcLoitxPo68Vnt0dze9dGfLJgJ/d/s4JUtXwSEZEKxFSmZ0RFR0dby5dXgUeiJe+Hz/tDehLcOR1qt3K6Irf05aKdvPDrRlrWDeKzO6KpF+zndEkiIiL5jDErLMuKPmtc4ayCOroTPr/KPjBw129QvbHTFbmlubEHeWTcKgJ8XIzs3Ijxy/aSkJhGWIgfo/u3YFBUYQeIRUREyl5R4Uztmyqq0CZw62S7/+bXg+zZNDlL75a1mfhgVzKycnlt1hbiE9OwgPjENMZMWseUVfFOlygiInIGhbOKrE5rGDERUg7CN4PtjgJylpZ1g/D1Ort7QFpWDq/M2OxARSIiIkVTOKvoGlwGQ7+DI1vhuyGQmep0RW7pwPH0QscTEtPKuRIREZFzUzirDC7pbT8HLX45jB8B2RlOV+R2wkIKPwzg8jAs2XGknKsREREpmsJZZdH6OrjuHdgxFybdC7k5TlfkVkb3b4FfgaVNL5ehmo8nQz9ewqjvVhJ37IRD1YmIiJyicFaZRI2E/v+BjT/BL3+FSnQSt6QGRYXz4uB2hIf4YYDwED9euak9S/7Rh8evbM6c2AP0ee0P3pi1hbRMBVsREXGOHqVRGf3+L5j/Clz+CPT9p/24DTmn+MQ0Xpy2iV/X7iMs2JcxA1txTUQ9jP7ZiYhIGdGjNKqS3v8Hl90Lf74DC193upoKITzEj3eHd2DC/V0J8ffmkXGruOWjJayPT3K6NBERqWIUziojY2DAf6HdzTDnBVj2mdMVVRidmoTyyyN/4T83tGPboRSufXchYyat40iKDlmIiEj58HS6ACkjHh4w6APISIapT4BvMLS7yemqKgSXh2F454Zc3a4eb83ZyteLdzF1bQKPXdmcW7s2wsulv9OIiEjZ0W+ZyszlBTd/CY0uh8n3w5aZTldUoQT7ezH22tb89lh32jcI4YVfNzLgrQXM33LI6dJERKQSUzir7Lz8YNg4qNMGJtwKu/90uqIK59LagXx9Vyc+uS2arJxcbvs8hnu+Ws7uI3rgr4iIlD6Fs6rANxhGToLgBvD9LbBvjdMVVTjGGPq2rsPMx3vw1FUtWbz9MH1fn8/Lv8WSkpHtdHkiIlKJKJxVFQE14bYp4BNk9+E8vM3piiokH08XD/a6hLlP9uKa9vX4YN52rnh1Hj+uiCM3t/I8lkZERJyjcFaVBNeH236yP/5mECTFOVtPBVY7yJfXh0Qy+aHLqRfixxP/W8PgD/5k9d5Ep0sTEZEKTuGsqql5Kdw6CdKT4OtBkHrY6YoqtKiG1Zn84OW8dnN74hPTGPTeIp6YsIaDRTRaFxEROR+Fs6qoXnsY/gMk7YVvB0P6cacrqtA8PAw3dqzP3Cd78UDPS/hlTQK9X53Hh39sJyNbraBEROTCKJxVVY0uhyHfwIENMG4oZKU5XVGFV83Hk6cHtGTm4z3oepzyLV8AACAASURBVElNXpoeS/835jN74wEqU5s0EREpW+qtWdWtmwg/3gN120HaUUiKt/em9RkLEUOcrq5C+2PLIV74ZQPbD6XSo3ktxl7TiktrBzpdloiIuImiemsqnAlMeQhWf3fmmJcfXPu2AloJZeXk8vXi3bw5ewtpmTnc1rUxf72yGcF+Xk6XJiIiDlPjcynazvlnj2Wl2X05pUS8XB7c/ZcmzHuyFzdHN+CLP3dyxavzGBezhxw9ekNERAqhcCZFP1JDj9ooNTWq+fDi4Hb88vBfaForgDGT1nHduwuJ2XnU6dJERMTNKJyJvcfsQsblorUND2bC/V15Z1gUx1IzGfLRYh4Zt4qERB3IEBERm8KZ2Jv/vfzOHPPwtMel1BljuLZ9GHOe6MWjfZoxc8N+rnhtHm/N3kp6lh69ISJS1Smcib3p/9q37d6bGPAKgNwcqNPW6coqNT9vF3/r25w5T/SkT8s6vDF7C31e+4Np6/bp0RsiIlWYTmvK2VKPwLsdoVZLuGMaeCjDl4fF24/w/C8biN2fTJemoTx7bRta1QtyuiwRESkjOq0pxRdQA/r+E/YshjXfO11NldH1khr8+shf+OegtsTuT+bqtxfw/6as41hqptOliYhIOVI4k8JFjoAGXWDmM3BCJwrLi6fLg1u7NGLek724rWtjxsXspder8/hy0U6yc3KdLk9ERMqBwpkUzsMDrnkdMo7DLB0MKG8h/t48d10bpj3anbbhQTz3y0YGvr2ARdvUqF5EpLJTOJOi1WkDXR6CVd/AniVOV1MltagbyLd3d+ajWzuSlpXDiE+Xcv83y9lz5ITTpYmISBnRgQA5t8xUeK8z+ATC/fPBpbZDTknPyuGzhTt5b+42snMt7u3ehId6XUqAj6fTpYmIyEXQgQC5ON4BMOBlOLgRlnzgdDVVmq+Xi1G9L+X3J3pxdbt6vDd3O1e8No8pq+L16A0RkUpEM2dSPOOGwY55MCoGQho4XY0AK3Yf4/lfNrA2LokODUPo1aIWPyyLIyExjbAQP0b3b8GgqHCnyxQRkSIUNXOmcCbFk7jHXt685AoY+p3T1Uie3FyLiSvjeOGXDaRknNldwM/LxYuD2ymgiYi4KS1rSsmENISeT0HsrxA7zelqJI+Hh2FIdAMCfc/eC5iWlcMrM2IdqEpEREpC4UyKr+soqNUKpv/dPiggbmN/Unqh4/GJ6Xz0x3aOpGSUc0UiInKxFM6k+Fxe9rPPkvbCH/91uho5TViIX6Hj3i4PXpweS9cXf+eRcav4c/thHR4QEXFzCmdyYRpdDpEjYfG7cGCj09VIntH9W+Dn5TpjzM/LxX9vimDW4z0Y0aUhf2w+yPBPlnLFa3/w8fztHFVbKBERt1SsAwHGmL8CXwDJwKdAFPC0ZVkzy7a8C6MDAeVEjdHd0pRV8bwyY3ORpzXTs3KYtm4f3y/dw/Ldx/B2eXBV27oM69SQLk1DMcY4WL2ISNVTotOaxpg1lmW1N8b0B0YBzwBfWJbVofRLvXgKZ+Vo5dfw8yNw/XsQNdLpauQCbd6fzLiYPUxaGcfx9Gya1gxgWKeG3NixPqEB3k6XJyJSJZQ0nK21LCvCGPMWMM+yrMnGmFWWZUWVRbEXS+GsHOXmwhcD4PAWeGQF+Ic6XZFchLTMHKau28e4mD2syJtNG9DOnk3r3ESzaSIiZamk4ewLIBxoArQHXNghrWNpF1oSCmfl7MAG+KgHtB8G17/rdDVSQrH7jzM+Zi8/rowjOT2bS2rlzaZ1qE91zaaJiJS6koYzDyAS2GFZVqIxJhSob1nW2tIv9eIpnDlg5jPw59tw1wxo2MXpaqQUpGXm8OvaBMbF7GHlnkS8PT0Y2LYuwzs34rLG1TWbJiJSSkoazroBqy3LSjXGjAQ6AG9ZlrW79Eu9eApnDlBj9Eotdv9xvl+6h8kr40nOyObS2tXyZtPCCfHXbJqISEmUeM8Z9nJmBPAN8Bkw2LKsnqVdaEkonDkkdiqMHw59/wndHnW6GikDJzKz+XWtvTdtVd5s2tXt6jG8c0OiG2k2TUTkYpQ0nK20LKuDMWYsEG9Z1mcnx8qi2IulcOYgNUavMjYmHGf8slOzac3yZ9PqE+yvmVMRkeIqaTj7A/gNuAvoDhzCXuZsV9qFloTCmYPUGL3KOZGZza9r9vF9zB5W703E57TZtI6aTRMROa+ShrO6wHBgmWVZC4wxDYFelmV9XfqlXjyFM4ctfBNmPwvDxkOLAU5XI+VoQ0IS42L2MGVVAikZ2TSvY8+mDY7SbJqISFFKFM7yXqAOcFnepzGWZR0sxfpKhcKZw3Ky4MPu9iGBUUvAO8DpiqScncjM5pc1CXy/dA9r4pLs2bSIeozo3JAODTWbJiJyupLOnA0BXgHmAQZ7aXO0ZVkTS7nOElE4cwO7/7QfTtvtMej7vNPViIPWx9uzaT+ttmfTWtQJZFinBtzQoT7BfppNExEpcfsmoO/J2TJjTC1gtmVZ7Uu90hJQOHMTU0bB2vFw/wKo09rpasRhqRn2bNq4GHs2zdfLg6vbhTG8c0M6NAzRbJqIVFklDWfrTt/8n/dQ2jU6ECCFUmN0KcL6+CS+j9nDT6viSc3MoWXdQIZ1asigqHDNpolIlVPScPYK9jPOxuUN3QKstSzrqVKtsoQUztyIGqPLOaRknNqbti7enk27JsKeTYtqoNk0EakaSuNAwI1AN+w9Z/Mty5pcjHuuAt7C7sX5qWVZLxX4ei/gJ2Bn3tAky7JeKM69hVE4cyNqjC7FtC7Onk37efWp2bThne3ZtCBfzaaJSOVV4nB2Ed/QBWwB+gJxwDJgmGVZG0+7phfwpGVZ11zovYVROHMzBzbYpzcjh6sxupxXSkY2P69O4PuY3ayPP46fl4tr29djWKeGRDYI4afVCbwyYzMJiWmEhfgxun8LBkWFO122iMhFKyqceZ7npmSgsPRmAMuyrKBz3N4J2GZZ1o681xoPXA+cM2CVwr3iLuq0ga6j7MboUSPVGF3OqZqPJ8M7N2R454Z5s2m7+Wl1AhOWxxEW7MuhlAyycuw/juIT0xgzaR2AApqIVDrn3KltWVagZVlBhbwFnieYAYQDe0/7PC5vrKCuxpg1xpjpxpg2F3gvxpj7jDHLjTHLDx06dJ6SpNz1fAqC6sOvj9vPQRMphnb1g3lxcARL/9GHf9/Q9oxgdlJaVg6vzNjsUIUiImWnLI/RFbajt+As3EqgUd4jOd4BplzAvfagZX1sWVa0ZVnRtWrVuuhipYz4VIOB/4WDG2HJB05XIxVMoK8XIzo3Ijun8O0X8YlpTFkVT9IJBX8RqTzOuaxZQnHA6R2w6wMJp19gWdbx0z6eZox53xhTszj3SgXS8mpoPgDmvQhtblBjdLlgYSF+xCemnTXuYeCxH1bj6WHo0rQG/drU4cpWdQgL8XOgShGR0lGWM2fLgGbGmCbGGG9gKPDz6RcYY+qavDPzxphOefUcKc69UsEM/K/9/renna1DKqTR/Vvg5+U6Y8zPy8VrN7Vn0kOXc0/3puxLSmPsTxu4/KXfufadhbwzZyux+49TVoeeRETKSpnNnFmWlW2MeRiYgf04jM8ty9pgjHkg7+sfAjcBDxpjsoE0YKhl/0la6L1lVauUg5CG0PPvMPs52DxdjdHlgpzc9F/Uac0ODavz9ICWbDuYwqyNB5i1cT+vz97Ca7O20DDUn76t69CvdR2iG4fi8tAz1ETEvZXZozScoEdpuDk1RpdydDA5nTmbDjJzw34WbTtCZk4uoQHeXNGyNv1a16F7s1r4ebvO/0IiImWk3J9z5gSFswpAjdHFASkZ2fyx+RCzNu5nTuxBktOz8fXyoHuzWvRrXYc+reoQGuDtdJkiUsUonIn7ONkY/YGFULuV09VIFZOVk0vMzqPM3LCfmRsPsC8pHQ8D0Y1D6de6Dv1a16VhDX+nyxSRKkDhTNyHGqOLm7Asiw0Jx/ODWuz+ZABa1g2kX+s69G1dl7bhQer1KSJlQuFM3Isao4sb2nPkBDM32kFt+a6j5FoQFuxL37yg1rlpKF4u/WVCREqHwpm4FzVGFzd3NDWTOZsOMGvjAeZvPUR6Vi5Bvp5c0bI2fVvXpWeLWlTzKctHRYpIZadwJu5HjdGlgkjLzGHB1kPM2niA2ZsOcOxEFt4uD7pdWoO+retyZeva1A70dbpMEalgFM7EPc18xm6MftcMNUaXCiE7J5cVu48xa+MBZm48wJ6jJzAGIhuE0K91Xfq1qcMltao5XaaIVAAKZ+KeMlLgvc7gGwT3zweXl9MViRSbZVlsOZCSf6BgXXwSAE1rBeQHtcj6IXjowbciUgiFM3FfsVNh/HDo+0/o9qjT1YhctITENGZvOsDMDQdYsuMI2bkWtQJ9uLJVHfq1qcPll9TAx1MPvhURm8KZuLfvh8LOP2BUjBqjS6WQlJbFvM0HmbnhAPM2HyQ1M4cAbxe9WtSmb+s69G5Rm2B/zRSLVGUKZ+LeEvfYy5uXXAFDv3O6GpFSlZGdw5/bjzBzg32g4FByBp4ehi5Na9CvTR2ubFWHsBA/pqyKL7J/qIhUPgpn4v4WvmE3Rh82Xo3RpdLKzbVYHZfIzA12g/bth1IBqB/ix/7j6WTnnvoz2c/LxYuD2ymgiVRSCmfi/tQYXaqgbQdTmLXxAK/P2kxWztl/HtcO9GHJmD46VCBSCRUVzvSoa3EfLi+45nVI2gN//NfpakTKxaW1q/Fgr0vILiSYARxMzqDjv2Zx/zfL+WzhTtbHJ5GTW3n+Ui0iZ9PjrcW9NLocIkfC4neh/VA1RpcqIyzEj/jEtLPGQ/y96NOqDjE7jzJjwwEAAn09uaxxKJ2a2G/twoPVVkqkElE4E/fT9wXYPBV+/RvcOQ3UdFqqgNH9WzBm0jrSsnLyx/y8XDx3bZv8PWcJiWks23WUJTuOErPzCL/HHsy/rmOj6nTOC2vtG4Tg66VHdohUVNpzJu4pvzH6+xA1wulqRMrFhZ7WPJScwbJdR1m64whLdx4ldn8yAN6eHkQ2CKFLk1A6NalBh0Yh+Hvr7+Ii7kYHAqRiyc2FL66Cw1vVGF2kmBJPZLJs1zGW7jhCzK6jrI9PItcCTw9Du/rBdG5Sg85NQunYuDpBvnrGmojTFM6k4lFjdJESSU7PYsXuYyzdeZSYnUdZG5dIVo6Fh4HWYUF0alyDzk1D6dQ4lOoB3k6XK1LlKJxJxTTz/8Gf76gxukgpSMvMYdWeYyzZae9ZW7UnkYzsXABa1AmkU5NQO6w1CaV2oK/D1YpUfgpnUjGpMbpImcnIzmFtXBIxO4+yZMcRVuw+xolM+0BC05oBp4W1GoSH+DlcrUjlo3AmFZcao4uUi6ycXDYkHCdm5xGW7jhKzK6jJKdnA1C/uh+dmoTSpUkNOjUJpVENf4xOUouUiMKZVGxqjC5S7nJyLWL3Hydm59H8sHY0NROAOkE+dMo7YNC5SSiX1q6msCZygRTOpGJTY3QRx1mWxbaDKSzdedR+23GEg8kZAIQGeNMp78G4nZuG0rJuEK4CLafU2F3kTApnUvGpMbqIW7Esi91HTth71nYeIWbnUeKO2V0Ogk7rYtC5aQ22H0zm/03ZcNZDdtXYXaoyhTOp+NQYXcTtxSemndqztvMoOw6nAmCAwn7bhIf4sejpK8q1RhF3oXAmlcPuP+GLAdDtMej7vNPViMh5HDyeTsyuozz8/aoir3n5xnZENqjOpbWrnbUUKlKZKZxJ5TFlFKwdDw8sVGN0kQqi20u/F9rY/fQZtQBvFxH1Q4hsGEL7+iFENQyhTpCetyaVl8KZVB6pR+DdjlCrlRqji1QQU1bFF9rY/T+D2hLRMITVexJZvdd+27TvONm59u+mesG+RDYIyX9rVz9YfUKl0igqnOm/cKl4AmpA3xfsxuirv1djdJEK4OSm/6JOa15Sqxo3dqwPQHpWDhsSkli9NykvsB1j+vr9AHgYaF4nkKiGJwOblkOl8tHMmVRMaowuUqUcTslgzd5Ts2tr9iZyPO8BuQHeLtrVDyayQXUiG2g5VCoOLWtK5aPG6CJVVm6uxc4jqcVaDm3fIIQILYeKG9KyplQ+ddpA14fsxuhRI9UYXaQK8fAwXFKrWiHLocfzw5qWQ6Wi0syZVGxqjC4i53AkJYM1cYms3pPIKi2HipvRsqZUXpt+hR9GqDG6iJzX6cuha+LsGbaNCWcvh7bPWxLVcqiUJYUzqbwsC8YNU2N0EbkoBZdD1+xNZM/RE4CWQ6VsKZxJ5XZst728eWkfNUYXkRK7mOVQNXaXC6VwJpWfGqOLSBnJzbXYdST1tMMG9unQrBz7d2iwnyfJ6dnknvYrVY3d5XwUzqTyy86Ej7pD5gk1RheRMnf6cuirMzaf0f3gJF9PD+7t0ZSWdYNoVS+QRjUCtCQq+RTOpGrYtQi+HKjG6CJSrpo8PZWifpu6PAw5eVNqfl4umtcNpHW9wLzAFkSLuoEE++mkeVWk55xJ1dC4G0SOgMXvQvuhaowuIuUiLMSv0Mbu4SF+zHmiJ9sOprBx33Fi9yWzad9xpq/fz7iYvWdc16qePbvWql4QLetqlq0qUziTyqfvC7B5Gvz6NzVGF5FyMbp/i0Ibu4/u3wJfLxdtw4NpGx6c/zXLsjhwPINN+46zaf9xNu1LJnbfceZuPljoLJsd2IJoWS+QIF/NslV2WtaUymnl13Zj9OvfV2N0ESkXpXFaMz0rh60HUvICW95M2/7jJJ7Iyr+m4Cxbq3pBNAr1x0OzbBWO9pxJ1aLG6CJSSZw+y7Zx33Fi99tLozsOpeSfDvXzctGibuAZga1FXc2yuTuFM6l6TjZGjxoB173jdDUiIqWq4Cyb/ZZMUtqpWbb61f1oWTfIPoCgWTa3owMBUvWc3hg9coQao4tIpeLrZT8Mt139M/ey7T+eTuy+ZDbmBbbY/cn8HnugkFm2Mw8gBGqWzW1o5kwqNzVGFxE5NcuWfwCh8Fm2VvWCaFX31NJowwKzbOqCULo0cyZVk081GPCy3Rh9yQdqjC4iVdK5ZtlOBrWTs2xzNp2aZfP3tmfZWtYNIis7h5/X7iMzOxeA+MQ0xkxaB6CAVso0cyaV38nG6NtmQ0BNSN4PwfWhz1iIGOJ0dSIibiU9K4ctB5Lzl0Zj9589y3a6AB8Xj1/ZnPAQP8JC/KgX4kvNAB/taysGzZxJ1WUMNOkBW6ZD8j57LGkv/JI3i6aAJiKSz9fLRUT9ECLqh+SPWZZF0zHTCu2CkJqRw7+mbjpjzNvlQb0QX+oF+xIW4kd4iB/1gv0IC/G1Pw7xo5qPIkhR9E9GqoYl7589lpUGc15QOBMROQ9jzDm6IPgy9dHuxCemsS8xnYSkNOIT00hITGdfYhpLth9h//H0M5rCAwT5ehKWN9sWFmKHuLDgU5/XCfLFy+VRTj+he1E4k6ohKe7CxkVE5AxFd0FoSYi/NyH+3rQJCy703uycXA4mZ5CQaAe3fUnpJCSm5b2ls3LPsTMetAvgYaB2oC9hIb7Uy5t9Cws+7eMQP6r7e2EqYRcYhTOpGoLr20uZBfkEQU6WTnGKiJzHyU3/F3Na09PlkT9LdtYGqzwnMrNJSDwttJ0W4DbEJzFr44H8wwgn+Xp55M+2nb6EenLvW1iwH37ermL/jO5yGrVMDwQYY64C3gJcwKeWZb1UxHWXAUuAWyzLmpg3tgtIBnKA7MI2zBWkAwFSpLUT7D1mWadNyRsXWDlQNwKufw/qRThXn4iInJNlWRxJzcyfbTsZ3PYlpecto6ZxKCWDgrEmNMC7wN433zOWU2sH+uLyMExZFV/ozOCLg9uVWUAr9w4BxhgXsAXoC8QBy4BhlmVtLOS6WUA68HmBcBZtWdbh4n5PhTM5p7UT7D1mSXGnTmt6+sDUJyDtGPzlb9DjSXtMREQqnMzsXA4cT89bOrVDnL0X7lSgS87IPuMel4ehbpAvh5IzyMzJPes1w0P8WPT0FWVSrxOnNTsB2yzL2pFXwHjgemBjgeseAX4ELivDWkTsjf+Fbf5v3B1+GwPz/wubfrFn0ep3LP/6RESkRLw9PWgQ6k+DUP8irzmenmUfXMjf/2YHt8mr4gu9PqGQQxBlrSzDWThw+iafOKDz6RcYY8KBG4ArODucWcBMY4wFfGRZ1sdlWKtUZf6hMPgjaDsYfnkMPrsSuj4Mvf8BXn5OVyciIqUoyNeLoLpetKgbeMZ4zM6jhZ5GDQsp/98DZXlGtbDjEwXXUN8EnrIsK6eQa7tZltUBGACMMsb0KPSbGHOfMWa5MWb5oUOHSlaxVG3N+8OoJRB1K/z5Nnz4F9izxOmqRESkHIzu3wI/rzMPD9inUVuUey1lGc7igAanfV4fSChwTTQwPm9/2U3A+8aYQQCWZSXkvT8ITMZeJj2LZVkfW5YVbVlWdK1atUr3J5CqxzcYrnsbbp0C2Znw+VUw/SnITHW6MhERKUODosJ5cXA7wkP8MNh7zcryMMC5lOWBAE/sAwF9gHjsAwHDLcvaUMT1XwK/WpY10RgTAHhYlpWc9/Es4AXLsn471/fUgQApVRkpMOd5iPkYQhrBde9A055OVyUiIpVEUQcCymzmzLKsbOBhYAawCZhgWdYGY8wDxpgHznN7HWChMWYNEANMPV8wEyl1PtVg4CtwxzTwcMHX19l70tKPO12ZiIhUYmp8LlIcmSdg7r/tNlCB9eDat6BZX6erEhGRCqzcZ85EKhVvf+j/b7hrJnhXg+9ugskPwomjTlcmIiKVjMKZyIVocBk8sAC6Pwlrf4D3u8CmX52uSkREKhGFM5EL5ekDfZ6B++ZCQG34YQRMvAtSi93MQkREpEgKZyIXq157O6D1/j/Y+DO81wnW/8hZjd1EREQugMKZSEm4vKDn3+H++fbjNibeBT+MhOT9TlcmIiIVlMKZSGmo0xrungV9X4Cts+C9zrB6nGbRRETkgimciZQWlyd0+ys8uAhqtYQpD8B3N0NSnNOViYhIBaJwJlLaajaDO6fDVS/D7kXwXhdY/oVm0UREpFgUzkTKgocHdHkAHvwTwiLh18fsDgNHdzpdmYiIuDmFM5GyFNoEbv8FrnkT4lfBB5fD0o8gN9fpykRExE0pnImUNWMg+k4YtQQaXQ7T/w5fDIDD25yuTERE3JDCmUh5Ca4PIybCoA/g0Cb4sBssegtysp2uTERE3IjCmUh5MgYih8OoGLikD8waC5/1hQMbna5MRETchMKZiBMC68LQ7+CmzyFxN3zUA/74L+RkOV2ZiIg4TOFMxCnGQNsb7Vm0VtfC3H/Dx71h3xqnKxMREQcpnIk4LaAm3PwF3PItpB60A9qcf0J2htOViYiIAxTORNxFq2vhoSUQcQsseNVe6oxb7nRVIiJSzhTORNyJfyjc8IF9qjMj2T4sMOP/IPOE05WJiEg5UTgTcUfN+tqzaB1uh8Xv2o/d2P2n01WJiEg5UDgTcVe+QXDtm3DbT5CbYz+4dtpoyEhxujIRESlDCmci7q5pL7tHZ+cHIOYT+KArbJ/rdFUiIlJGFM5EKgKfajDgZbhzOri84ZtB8PMjkJ7kdGXlZ+0EeKMtPBdiv187wemKRETKhMKZSEXSqCs8sBC6/RVWfQvvdYEtM5yuquytnQC/PApJewHLfv/LowpoIlIpGcuynK6h1ERHR1vLl+vRA1JFxK+AKaPsPp0RQ6FhZ1jwOiTF2X08+4yFiCFOV3luubmQkwk5GZB98n2GPXb6+4l3QOrhs+8PbgCPry/3skVESoMxZoVlWdEFxz2dKEZESkF4R7j/D1jwmt36ae34U187ObMEpwKaZZ0des4IRecIR2ddV/Cec917ju+VW8Km70l74dhuqN6oZK8jIuJGNHMmUhm82hxSDpw9bjzAyz8vTGWW3vczHuDyAU/vvPc+9l64ot6fNXb6ved6jbyvT7zb7p5QlDptocUAaDEQwqLs1lgiIm5OM2cilVlKEcHFyoUOtxUvHLm8ix+YXOX8R0f/f9szgVlpp8a8/KD3/wEGNk+zZxDnvwKB9fKC2tXQpLtds4hIBaJwJlIZBNfP2yxfcLwBXPVi+ddT2k4uzc55ofA9dZc/DKlHYOtM2DwV1vwAyz8H72pwaR87qDXra3dgEBFxc1rWFKkMTp5mLDizdO3b7n8ooCxkpcPO+XZQ2zzdXvI1Lmh0ub302XIgVG/sdJUiUsUVtaypcCZSWaydUPTMUlWWmwsJq04FtYMb7fHare2gdnKfmoeeLCQi5UvhTEQE4OgOO6Rtnm73K7VyoFrdUwcKmvQAL1+nqxSRKkDhTESkoBNH8/apTYNtcyAzBbwC8vapDYTm/bVPTUTKjE5riogU5B8K7Yfab1npsGuBHdQ2T4dNP9v71Bp2tWfVWg6E0KZOVywiVYBmzkRECsrNhX2r7JAWOw0ObrDHa7XKC2pXQ1gH7VMTkRLRsqaIyMU6tisvqE09bZ9aHWh+lR3UmvSwT8eKiFwAhTMRkdKQdgy2zrKD2rbZefvU/OGSK+yg1qw/BNRwukoRqQC050xEpDT4VbcfURIxxO4ZumuBvfS5eTrE/mq3tmrQxd6j1mIg1LjE6YpFpILRzJmISGmwLNi3Oi+oTYMD6+3xmi1OBbXwaO1TE5F8WtYUESlPx3bnPU9tGuxeBLnZEFAbWlxlB7WmvbRPTaSKUzgTEXFK2jHYOjvveWqzIeP4qX1qLQbYBwsCaqrLg0gVoz1nIiJO8asOETfbb9mZec9Ty5tVO7lPrXpTSNwNuVn2PUl77X6poIAmUsUonImIlCdPb7sDwaV9YOArsG+NHdQWvHYqmJ2UlQbTngTvAKjRDEKbgMvLmbpFpNwonImIOMUYCIu03/54ufBr0pNg/HD7Yw9PqN4YajaHms3swHbyY7WZEqk0FM5ERNxBcH17KbOgoHC45Rs4vDXvbYv9fttsyMk8dZ1/NhYMnQAACkBJREFUDTuo1bj0VGCr2RxCGoFLf9SLVCT6P1ZExB30GWvvMctKOzXm5QdXPgfhHe230+Xm2HvUDm/LC2xb4Mg22PIbrPrm1HUeXnZP0JrNTgW2kyHOL6Q8fjIRuUAKZ/L/27vXGLnKOo7j3393t92lZdsKBXG3pSUoSggqIqJVIBYN3qOJd33hC42JV2K8JhoTX+gL4y0xaoNoIkZCColK8JYo1EsE1BJoKSJpga6AQqFFsC3d9u+L54x7YbadaTt7ZqbfT3IyZ86cM/Ofnt321+c8z3MkdYNGp/9WR2suGCih6xlnwHNePfO1PY9Nhbad01rc7v5lmdKjYfEpM0PbSdX6slXl/SXVwqk0JOl4cWB/mX9t5z+mWtsaIW7Po1P7DSwqdzaY0a/tzLI+PFpf/VKfcSoNSTreDQyVkHXymWV+teme3DkttFWtbf/aAluvLzd6bzjxtFn92qrwNjp++LsfOI+b1BLDmSSp3Kx98Umw6sKZ2yefgse2T10a3Vm1tG3eUEaSNgyOVKHtzJn92k46ExYtKcFsep8653GT5mQ4kyTNbXAhrDirLNNlwpOPzByM8Mjd8MBtcOdPIQ9O7Ts6Bv99pNwofrr9e0pLmuFMmsFwJklqXwQsWVGW1WtnvrZ/Lzy6bdpl0nvg9qubv8/uHfDAJnjmuQ5CkCqGM0nSsTU0DKeeXZaG+/7YfB43gPWXwPAyWP3yckP4NReVy6IR81Cs1H0MZ5KkzptrHrdXfakEs+03leWu68trS55ZQtoZF8Oai2HZynrqlmpgOJMkdd7h5nE7963l8dHtsH1jCWrbfgd3XFO2L19TBbWLSlhbfPL8fwdpnnR0nrOIuAz4JjAAXJGZX5ljvxcDfwbenpkb2jl2Ouc5k6Q+kgn/3lq1qm2Ee/8A+x4vr516zlRQO/1lzr+mnjTXPGcdC2cRMQDcDbwKmABuBd6ZmXc22e83wF7gyszc0OqxsxnOJKmPHZiEB2+rWtVugh03w+ReiAEYO68EtTMuhvELSr83qcvVMQntBcA9mbmtKuBq4E3A7ID1EeBa4MVHcKwk6XgxMAjj55flFZ8oo0InbilBbftN8Ievw++/CoPDsPIlVZ+1S+C0F3jzd/WUTv60jgHTh+ZMAC+ZvkNEjAFvBl7JzHB22GOnvccHgA8ArFq16qiLliT1iKHh6tLmRcDnYe/jcN+fpvqs/fZLZVk0CqevneqzdsrZjgRVV+tkOGv2kz/7Guo3gE9n5oGY+YvSyrFlY+Z6YD2Uy5pHUKckqR8Mj8JZl5UF4ImH4d7fT10GvfsXZfviFVOhbs3F8Iw19dUsNdHJcDYBTB/7PA48MGuf84Grq2B2MvDaiJhs8VhJkua2ZAWc85ayAOy6v2pV21jC2uZry/Zlq6qgdkl5PPHU2kqWoLMDAgYpnfrXAf+kdOp/V2ZumWP/HwLXVwMC2jq2wQEBkqSWZJa7F2zfCNtuLC1sjXuFrnju1OCC09fCyLJaS1X/mvcBAZk5GREfBn5FmQ7jyszcEhEfrF7/brvHdqpWSdJxJmLqnqEXvB8OHoCHbq8GF2yETT+CW74HsaAMKGhMiLvyQlh4Qt3Vq891dJ6z+WbLmSTpmJjcBxN/mRpcMHErHJyEgYVlqo7G4IKxF8HAUDnm9mvmnmRXamLe5zmrg+FMktQR+56A+/8M228srWsP3QEkLFxSJsFdNAp3/byEuoahEXjDtwxomlMd85xJktQfFi2BZ19aFoD/PlqNBK0GF+z8x9OP2b8HbvhkaXEbXgYjy0v/tZHl5bkT5WoOtpxJknS0vriMOWZ8mtvgyNMD2/+fN9tWPR9eCgsGOvI1NL9sOZMkqVOWjsPuHU/fPjoG77sB9jwGe3ZVj4/B3l2ztu2CXfeV21Pt2QX7nzzEh0WZ021k+dwBrvHa7G1DI0c3AW+/96vrku9nOJMk6Wit+wL8/KPlUmbD0Ahc+kVYvros7ZjcV0La00Jck2C3d1cJho1teWDu9x1YePgA1yzkDS+FLdfN/I67d5Tn0N0BLbNaDgKHWN98Hfzi0zBZ//fzsqYkScdCN7S6ZMK+/0wLcLNCXNNt1fpTTxzmzYOml24XDMEpz2Mq7CSHDEEz1mlhn8b7Haw+vrGeh18/FpauhMs3H5v3msXLmpIkddK5b6u/BSmqS57Do+XOB+04sP/QrXU3frn5cQf3w4mnlTnhIoAoj/9fX9DiOm3uX12ebWv/Q9T2m883/367J9r7czwGDGeSJKnM17ZkRVma2XRV8351S1fCu6/pbG3z4Zb1c3y/8XkvZcG8f6IkSeo9675Q+tFNNzRStveDLvp+hjNJknR4576tTKq7dCUQ5bGfJtntou/ngABJkqQazDUgwJYzSZKkLmI4kyRJ6iKGM0mSpC5iOJMkSeoihjNJkqQuYjiTJEnqIoYzSZKkLmI4kyRJ6iKGM0mSpC5iOJMkSeoifXX7poh4GLivwx9zMvBIhz9DneU57G2ev97nOex9nsNj4/TMXDF7Y1+Fs/kQEX9pdh8s9Q7PYW/z/PU+z2Hv8xx2lpc1JUmSuojhTJIkqYsYztq3vu4CdNQ8h73N89f7PIe9z3PYQfY5kyRJ6iK2nEmSJHURw1mLIuKyiPh7RNwTEZ+pux61JyJWRsTvImJrRGyJiI/VXZOOTEQMRMSmiLi+7lrUvohYFhEbIuKu6vfxpXXXpNZFxOXV36GbI+InETFcd039yHDWgogYAL4NvAY4G3hnRJxdb1Vq0yTwicx8HnAh8CHPYc/6GLC17iJ0xL4J/DIznws8H89lz4iIMeCjwPmZeQ4wALyj3qr6k+GsNRcA92Tmtsx8CrgaeFPNNakNmflgZv6tWv8P5R+EsXqrUrsiYhx4HXBF3bWofRExClwEfB8gM5/KzF31VqU2DQIjETEInAA8UHM9fclw1poxYMe05xP4D3vPiojVwAuBm+utREfgG8CngIN1F6IjcgbwMPCD6tL0FRGxuO6i1JrM/CfwVeB+4EFgd2b+ut6q+pPhrDXRZJvDXHtQRCwBrgU+npmP112PWhcRrwf+nZl/rbsWHbFB4DzgO5n5QuBJwD68PSIillOuGq0BngUsjoj31FtVfzKctWYCWDnt+Tg25faciBiiBLMfZ+Z1ddejtq0F3hgR91K6FrwyIq6qtyS1aQKYyMxGq/UGSlhTb7gU2J6ZD2fmfuA64GU119SXDGetuRV4dkSsiYiFlA6QP6u5JrUhIoLSz2VrZn6t7nrUvsz8bGaOZ+Zqyu/gbzPT/7X3kMx8CNgREWdVm9YBd9ZYktpzP3BhRJxQ/Z26Dgd0dMRg3QX0gsycjIgPA7+ijE65MjO31FyW2rMWeC9wR0TcVm37XGbeUGNN0vHoI8CPq//obgPeV3M9alFm3hwRG4C/UUbAb8I7BXSEdwiQJEnqIl7WlCRJ6iKGM0mSpC5iOJMkSeoihjNJkqQuYjiTJEnqIoYzSZKkLmI4kyRJ6iKGM0mSpC7yP4A24StbBObbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,6))\n",
    "plt.plot(solver.loss_history, marker='o', label=\"training loss\")\n",
    "plt.plot(solver.val_loss_history, marker='o', label=\"validation loss\")\n",
    "plt.ylabel('loss')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-03T09:48:36.153977Z",
     "iopub.status.busy": "2020-11-03T09:48:36.152097Z",
     "iopub.status.idle": "2020-11-03T09:48:36.351614Z",
     "shell.execute_reply": "2020-11-03T09:48:36.352258Z"
    },
    "papermill": {
     "duration": 0.270597,
     "end_time": "2020-11-03T09:48:36.352446",
     "exception": false,
     "start_time": "2020-11-03T09:48:36.081849",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmcAAAFlCAYAAACukk+HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdeVzVVf7H8dfhsguC4g4iWG6poIhbVlpm2uZYmVla2V7zK5tppmmZaZlmmmmmpqmmbVps0zIzM1tttMwWLcEF910R3HABUQHhcn5/fBFZlZTLvcD7+XjwgHu+y/1cs3h3zvmeY6y1iIiIiIhv8PN2ASIiIiJyjMKZiIiIiA9ROBMRERHxIQpnIiIiIj5E4UxERETEhyiciYiIiPgQf28XUJtatGhh4+LivF2GiIiIyAmlpqbusda2rNjeoMJZXFwcKSkp3i5DRERE5ISMMVuratewpoiIiIgPUTgTERER8SEKZyIiIiI+pEHNOatKYWEhGRkZ5Ofne7sUqYHg4GBiYmIICAjwdikiIiJe0eDDWUZGBuHh4cTFxWGM8XY5chzWWvbu3UtGRgbx8fHeLkdERMQrGvywZn5+PlFRUQpm9YAxhqioKPVyiohIo9bgwxmgYFaP6J+ViIg0do0inHlTdnY2L7744klde9FFF5GdnX3ccx5++GHmzJlzUvc/njfffJM777wTgJdffpm333671t9DREREKmvwc85+qZlLMnly9lq2Z+fRLjKEe4d3YVTv6JO+39Fw9utf/7rSMbfbjcvlqvbazz///IT3f+yxx066tpq6/fbbPf4eIiIi4lDPWRkzl2TywIzlZGbnYYHM7DwemLGcmUsyT/qe999/Pxs3bqRXr17ce++9zJs3j3PPPZdrrrmGnj17AjBq1Cj69OlD9+7deeWVV0qvjYuLY8+ePWzZsoVu3bpxyy230L17dy644ALy8vIAmDBhAtOnTy89/5FHHiEpKYmePXuyZs0aALKyshg2bBhJSUncdtttdOjQgT179lSq9Y033qBz584MHjyYH374obT90Ucf5amnngJgw4YNnH/++SQmJpKUlMTGjRsBePLJJ+nbty8JCQk88sgjJ/3nJSIivmvmkkwGPfE18fd/xqAnvj6l349SvUbVc/bnT1ayavuBao8vSc/miLu4XFteoZs/TE/jvZ/Tq7zmjHZNeeTS7tXe84knnmDFihUsXboUgHnz5vHzzz+zYsWK0icSJ02aRPPmzcnLy6Nv375cccUVREVFlbvP+vXree+993j11VcZM2YMH374IePHj6/0fi1atGDx4sW8+OKLPPXUU7z22mv8+c9/5rzzzuOBBx7gyy+/LBcAj9qxYwePPPIIqampREREcO6559K7d+9K540bN47777+fyy67jPz8fIqLi/nqq69Yv349P//8M9ZaRo4cyfz58znnnHOq/XMREZH65WgHRl6hGzjWgQGc0giTVObRnjNjzAhjzFpjzAZjzP3VnDPEGLPUGLPSGPNtSVt7Y8w3xpjVJe13e7LOoyoGsxO1n6x+/fqVWyriueeeIzExkQEDBrBt2zbWr19f6Zr4+Hh69eoFQJ8+fdiyZUuV97788ssrnfP9998zduxYAEaMGEGzZs0qXffTTz8xZMgQWrZsSWBgIFdddVWlc3Jzc8nMzOSyyy4DnDXJQkND+eqrr/jqq6/o3bs3SUlJrFmzpsrPICIi9UORu5jc/EKycgvYtu8w63fl8vjnq0uD2VF5hW6enL3WS1U2XB7rOTPGuIAXgGFABrDIGDPLWruqzDmRwIvACGttujGmVcmhIuB31trFxphwINUY87+y156M4/VwAQx64msys/MqtUdHhvD+bQNP5a3LadKkSenP8+bNY86cOSxYsIDQ0FCGDBlS5VISQUFBpT+7XK7SYc3qznO5XBQVFQHO+mE1caInJau7j7WWBx54gNtuu61G7yMi0hDV9pzliorcxeQVuskvLCa/0F3ydbTNXfq9qvaCwmLyjrjJL3KXfC8mv9xrN3lHiikoOb+ouGa/NwC2Z+fhLra4/PS0fW3x5LBmP2CDtXYTgDFmKvAroGzAugaYYa1NB7DW7i75vgPYUfJzrjFmNRBd4dpad+/wLuW6bAFCAlzcO7zLSd8zPDyc3Nzcao/n5OTQrFkzQkNDWbNmDQsXLjzp96rOWWedxbRp07jvvvv46quv2L9/f6Vz+vfvz913383evXtp2rQpH3zwAYmJieXOadq0KTExMcycOZNRo0ZRUFCA2+1m+PDhPPTQQ4wbN46wsDAyMzMJCAigVatWld5HRKQhqmrI7w/T00jLyCaxfWSdB6ayAv39CPb3IyTQRXCAi5AAF0EBLkIC/GjeJJCQSKfd+fIjJODYecEBfqXHHpm1kn2HjlS6vwX6Pj6HIV1acn631pzdqQXhwdrl5VR4MpxFA9vKvM4A+lc4pzMQYIyZB4QDz1pry63ZYIyJA3oDP3mq0KOO/h9Obf6fT1RUFIMGDaJHjx5ceOGFXHzxxeWOjxgxgpdffpmEhAS6dOnCgAEDTukzVOWRRx7h6quv5v3332fw4MG0bduW8PDwcue0bduWRx99lIEDB9K2bVuSkpJwu92V7vXOO+9w22238fDDDxMQEMAHH3zABRdcwOrVqxk40OldDAsLY/LkyQpnItJoPDl7baUhvyPuYib9sKXK8wNdfqXBJyTQRbC/i+BAF8H+TmAKjihpLxOOQsqEp6DS15VDVEjJfUICXQT5u2qtR8tdbCt1YAQH+HFlcnty8wqZu3o3MxZnEuAyDOgYxXldW3F+t9a0bx5aK+/fmJiaDnn94hsbcyUw3Fp7c8nra4F+1tq7ypzzPJAMDAVCgAXAxdbadSXHw4BvgcettTOqeZ9bgVsBYmNj+2zdurXc8dWrV9OtW7da/nT1S0FBAS6XC39/fxYsWMAdd9xR+oCCL9I/MxGpb+Lv/4yqfpsa4H/3DC7XIxUcUHuBqa4db+i2yF1M6tb9zF2zm7mrd7Ex6xAAnVuHcV7X1pzfrRW9Y5vV28/uCcaYVGttcsV2T/acZQDty7yOAbZXcc4ea+0h4JAxZj6QCKwzxgQAHwJTqgtmANbaV4BXAJKTkz2TNOu59PR0xowZQ3FxMYGBgbz66qveLklEpEEodBfz7Jz1VQYzgHaRIZzeKqxOa/KkUb2jqx1N8nf50b9jFP07RvHgRd3YsucQc1bv4us1u3ntu028/O1GmoUGcG6XVgzt1ppzOmv4szqeDGeLgE7GmHggExiLM8esrI+B540x/kAgzrDnv40zM/11YLW19mkP1tgodOrUiSVLlni7DBGRBmXr3kPcPXUpS7dl0z++GcsycsgvPPZ0/6nOWa7v4lo04eazO3Lz2R3JyStk/ros5q7exdw1u5mxxBn+7B9/bPgzNkrDn0d5LJxZa4uMMXcCswEXMMlau9IYc3vJ8ZettauNMV8CaUAx8Jq1doUx5izgWmC5Mebo+NuD1toTL5kvIiLiQdZaPlqSyUMzV+DnZ3j+mt5cktDO409r1mcRIQFcmtiOSxPbUeQuZnF6NnNX72LO6l089ukqHvt0FZ1ahTG0W2uGdmtFUiMf/vTYnDNvSE5OtikpKeXaNH+p/tE/MxHxVQfyC/nTRyuYtWw7/eKa8++xvYiODPF2WfXalj2HmLtmN1+v2cVPm/ZRVGxLhz/P69aKczq3pGkDHf70xpwzERGRBiN16z7unrqUHTn5/G5YZ3597umNunentsS1aMJNZ8Vz01nxHMg/Ovy5m6/XOsOf/n6G/h2bM7Sr06vWIarJiW9azymciYiIHEeRu5jnv9nAc3PXE90shA9uH0hSbOWdVuTUNQ0O4JKEdlyS0A53sWVx+n7mrN7F3NW7S4c/T28VxtBurRjatTVJsZH4uxreNuEN7xM1AGFhzpM927dvZ/To0VWeM2TIECoO4Vb0zDPPcPjw4dLXF110EdnZ2bVXaIma1CsiUh9t23eYsa8s5Jk56xnVK5rPJ56tYFZHXH6GvnHNeeDCbsy5ZzDf3juEhy85g9ZNg3j9u82M+e8Ckh+fw2/fX8qnads5kF/o7ZJrjeacVZQ2DeY+BjkZEBEDQx+GhDG1XOnxhYWFcfDgweOeM2TIEJ566imSkysNVZeKi4sjJSWFFi1a1HaJ5dSk3l9Cc85ExBfMWradP85YjgX+OqqHJvf7kAP5hXy3bg9zV+/im7W72X+4EH8/Q7/45s5DBV1bEdfC94c/q5tzpp6zstKmwScTIWcbYJ3vn0x02k/Sfffdx4svvlj6+tFHH+Vf//oXBw8eZOjQoSQlJdGzZ08+/vjjStdu2bKFHj16AJCXl8fYsWNJSEjgqquuKre35h133EFycjLdu3fnkUceAZzN1Ldv3865557LueeeCzhhbc+ePQA8/fTT9OjRgx49evDMM8+Uvl+3bt245ZZb6N69OxdccEGVe3hu3ryZgQMH0rdvXx566KEq63W73fz+97+nZ8+eJCQk8J///AeA1NRUBg8eTJ8+fRg+fDg7duw46T9bERFPOFhQxD3TljLxvSV0ah3GF3efrWDmY5oGB3BxQluevqoXKX8axvTbB3Lz2R3Jyi3gL5+uYshT8xj6r3n8/fPV/Lx5H0Xu4hPf1Ic0rjlnX9wPO5dXfzxjEbgLyrcV5sHHd0LqW1Vf06YnXPhEtbccO3Ysv/nNb/j1r38NwLRp0/jyyy8JDg7mo48+omnTpuzZs4cBAwYwcuTIajcff+mllwgNDSUtLY20tDSSkpJKjz3++OM0b94ct9vN0KFDSUtLY+LEiTz99NN88803lXrOUlNTeeONN/jpp5+w1tK/f38GDx5Ms2bNWL9+Pe+99x6vvvoqY8aM4cMPP2T8+PHlrr/77ru54447uO6663jhhReqrPeVV15h8+bNLFmyBH9/f/bt20dhYSF33XUXH3/8MS1btuT999/nj3/8I5MmTar2z09EpC4t3ZbN3VOXsG3fYSYO7cTE805vkHOaGhKXnyE5rjnJcc25/8KupO89zNw1zjy1ST9s5r/zNxEZGsCQzi0Z2q01g7v4/tOfjSucnUjFYHai9hro3bs3u3fvZvv27WRlZdGsWTNiY2MpLCzkwQcfZP78+fj5+ZGZmcmuXbto06ZNlfeZP38+EydOBCAhIYGEhITSY9OmTeOVV16hqKiIHTt2sGrVqnLHK/r++++57LLLaNLE6fK9/PLL+e677xg5ciTx8fH06tULgD59+rBly5ZK1//www98+OGHAFx77bXcd999lc6ZM2cOt99+O/7+zl+x5s2bs2LFClasWMGwYcMAp3etbdu2J/ojFBHxOHex5eVvN/Lv/62jddNg3r9tIH3jmnu7LDkJsVGh3DAonhsGxZObX8h36/cwZ/Uu5q3NYubS7fiXzGUb2s1Z/Lbs8KevrFXXuMLZcXq4APh3j5IhzQoi2sMNn530244ePZrp06ezc+dOxo4dC8CUKVPIysoiNTWVgIAA4uLiyM/PP+59qupV27x5M0899RSLFi2iWbNmTJgw4YT3Od48w6CgoNKfXS5XlcOa1dVS8T0qnmOtpXv37ixYsOC414qI1KXt2Xn89v2l/LR5H5cktOXxy3oSEeLbPStSM+HBAVzUsy0X9WyLu9iydNt+5qx29v7862er+etnq+nYsgnnd2tNoMvw2vebS3d5yMzO44EZzmhbXQc09dWWNfRhCKiwmGBAiNN+CsaOHcvUqVOZPn166dOMOTk5tGrVioCAAL755hsqbthe0TnnnMOUKVMAWLFiBWlpaQAcOHCAJk2aEBERwa5du/jiiy9KrwkPDyc3N7fKe82cOZPDhw9z6NAhPvroI84+++waf55BgwYxdepUgNKaKrrgggt4+eWXKSoqAmDfvn106dKFrKys0nBWWFjIypUra/y+IiK17fPlOxjxzHyWZ+bw1JWJ/Ofq3gpmDZTLz9CnQ3PuG9GVr347mO/+cC6PXnoG0ZEhvPHDZp7/ZmO57bcA8grdPDl7bZ3XqnBWVsIYuPQ5p6cM43y/9LlTflqze/fu5ObmEh0dXTqMN27cOFJSUkhOTmbKlCl07dr1uPe44447OHjwIAkJCfzzn/+kX79+ACQmJtK7d2+6d+/OjTfeyKBBg0qvufXWW7nwwgtLHwg4KikpiQkTJtCvXz/69+/PzTffTO/evWv8eZ599lleeOEF+vbtS05OTpXn3HzzzcTGxpKQkEBiYiLvvvsugYGBTJ8+nfvuu4/ExER69erFjz/+WOP3FRGpLYcKirhvehq/nrKY+BZN+Hzi2YzuE3PCUQFpONo3D2XCoHjeuak/ix8aVu1527OrHkHyJC2lIT5H/8xExJOWZ+QwceoStuw9xB2DT+O3wzoToEn/jd6gJ74ms4ogFh0Zwg/3n+eR99RSGiIi0qgVl0z6v/ylH8g74ubdmwfwhxFdFcwEgHuHdyEkwFWuLSTAxb3Du9R5LY3rgQAREWmUdubk87sPlvLDhr2M6N6GJ67oSWRooLfLqn98YKF2TxnVO5robZ/SfvGTtLJZ7DYt2ZZ0L317j6jzWhTORESkQftq5U7u+zCN/MJinri8J1f1ba+5ZSfj6ELthSVDfznbYNZdkLcfzhgFxq/kyzjf/Vxl2ip++eCff9o0+i5/BMgDA23Ios3yRyCuWZ0H0EYRzqpa1kF8U0OaAyki3pV3xM1fP1vFlJ/S6d6uKc9d3ZvTWoZ5uyzfVnAQDmQ6wSsno/xX+o9Q7C5/flE+fPEH5+sXMeXDWrkgd/RYNeHOr6qw56pwbXXh0FR9Xz8XbJgLRRXmnBXmOT2FCme1Kzg4mL179xIVFaWA5uOstezdu5fg4GBvlyIi9dzK7TncPXUpG3Yf5NZzOvK7CzoT5O868YUNWbEbDu4qCVsVw1fJ67z95a8xLmjazhnCrBjMyrr4abDFYG3J92Kw7jI/l3wVF1duKz2vzLXFVVxrbTX3rOL6SufZau559FxbOZgdlZNRe/8MaqjBh7OYmBgyMjLIysrydilSA8HBwcTExHi7DBGpp4qLLZN+2Mw/v1xLZGgA79zUj7M7tfR2WXUj/0D5sHUgs8Lr7VBcVP6a4Ahn2aiIGGjf3/l+9HVEDIS1AVdJVDjeQu19b/L85/O0aj9f3f9OavDhLCAggPj4eG+XISIiHrY7N5/ff5DG/HVZnN+tFf+4IoGosKATX1gfuIsgd0flnq6yXwUV1p308y/p9WoPsQOPBa6j4atpNAQ3rXkNQx8uP+cMamWhdp/hQ5+vwYczERFp+L5es4t7P0jjYEERfxnVg/H9Y+t+KsvJPsloLeTnHD945W53huDKCmnmvE+zDhA3qHL4CmvtzKWqLUc/SwN9WtOXPl+DX4RWREQarvxCN098sYY3f9xC1zbhPHd1bzq3Dq/7Qio+yQhOr8ulz0H3y5whxeOFryMVttpzBTo9WxWHGUtfR0NgE6R+q24RWoUzERGpl9buzGXie0tYuyuXGwbFcd+IrgQHeGnSf3XzlYyf0zNGhd+1oS0q93SV/blJS+epRGnQqgtnGtYUEZF6xVrL2wu28vjnq2ka7M+bN/RlSJdW3iyo+if6bDEMvr98+GraDgJD67ZGqVcUzkREpN7Ye7CAP0xPY+6a3Qzp0pInRyfSMtxLk/6L3bDmU/jhOSr1jB0V0R7OfaBOy5L6T+FMRETqhfnrsvjdB8vIySvk0UvP4Poz47yzfmVhHix9FxY8D/s2QbN46HUNrPio/FpZDelJRqlTCmciIuLTCorcPPnlWl77fjOdWoXx9o396Nb2FywBUVsO74NFr8FP/4XDeyC6D4x5G7pe4jwV2fFcn3jST+o/hTMREfFZG3bnMvG9pazacYDrBnbgwYu61f2k//1bYMELsGQyFB6GTsNh0N3Q4czye0QmjFEYk1qhcCYiIj7HWsu7P6fzl09XERroz2vXJXP+Ga3rtojtS5z5ZKtmOtsYJVwFZ94JrbrVbR3S6CiciYiIT9l/6Aj3fZjGV6t2cXanFvzrykRaNa2jPXetdTbA/vFZ2DwfgprCmXdB/9udpyxF6oDCmYiI+IwfN+zht9OWsu/QEf50cTduHBSPn18dTPp3F8KKD52est0rIbwdDPsL9Jnwy7Y4EqkFCmciIuJ1R4qKefp/6/jv/I3Et2jC69f3pUd0hOffuCAXUt+ChS86G4W37AajXoIeo8E/0PPvL1IFhTMREfGqTVkHuXvqUpZn5nB1v1geuqQboYEe/vWUuxMWvgQpbzgbhsedDZc8A52GlZ/kL+IFCmciIuIV1lo+SMng0U9WEujvx8vj+zCiRxvPvmnWWvjxOWcvzOIi6DYSBk10lsUQ8REKZyIiPmLmkkyenL2W7dl5tIsM4d7hXRjVO9rbZdWasp+vTUQwrcODWJqRw8COUTx9VSJtI0I888bWQvoCZz7Zui/APwSSroeBv4bmHT3zniKnQOFMRMQHzFySyQMzlpNX6AYgMzuPB2YsB2gQAa3i59uRk8+OnHwuSWjLs2N74/LEpP9iN6z5zOkpy1gEoVEw5AHoews0iar99xOpJQpnIiI+4MnZa0uDy1F5hW7unb6MST9sPun71mrkOYW5WKu251Dorrz/5JL07NoPZoV5sOw9+PF52LcRmsXBRU9Br3HacFzqBYUzEREvs9aSmZ1X5bFCtyWqyck9NVjNVtwnd69TvFlVwQxgezWf+6RU3F6pXRJc+aYzr8yvjncVEDkFHg1nxpgRwLOAC3jNWvtEFecMAZ4BAoA91trBNb1WRKS+23OwgPs/XF7t8ejIEN64oV8dVuQZg574usoA2i6yFuaZ7d9asr3SOyXbK10AZ06EuLP05KXUSx4LZ8YYF/ACMAzIABYZY2ZZa1eVOScSeBEYYa1NN8a0qum1IiL13ZxVu7h/RhoH8osY1asds1fuJK+wuPR4SICLe4d38WKFtefe4V3KzTmDWvh825c688lWzgTj5+xreeZd2l5J6j1P9pz1AzZYazcBGGOmAr8Cygasa4AZ1tp0AGvt7l9wrYhIvXSooIi/fraK937eRre2TZlycy+6tAlv0E9rHv0cp/z5rIWNc50nLzd/C4HhMPD/YMAd2l5JGgxPhrNoYFuZ1xlA/wrndAYCjDHzgHDgWWvt2zW8FgBjzK3ArQCxsbG1UriIiKekbt3PPdOWkr7vMLcN7sg9wzoT5O/MhxrVO7rBhLGqnNLncxfCihlOT9muFRDeFoY9VrK9Uh3sJCBShzwZzqoa6K84I9Qf6AMMBUKABcaYhTW81mm09hXgFYDk5OTanP8qIlJrCt3FPDd3PS98s4G2ESFMvWUA/TtqOYcTKt1e6SU4kOFsr/SrF6HnldpeSRosT4azDKB9mdcxwPYqztljrT0EHDLGzAcSa3itiEi9sGH3QX77vrM90RVJMTw68gzCgwO8XZZvy90JP70MiyY52yt1OAsueRpOHwZ+ft6uTsSjPBnOFgGdjDHxQCYwFmeOWVkfA88bY/yBQJyhy38Da2pwrYiIT7PW8vaCrfzt89WEBrp4aVwSF/Zs6+2yfFvWupLtld4v2V7pUjjzbojR9krSeHgsnFlri4wxdwKzcZbDmGStXWmMub3k+MvW2tXGmC+BNKAYZ8mMFQBVXeupWkVEatuuA/ncOz2N+euyGNy5JU+OTqBV02Bvl+WbrIX0hU4oW/s5+AdD0nXORH9trySNkLGnurKgD0lOTrYpKSneLkNEGrnPl+/gwY+Wk1/o5o8XdWP8gA4YrbflbDY+9zHIyYCIGDjvTxDYBH541tleKaQ59LsV+t0CTVp4u1oRjzPGpFprkyu2a4cAEZFaciC/kEdnrWTG4kwSYyJ4+qpenNYyzNtl+Ya0afDJRGdrJYCcbfDR7YDV9koiFSiciYjUgoWb9vK7acvYeSCfiUM7cdd5pxPg0sT1UnMePRbMSllnM/I7U8GlX0ciR+nfBhGRU1BQ5Obpr9bxyneb6NA8lA9uH0hSbDNvl+V9RUecocrN38Kmb+FAZtXnHd6nYCZSgf6NEBE5SWt2HuA3U5eyZmcuV/eL5U8Xd6NJUCP9z2pxsbM47KZ5TiDb+qOzzyUG2vWGoHBnzbKKImLqulIRn9dI/ysiInLyiostr3+/mSdnr6VpiD+vX5/M0G6tvV1W3bIW9m92esU2fwub58Phvc6xqE7O/LGOg53Nx0OaVZ5zBhAQAkMf9k79Ij5M4UxE5BfIzM7jd9OWsnDTPoad0ZonLu9JVFiQt8uqGwd3OyHsaO9YdrrTHt7WWRy24xCIPwciqtiiKWGM873s05pDHz7WLiKlFM5ERGrAWsvHS7fz0McrKC62/POKBK5MjmnYS2QU5MKWH47NG9tdstxkUATEnw0D73ICWYtOUJM/h4QxCmMiNaBwJiJyAtmHj/DHmSv4LG0HfTo0499jehEb1QCXfCg3iX8eZKY6q/S7giB2gNPTFT8E2iZqEr+IB+nfLhGR4/hufRa//2AZew8e4d7hXbh98Gm4/BpIb1lxMexa7vSKbZoH6QucSfzGD9r2gjMnOvPG2vd35oeJSJ1QOBMRqUJ+oZsnvljDmz9u4fRWYbx+fV96REd4u6xTYy3s23RsmHLzfMjb5xxr0blkEv+Qkkn8kd6sVKRRUzgTEalgRWYOd09dwsasQ0w4M477L+xKcIDL22WdnNJJ/N/ApvmQc3QSfzvoPBziBzu9Y03bebdOESmlcCYiUsJdbHn52438+3/riAoL5J2b+nF2p5beLuuXKTeJfx7sXuW0B0dA3NkwaKITyGo6iV9E6pzCmYgIkL73ML+dtpTUrfu5OKEtj4/qQWRooLfLOrGiAmcS/9H1xjJSwLrLTOJ/xOkZa9sL/Opp759II6NwJiKNmrWWaSnbeOyTVfj5GZ4d24uRie18d4mM0kn885xAtvVHKMpzJvG36w2D7nbmjbXvDwHBXi5WRE6GwpmINFp7DhbwwIzl/G/VLgZ2jOKpMYlER3rxqcS0aZUXae15ZZlJ/PNg83dlJvF3gaRrnWFKTeIXaTAUzkSkUZq7ehf3fZjGgbwi/nRxN24cFI+fN5fIqLi9Uc42+Oh2+OK+Y/LI7eQAACAASURBVGEsvB10HuEMU8afo0n8Ig2UwpmINCqHCor462eree/ndLq2CWfyzf3p2qapt8uCuX8uv+8kOHPHCvPgoqecocqo0zWJX6QRUDgTkUZjcfp+7nl/KVv3Hea2wR25Z1hngvy9PEm+uBjWfOoMZValKB/63VK3NYmIVymciUiDV+gu5j9z1/P8NxtoGxHCe7cMYEDHKO8WZa0Tyub9w5ng7+fvbJVUUURM3dcmIl6lcCYiDdrGrIP89v2lpGXkcHlSNI+O7E7T4ADvFWQtrPkMvn0Cdi6H5qfBZf8FC3z2m/JDmwEhzkMBItKoKJyJSINkrWXywq08/vlqggNcvDguiYt6tvVmQbD2C5j3d9iZBs07wqiXnacxj24i7udX+WnNhDHeq1lEvELhTEQanN0H8rl3ehrfrsvinM4teXJ0Aq2bemnNL2th3ZdOKNuxDJrFw6iXoOeYY6HsqIQxCmMionAmIg3LF8t38MBHy8kvdPPYr7pz7YAO3llQ1lpYN7sklC2FZnHwqxch4arKoUxEpAz9F0JEGoQD+YU8OmslMxZnkhATwdNjenF6q7C6L8RaWP+VE8q2L4HIDjDyeUgcCy4vznUTkXpD4UxE6r2fNu3lnmnL2JGTx8TzTueuoZ0IcPnVbRHWwoY5TijLTIXIWBj5H0i8WqFMRH4RhTMRqbcKitw8/b91vDJ/E7HNQ/ng9jPp06FZ3RZhLWyYWxLKUiAiFi59zgll/vVg43QR8TkKZyJSL8xcksmTs9eyPTuPdpEhjB8Qy6xlO1i94wBX94vlTxd3o0lQHf4nzVrYOBfmPQEZiyCiPVz6LCReo1AmIqdE4UxEfN7MJZk8MGM5eYVuADKz8/jHl2sJC3Lx2nXJnH9G67orxlrY9A1883fI+NkJZZc8A73GKZSJSK1QOBMRn/fk7DWlwayssOCAugtm1sKmeU5P2baF0DQGLn4aeo8H/6C6qUFEGgWFMxHxOdmHj7AsI4dl27JZti2bzOz8Ks/blVN1e62yFjZ/64Sy9AXQNBou/hf0vlahTEQ8QuFMRLwqv9DNyu0HnCCW4YSxLXsPA2AMnN4yjNBAF4ePVO45axcZ4rnCrIXN80tC2Y8Q3g4uegqSrlMoExGPUjgTkTpTXGzZmHWQpaVBLIfVOw5QVGwBaNM0mMT2EYzp255eMZH0iImgaXBApTlnACEBLu4d3sUzhW7+znn6cusPEN4WLnzSCWUBXtplQEQaFYUzEfGYnTn5LN22n6XbnCHK5Zk5HCwoAiA8yJ+E9hHcek5HEttHkhgTSZuIqsPPqN7RAOWe1rx3eJfS9lqz5Xunp2zLdxDWBi78JyRdr1AmInVK4UxEasWB/EKWZ+Q4vWIlPWO7DhQAEOAydGvblMt6R5PYPpJe7SPo2CIMP7+ab6s0qnd07Yexo7b84PSUbfkOwlrDiH9AnwkKZSLiFQpnIvKLHSkqZs1OZ57YkpIwtjHrUOnxji2acOZpLUiMiSCxfSTd2jYlOMDlxYqrsXUBzPubM7csrDWMeKIklHlwLpuIyAkonIk0FGnTYO5jkJMBETEw9GFIGHPKty0utmzZe6h0jtjSbdms2n6AI+5iAFqEBdKrfSSjekWXDk9GhPr4dkXpC+GbvzlPYTZpBcP/Dsk3KJSJiE/waDgzxowAngVcwGvW2icqHB8CfAxsLmmaYa19rOTYb4GbAQssB26w1tbBc/Mi9VDaNPhkIhTmOa9ztjmv4RcHtN25+SwrmSN29OnJA/nOPLHQQBc9oyO4YVCcE8TaR9IuIhhjaj486VXpPznDl5u+gSYt4YLHIflGCAz1dmUiIqU8Fs6MMS7gBWAYkAEsMsbMstauqnDqd9baSypcGw1MBM6w1uYZY6YBY4E3PVWvSL0297FjweyowjyYdZezCXezOGgWX/K9Q2kP0aGCIpZnlg1iOWRmO/dx+Rm6tA7n4oR29GrvDE92ahWO6xfME/MZ2352QtnGr0tC2V8h+SaFMhHxSZ7sOesHbLDWbgIwxkwFfgVUDGfV8QdCjDGFQCiw3SNVijQEORlVtxflw5LJcORgueZs/5Zss61YeySKrcWt2Gpb4w7vwDkxnTntzK70im1G93YRhAT64DyxXyIjxRm+3DgXQlvAsL9A35sgsIm3KxMRqZYnw1k0sK3M6wygfxXnDTTGLMMJX7+31q601mYaY54C0oE84Ctr7VdVvYkx5lbgVoDY2NjarF+kfti7kWJj8LO20qH9Aa15vudHbE7fyqGdG2jt3kkHs4tOZNElaB8jXKsJOzLfObkA2AhsC4fmceV725rHOz9HxIDLx+eTAWSkOj1lG/4HoVEw7DHoe7NCmYjUC54MZ1WNfVT87bEY6GCtPWiMuQiYCXQyxjTD6WWLB7KBD4wx4621kyvd0NpXgFcAkpOTK/92EmnIstbB2yM5bAPxt26CTWHpocM2kEcOXcHsn9LpGR1JYv/znWUsYiJp3zzk2DyxwjzYvxX2b4b9W2BfyfestbDuK3AXHHs/44LI9seCW/P48iEuuGndffaqZKY665St/wpCmsP5j0LfWyAozLt1iYj8Ap4MZxlA+zKvY6gwNGmtPVDm58+NMS8aY1oA5wKbrbVZAMaYGcCZQKVwJtJo7VoFb48EDJcXPEZXs4U/+E+jndnLdhvFP4vG8EnxWaz7y3ACXH7V3ycgBFp1db4qKi6G3B1OcDsa2o6GuFUfQ96+8ueHRlXubTv6c1gb8DtOHacic3FJKJvthLKhj0C/WxXKRKRe8mQ4W4TTCxYPZOJM6L+m7AnGmDbALmutNcb0A/yAvTjDmQOMMaE4w5pDgRQP1ipSv+xYBm+PcvZ4vP4Tsv+7lVm5Mcw6cla506IjQ44fzE7Ezw8iop2vuLMqH8/PKd/bdjTEZSyClR+BLbMfpn8wRHao3NvWPN5pP9GCr1UtFdKikxPK1n0JIc2ctn63QlD4yX9mEREv81g4s9YWGWPuBGbjLKUxyVq70hhze8nxl4HRwB3GmCKcEDbWWmuBn4wx03GGPYuAJZQMXYo0epmp8M5lENQUrp9FVkA0he7NlU7z6N6TRwVHQNtE56sid6GzpMe+zRWGTLc62yRVeEiB8Hblg1vZnzfOrbxUyEe3gS2G4Eg47yHof5tCmYg0CMZWMYm4vkpOTrYpKepgkwYsfSFMHg1NouD6T8hvEs3YVxayZucBfj3kNN5flOHZvSdri7VwaE/5YdKyIS53R4ULDJWnrAJBEfDbFd6f6yYichKMManW2uSK7dohQKS+2PwdvHsVNG0L182iOLwdv3tvCcsysnlpXB9G9GjDxKGdvV1lzRgDYS2dr/Z9Kx8/chiy048Nk85+oOr7FBxQMBORBkfhTKQ+2Pg1vHeNs4DsdbMgvDX/mr2Gz5bv4MGLujKiRxtvV1i7AkPLP6Sw8EVnKLOiiJi6rUtEpA546NEpEak162bDu2Mh6nSY8BmEt2ZayjZe+GYjV/eL5ZazO3q7Qs8b+nDlfS8DQpx2EZEGRuFMxJet/gSmjoNW3eD6WdCkBT9u3MODM5ZzdqcWPPar7vVnX8tTkTAGLn0OItoDxvl+6XO1srG7iIiv0bCmiK9a8SF8eAtE94Hx0yE4go1ZB7lj8mLiWzTh+WuSTm2ZjPomYYzCmIg0Co3ov+wi9ciyqfDhzRA7AK6dAcER7Dt0hBvfXIS/n2HShL5EhNSDbZREROQXUzgT8TWpb8FHt0Pc2TDuAwgKp6DIzW3vpLAzJ59Xr0+mffNQb1cpIiIeonAm4kt+ftVZbPX08+Ga9yGwCdZa/jA9jUVb9vOvMYkkxTbzdpUiIuJBCmcivuLH5+Hz30OXi2HslNKnE5+du56Pl27n3uFduCShnZeLFBERT9MDASK+4Lt/OftGnjEKrngNXM58splLMnlmznpG94nh10NO83KRIiJSFxTORLzJWpj3d/j2H9BzDIx6CVzOv5aLtuzjD9PTGNCxOX+7rGfjWDJDREQUzkS8xlqY8yj88Az0Hu+s2+XnAmDLnkPc+nYKMc1CeHl8HwL9NQNBRKSxUDgT8QZrYfaDzrZEyTfBRU+BnxPAsg87S2YATJrQl8jQQG9WKiIidUzhTKSuFRc7E/9TXof+d8CIvzsbgQNHioq5fXIqGfvzmHJLf+JaNPFysSIiUtcUzkTqUrHbWSpjyWQY9Bs4/9HSYGat5cGPlrNw0z6euaoXfeOae7VUERHxDoUzkbriLoKZd8DyaTD4fhhyf2kwA3hx3kamp2Zw99BOjOod7cVCRUTEmxTOROqCu9DZjmnVTDjvITjn9+UOf5q2nSdnr2VUr3b85vxOXipSRER8gcKZiKcVFcAHN8Daz+CCx+HMO8sdXpy+n3umLaNvXDP+MTpBS2aIiDRyCmcinlSYB+9fCxv+5zyR2e+Wcoe37TvMLW+l0DYimP9em0yQv8tLhYqIiK9QOBPxlCOHYOo1sOlbZw2zPteXO5yTV8iNby6i0F3MpAl9ad5ES2aIiIjCmYhnFOTCu1dB+gJn1f9eV5c7XOgu5s53F7N5zyHevqkfp7UM81KhIiLiaxTORGpbfg5MHg2Zqc4+mT2uKHfYWsvDH6/ku/V7eHJ0Amee1sJLhYqIiC9SOBOpTYf3weTLYecKGPMWdLu00imvfreJ935O5//OPY0rk9t7oUgREfFlCmciteXQHnh7FOxZB2OnQOfhlU75csVO/v7FGi7u2ZbfDevihSJFRMTXKZyJ1IbcXfD2SNi/Fa5+D04fWumUtIxsfvP+EhJjIvnXmET8/LRkhoiIVKZwJnKqDmyHty6FAztg3AcQf3alUzKz87jprRRahAXx6nXJBAdoyQwREamawpnIqchOd4LZob1w7QyIHVDplNz8Qm56cxH5R9y8e3N/WoYHeaFQERGpLxTORE7Wvk3w1kgoOADXfQwxfSqdUuQu5q73lrB+90HevKEvnVqHe6FQERGpTxTORE7GnvVOj1lRAVz/CbRNrPK0v3y6inlrs/jbZT05u1PLOi5SRETqI4UzkV9q92qnxwwLEz6F1t2rPO2NHzbz1oKt3HpOR67pH1u3NYqISL3l5+0CROqVHWnw5sVg/GDC59UGs7mrd/GXT1dxwRmtuW9E1zouUkRE6jOFM5Gaykx1hjL9Q+CGz6Fl5ypPW7k9h7veW0L3dhE8M7YXLi2ZISIiv4CGNUVqIv0nmDIaQpo5c8yadajytJ05+dz0ZgoRIQG8fn0yoYH6V0xERH4Z9ZyJnMiW7+Gdy6BJS6fHrJpgdqigiJveWkRufiGTJvSlVdPgOi5UREQaAoUzkePZ+I2ziXlEjBPMImKqPM1dbLl76lJW7zjA89ck0a1t0zouVEREGgqPhjNjzAhjzFpjzAZjzP1VHB9ijMkxxiwt+Xq4zLFIY8x0Y8waY8xqY8xAT9YqUsm6r+Ddq6B5R5jwGYS3qfbUv32+mjmrd/HIpd05t2urOixSREQaGo9NiDHGuIAXgGFABrDIGDPLWruqwqnfWWsvqeIWzwJfWmtHG2MCgVBP1SpSyZrPYNr10PoMuHYmhDav9tR3Fm7l9e83M+HMOK4/M67uahQRkQbJkz1n/YAN1tpN1tojwFTgVzW50BjTFDgHeB3AWnvEWpvtsUpFyloxA6Zd5ywse92s4wazb9dl8eislZzXtRUPXXJGHRYpIiINlSfDWTSwrczrjJK2igYaY5YZY74wxhxdNKojkAW8YYxZYox5zRjTpKo3McbcaoxJMcakZGVl1eoHkEZo2fvw4U0Q0xeumwkhkdWeunZnLv83ZTGdW4fz3NW9tWSGiIjUCk+Gs6p+U9kKrxcDHay1icB/gJkl7f5AEvCStbY3cAioNGcNwFr7irU22Vqb3LKltseRU7D4HfjoNog7C8Z/CEHV74O5OzefG99cRGigi0kTkgkL0pIZIiJSOzwZzjKA9mVexwDby55grT1grT1Y8vPnQIAxpkXJtRnW2p9KTp2OE9ZEPGPRazDrTjh9KFwzDQKr7KgFIO+Im1veSmHfoSO8fn1f2kaE1GGhIiLS0HkynC0COhlj4ksm9I8FZpU9wRjTxhhjSn7uV1LPXmvtTmCbMaZLyalDgYoPEojUjgUvwGe/g84Xwth3IaD6sFVcbLln2lLSMnN4dmwvesZE1GGhIiLSGHhsLMZaW2SMuROYDbiASdbalcaY20uOvwyMBu4wxhQBecBYa+3Roc+7gCklwW4TcIOnapVG7LunYe6fodtIuOJ18A887un/nL2WL1bs5E8Xd+OC7tUvrSEiInKyzLEsdIITjekAdLLWzjHGhAD+1tpcj1b3CyUnJ9uUlBRvlyH1gbXw7T9g3t+h55Uw6mVwHf//Vab+nM79M5Yzrn8sfx3Vg5JOXxERkZNijEm11iZXbK9Rz5kx5hbgVqA5cBrO/LGXcYYbReqHtGkw9zHIyYCgMCjIhV7jYOR/wM913Et/2LCHP81cwdmdWvDnkd0VzERExGNqOufs/4BBwAEAa+16QMugS/2RNg0+mQg52wDrBDPjgvjBJwxmG3bncvvkVDq2bMIL45Lwd2nXMxER8Zya/pYpKFlIFgBjjD+Vl8UQ8V1zH4PCvPJt1g1f/+W4l+09WMANby4iyN/FpAl9aRoc4MEiRUREah7OvjXGPAiEGGOGAR8An3iuLJFalpPxy9qB/EI3t76Tyu4DBbx2fTIxzbSDmIiIeF5Nw9n9OCv2LwduAz4H/uSpokRqXUTML2ovLrbcOz2N1K37+fdVvejVvvqdAkRERGpTTcNZCM5SGFdaa0cDk0raROqHs+6p3BYQAkMfrvL0Z+as45Nl27lvRFcu6tnWw8WJiIgcU9NwNpfyYSwEmFP75Yh4yJGDzvewNoCBiPZw6XOQMKbSqR+mZvDc1xu4Krk9tw/uWLd1iohIo1fTRWiDj26zBGCtPWiM0QQcqR+KiyFlEsQOhBu/PO6pP23ay/0z0jjztCj+epnWMhMRkbpX056zQ8aY0r0tjTF9cFb0F/F9m+fB/s2QfNPxT9tziNsmpxLbPJSXxvUhQEtmiIiIF9S05+w3wAfGmKMbl7cFrvJMSSK1bNHrEBoFZ4ys9pT9h45wwxs/42cMb0zoR0SolswQERHvqFE4s9YuMsZ0BboABlhjrS30aGUiteHAdlj7BZx5J/gHVXlKQZGb295JZXtOPu/d0p/YKI3Yi4iI9xw3nBljzrPWfm2MubzCoU7GGKy1MzxYm8ipW/y2s9hsnxuqPGyt5YEPl/Pzln08O7YXfTo0r+MCRUREyjtRz9k5wNfApZTfEcCUvFY4E9/lLoLUt+C0odA8vspTnv96AzOWZHLPsM78qld0HRcoIiJS2YnCWa4x5h5gBU4YO/romrZuEt+37kvI3Q4XP1Xl4Y+XZvKv/63j8t7R3HXe6XVcnIiISNVOFM7CSr53AfoCH+MEtEuB+R6sS+TUpbwOTaOh0/BKh1K37uPe6Wn0i2/O36/oqSUzRETEZxw3nFlr/wxgjPkKSLLW5pa8fhRnf00R37RvE2z8GoY8CC7nr/nMJZk8OXst27PzMAaahwbw3/F9CPJ3eblYERGRY2q6kFMscKTM6yNAXK1XI1JbUt4A44Kk6wAnmD0wYzmZ2XlYoNhCboGbb9dlebdOERGRCmoazt4BfjbGPGqMeQT4CXjLc2WJnILCfFgyGbpeBE2dfTGfnL2WvEJ3udMKiop5cvZab1QoIiJSrZquc/a4MeYL4OySphustUs8V5bIKVg9C/L2ldsRYHt21RtaVNcuIiLiLTXdIQBr7WJgsQdrEakdi16H5h0hfnBpU7vIEDKrCGLtIkPqsjIREZET0uaB0rDsWgnbFkLyjeB37K/3jWfFVTo1JMDFvcO71GFxIiIiJ6ZwJg1LyiRwBUGvceWa9x50nmdp3TQIA0RHhvD3y3syqrcWnhUREd9S42FNEZ9XcBCWvQ/dL4PQY9swFRS5mZayjWFntObV65K9WKCIiMiJqedMGo7lH8CRXOh7U7nmL1fsZM/BI4wf0MFLhYmIiNScwpk0DNY6OwK07gExfcsdmrIwnQ5RoZx9egsvFSciIlJzCmfSMGSkwM7lzoMAZbZiWrPzAD9v2ce4/rH4+WmLJhER8X0KZ9IwpEyCwDBIGFOuecrCdAL9/biyT3svFSYiIvLLKJxJ/Xd4H6yc4QSzoPDS5oMFRcxYnMElCW1p1iTQiwWKiIjUnMKZ1H/L3oOi/HI7AoCzn+ahI249CCAiIvWKwpnUb9Y6Q5rt+0ObHmWaLZMXbqV7u6b0bh/pxQJFRER+GYUzqd82fwt7NzgPApSRunU/a3bmMn5AB4zRgwAiIlJ/KJxJ/ZYyCUKawRmjyjVPXriV8CB/ftWrnZcKExEROTkKZ1J/5e6ENZ85WzUFBJc27z1YwOfLd3JFnxhCA7UJhoiI1C8KZ1J/LX4HiosqDWlOS8ngiLuY8QNivVSYiIjIyVM4k/qp2A2pb0LHcyHqtNJmd7Flyk9bGdCxOae3Cq/+ehERER+lcCb107rZcCCjUq/Z/HVZZOzP49oBcd6pS0RE5BR5NJwZY0YYY9YaYzYYY+6v4vgQY0yOMWZpydfDFY67jDFLjDGferJOqYdSJkF4W+hyYbnmdxZupWV4EBd0b+2lwkRERE6Nx2ZLG2NcwAvAMCADWGSMmWWtXVXh1O+stZdUc5u7gdVAU0/VKfXQ/i2wYQ4M/gO4Akqbt+07zDdrd3PnuacT4FKnsIiI1E+e/A3WD9hgrd1krT0CTAV+VdOLjTExwMXAax6qT+qr1DfB+EHS9eWa3/05HQNc3U8PAoiISP3lyXAWDWwr8zqjpK2igcaYZcaYL4wx3cu0PwP8ASg+3psYY241xqQYY1KysrJOuWjxcUUFzlOaXS6EiGN/nQqK3ExbtI2h3VrTLjLEiwWKiIicGk+Gs6qWZbcVXi8GOlhrE4H/ADMBjDGXALuttaknehNr7SvW2mRrbXLLli1PtWbxdas/gcN7IPmGcs1frtjJ3kNHuFb7aIqISD3nyXCWAbQv8zoG2F72BGvtAWvtwZKfPwcCjDEtgEHASGPMFpzh0POMMZM9WKvUFymToFkcdDyvXPPkhVvpEBXKWae38E5dIiIitcST4WwR0MkYE2+MCQTGArPKnmCMaWNKNj40xvQrqWevtfYBa22MtTau5LqvrbXjPVir1Ae7V8PWH6DPDeB37K/ump0HWLRlP+P7d8DPT/toiohI/eaxpzWttUXGmDuB2YALmGStXWmMub3k+MvAaOAOY0wRkAeMtdZWHPoUcaRMAlcg9C6f0ycv3Eqgvx+j+8R4qTAREZHa49GNB0uGKj+v0PZymZ+fB54/wT3mAfM8UJ7UJ0cOwbKpzgbnTY4NXR4sKOKjxZlcmtCOZk0CvVigiIhI7dBiUFI/LJ8OBQcq7Qjw0ZJMDh1xax9NERFpMBTOpH5ImQStzoDYAaVN1lqmLNxKj+im9Gof6cXiREREao/Cmfi+zFTYsdTpNTPHJvynbN3Pmp25jO/fAWP0IICIiDQMCmfi+xZNgoAmkHBVuebJC7cSHuzPyF7tvFSYiIhI7VM4E9+Wtx9WfAgJV0LwsS1W9xws4PPlO7giKYbQQI8+1yIiIlKnFM7Ety2bCkV5lR4EmJayjUK3Zbx2BBARkQZG4Ux8l7XOgwDRydA2sbTZXWyZsjCdgR2jOL1VmBcLFBERqX0KZ+K7tnwPe9ZB35vKNX+7bjeZ2XnqNRMRkQZJ4Ux8V8rrEBwJ3S8r1/zOgq20DA/igu6tvVSYiIiI5yiciW86uBtWfwK9xkFASGnztn2Hmbcui6v7tifApb++IiLS8Oi3m/imxW9DcREk31CuecpP6fgZw9X9tSOAiIg0TApn4nuK3ZD6FsSfAy06lTYXFLmZlrKNoV1b0TYi5Dg3EBERqb8UzsT3bJgDOemQXP5BgC+W72TfoSNcO1APAoiISMOlcCa+Z9HrENYaul5crnnywq3ERYUy6LQWXipMRETE8xTOxLdkp8P6ryDpOnAFlDav3nGAlK37GT+gA35+2kdTREQaLoUz8S2pbzqbmyddX6558sKtBPn7MbpPjHfqEhERqSMKZ+I7io44T2l2Gg6R7Uubc/MLmbkkk0sT2xEZGujFAkVERDxP4Ux8x5pP4VBWpR0BZi7J5NARt3YEEBGRRkHhTHxHyiSIjIXTzittstYyeWE6PaMjSIyJ8GJxIiIidUPhTHxD1jrY8h30uQH8XKXNi7bsZ+2uXMYPiMUYPQggIiINn8KZ+IaUSeAXAL2vLdc8eeFWwoP9GZkY7aXCRERE6pbCmXjfkcOw7F04YySEtSxtzsot4IsVOxjdJ4aQQNdxbiAiItJwKJyJ962cAfk5lXYEmJayjUK3ZVx/PQggIiKNh8KZeN+i16FlV+hwZmmTu9jy7k/pnHlaFKe3CvNicSIiInVL4Uy8a/sS2L4Ykm90Fp8tMW/tbjKz87R8hoiINDoKZ+JdKZMgIBQSrirX/M7CrbQKD2LYGa29VJiIiIh3KJyJ9+TnwPLp0OMKCIksbU7fe5hv12Uxtl8sAS79FRURkcZFv/nEe5a9D4WHK+0IMOXnrfgZw9X92ldzoYiISMOlcCbeYS2kvA7tejtfJfIL3XyQksH53VrRNiLEiwWKiIh4h8KZeEf6AshaU2n5jC9W7GDfoSNcOyDOO3WJiIh4mcKZeMei1yEowplvVsbkhenEt2jCmadFeakwERER71I4k7p3MAtW/X97dx5ddX3nf/z5zkbCkkQgbCGyCAMiEpZAgktHiw5aQazbsKpsjq2KXYYZbedne3rm92vPOD9tXVorsighWrSItmrpjDo/bTXXLOwgyJaFNSxJWBK4ST6/P24aEoiQYG6+9yavxzmc3Lzz/d77yrmBvPiub8PIaRDTsW68ZV85eQXHmJF+REKFfQAAFrxJREFUORERuo+miIi0Typn0vrWZUKNP3Bts3oyfQV0iIrg7jF9PQomIiLiPZUzaV01NZC7FPpdB0lD6sbHK/2sXruX21P7kNgxxsOAIiIi3lI5k9a18wMoLYCxDbeavbV2L6fOVOuOACIi0u4FtZyZ2S1mts3MdpjZ4418/QYzKzOzdbV/nqydp5jZR2a21cw2m9ljwcwprSh3CXRKgqGT60bOOTKzC7g6OYHUlMQLrCwiItL2RQXric0sEngBuBkoBnLM7B3n3JZzFv3EOTfpnFkV8EPnXL6ZdQHyzOy/GllXwklpEWz/E1z7PYg6u+vy891H2X7wBP9x1wgPw4mIiISGYG45GwfscM7tcs6dAV4HpjRlRefcfudcfu3j48BWIDloSaV15L8auPjsmAcajDN9hcTHRjE5tY83uUREREJIMMtZMlBU7/NiGi9Y481svZm9b2ZXnftFM+sPjAJ8wQgpraTaHyhng2+Gy84eV1Zy/DR/2rSfu8ekEBcT6WFAERGR0BDMctbYharcOZ/nA/2cc6nAc8DqBk9g1hn4PfA951x5oy9i9qCZ5ZpZbklJSQvElqD44l04ceC8OwKszC3CX+2YkXG5R8FERERCSzDLWTFQ/87VfYF99RdwzpU7507UPn4PiDaz7gBmFk2gmK1wzq36qhdxzr3knEtzzqUlJSW19PcgLSV3CSSkBLac1aqucWT5Crl2UDeuSOrsYTgREZHQEcxylgMMNrMBZhYDTAXeqb+AmfUyM6t9PK42z5Ha2WJgq3Pu6SBmlNZweAfs/n8w5n6IOLvr8qMvDrG3tIKZ6bp8hoiIyN8E7WxN51yVmT0CrAEigSXOuc1m9lDt118E7ga+Y2ZVQAUw1TnnzOw6YBaw0czW1T7lj2q3rkm4yVsKEVEw6r4G4+XZBfSM78BNw3p6FExERCT0BK2cQd2uyvfOmb1Y7/HzwPONrPcXGj9mTcKNvwLWZsLQSdDlbAkrOHKSj78sYcE3BxMdqWshi4iI/I1+K0pwbX4LKkthbMMTAbJ8hUSYMW2cTgQQERGpT+VMgit3CXQbDP2vrxtV+qtZmVvEzVf2pFdCrIfhREREQo/KmQTP/g1QnANpc8DO7qV+b+N+jp3yM2u8TgQQERE5l8qZBE/uYoiKg5HTGowzswsY2L0T11zRzaNgIiIioUvlTIKjshw2vAHD74K4y+rGm/eVkV9YyoyMfpjpnA8REZFzBfVszbZk9dq9PLVmG/tKK+iTGMfCiUO4Y5Ru9/mVNvwO/CcDuzTrycwuJDY6grtH9/UomIiISGhTOWuC1Wv38sSqjVT4qwHYW1rBE6s2AqigNcY5yF0KvVMheXTduLzSz9vr9jJ5RB8SOkZ7GFBERCR0abdmEzy1ZltdMfubCn81T63Z5lGiEFfkg0ObzzsR4K38vZw6U60TAURERC5A5awJ9pVWNGve7uUshg7xcPU9dSPnHJnZBYzom8CIvokehhMREQltKmdN0Ccxrlnzdu3kEdiyGlKnQkynurFv91G+PHSCmRnaaiYiInIhKmdNsHDiEOKiI8+b3zg0yYM0IW5dJlSfaeREgALiY6OYPKKPR8FERETCg8pZE9wxKpmf33k1yYlxGNAnIZZBSZ3I8hXy7ob9XscLHTU1gRMBLr8GelxZNz50vJI/bTrAPWkpxMWcX3JFRETkLJ2t2UR3jEpucGbmqTNV3L/kcxa8vpaoSGPiVb08TBcidn0Ex3bDjT9uMF6ZU0RVjWNGuu6jKSIicjHacnaJOsZEseSBsVydnMAjWfl8+MVBryN5L3cJdOwOw26vG1XXOLJ8hVw3qDsDkzp7GE5ERCQ8qJx9DV1io3llzjiG9ornocx8Pt5e4nUk75TthW3vw6iZENWhbvzhF4fYV1bJzAxtNRMREWkKlbOvKSEumuVzx3FFUmfmv5rLpzsPex3JG/mvgquBMQ80GC/PLqBnfAduurKnN7lERETCjMpZC0jsGEPm3HH069aRuctyydlz1OtIrau6CvJfgUEToOuAunHBkZN8vL2EaeMuJypSP2oiIiJNod+YLaRb5w5kzkund0Iss5fmsLbwmNeRWs/29+H4/vMun7HCV0hkhDFtnHZpioiINJXKWQvq0SWWrPkZdOscw31LPmdjcZnXkVpHzmKIT4bBE+tGlf5qVuYW8Q/DetIzPtbDcCIiIuFF5ayF9UoIFLSEuGhmLfGxZV+515GC68jOwCU0xjwAkWevzPLuhv2UnvIzS3cEEBERaRaVsyBITozjtfkZxEVHMnOxj+0Hj3sdKXjyloJFwqhZDcaZvgIGJnVi/BXdPAomIiISnlTOmmrDSnhmOPw0MfBxw8oLLp7StSNZ8zOIijCmL/Kxs+REKwVtRf5KWLsCht4G8b3rxpv2lrG2sJQZ6f0wMw8DioiIhB+Vs6bYsBL+sADKigAX+PiHBRctaAO6dyJrfjrgmL4om4IjJ1slbqvZ8jZUHG3kRIACYqMjuHt0X4+CiYiIhC+Vs6b44Gfgr2g481cE5hcxqEcXMuelc6aqhumLfBQfOxWkkB7IXQxdr4ABf183Kq/0s3rtPm5P7UNCx2gPw4mIiIQnlbOmKCv+inlR4GbfFzG0VzzL56ZzvNLPtEXZ7C+ruOg6Ie/AJijyBbaaRZz9MVqVV0yFv5pZGf29yyYiIhLGVM6aIuECu+d+ez1s/SM4d8GnGJ6cwPK56ZSe9DN9kY9D5ZUtHLKV5S6ByA4wcnrdyDlHpq+Q1L4JXN03wcNwIiIi4UvlrCkmPAnRcQ1n0XGQNjewe/N3M+C33wjcW/ICJS01JZFlc8ZysLyS6S/7OHzidJCDB8np47DhdzD8TujYtW6cvesoOw6dYKYunyEiInLJVM6aYsS9MPlZSEgBLPBx8rMw6Wl4+HO44zdwuhxemwqLboTtf/7KkjamX1eWPjCW4mOnmPmyj2Mnz7Tu99ISNr4BZ04Eymk9mb4CEuKimZzax6NgIiIi4c/cRXbHhZO0tDSXm5vrzYtX+2H96/Dxf0BpISSnwY1PwBUToJHLSfx1x2HmLMthUI/OZM3LCJ+D552DF68PPH7ok7rv7VB5Jdf84kMeuKY//zZpmIcBRUREwoOZ5Tnn0s6da8tZS4mMhtGz4JE8mPwrOHEQMu+CJRNh50fnbUm7dlB3fjtrDF8ePMF9S3yUV/o9Ct5MxblwcCOMndOgdP4up4iqGscM7dIUERH5WlTOWlpUTOBWRo/mwW1PB870XH4HLP0W7P6kwaI3DOnBr2eMZvO+cmYvzeHE6SpvMjdH7mKI6QxX31M3qqquIevzQq4f3J0B3Tt5GE5ERCT8qZwFS1QHGDsXHs2HW5+CY7vhlUmwbBLs+WvdYjcN68lz00axrqiUuctyqDhT7WHoizh1FDatghH/CB261I0//OIQ+8sqmZGurWYiIiJfl8pZsEXHQvqDsGAt3PILOLwdln0LXrkdCrMBuPXq3jx9byo5e44y/9VcKv0hWtDWZUH16fPuCLA8u4Be8bHcdGUPj4KJiIi0HSpnrSU6DjK+AwvWwT/8bzi0JXA82vJvQ1EOU0Ym89Tdqfx152EeyszjdFWIFbSamsC1zVLSodfwuvGewyf55MvDTBt3OVGR+nESERH5uvTbtLXFdIRrHoHH1sPNP4P962HxTZB5N3f1OsjPv301/7OthIdXrOVM1cXvPtBq9nwMR3eed/mMFb4CIiOMqeNSPAomIiLStqiceSWmE1z7GDy2ASb8BPbmwqJvMnXHQp6/MYL/3nqQx15fS1V1iBS0nMUQ1xWGTakbVfqreSOvmIlX9aRnfKyH4URERNqOoJYzM7vFzLaZ2Q4ze7yRr99gZmVmtq72z5NNXbfN6NAZrv9BoKR989+gMJtJn03lf1IWsWezjx+sXE91jcfXoivfD1+8C6NmBI6hq/XHDfspPeVnpk4EEBERaTFBK2dmFgm8ANwKDAOmmVljVyf9xDk3svbPz5q5btsRGw/fWAjf2wA3/Ij+5fm83+EJbtnyLzyz4i1qvCxoa5eDq4YxsxuMM7MLGJjUifFXdPMomIiISNsTzC1n44AdzrldzrkzwOvAlIus0xLrhrfYBLjhXwMl7e//lQkxm/nnnbPZ9Ks7cYe2tn6e6irIWwYDb4RuV9SNN+0tY11RKTPT+2GN3AFBRERELk0wy1kyUFTv8+La2bnGm9l6M3vfzK5q5rqY2YNmlmtmuSUlJS2ROzTEJcKNPyLmh5v4LHk2A0s/xf16PO7NuVCyvfVyfLkGyvcGrtlWT2Z2AbHREdw1pm/rZREREWkHglnOGtuccu6+uXygn3MuFXgOWN2MdQND515yzqU559KSkpIuOWyoso5dyZj3DC+NeosXqybj3/Iu7tfpsOpBOLIz+AFyl0CX3vB3t9aNyir8vL1uH1NSk0mIC5N7goqIiISJYJazYqD+9RX6AvvqL+CcK3fOnah9/B4QbWbdm7Jue2JmfH/KeA6Ne5yMU8/wea/puK1/gOfT4K3vwNFdwXnho7thxwcw+n6IjKobr8ovpsJfzazxOhFARESkpQWznOUAg81sgJnFAFOBd+ovYGa9rPaAJTMbV5vnSFPWbW/MjJ9MHsYt6cP5x93fYtGoVZDxXdi8Cp5Lg7cfhmN7WvZF85aBRcCY++tGzjkyswtITUlkeHJCy76eiIiIBK+cOeeqgEeANcBWYKVzbrOZPWRmD9UudjewyczWA88CU11Ao+sGK2u4MDP+fcpw7hnTl//z8VFeiJkduJjtuAdhwxvw3Bh4ZwGUFn79F6s6HThLc8itEN+nbvzZriPsLDnJrAxtNRMREQmGqIsvculqd1W+d87sxXqPnweeb+q6AhERxi/uGoG/uoan1myjQ9SVzLv1F3DtAvjLM4GtXeuyYPR9cP0PIaHR8ygubss7cOrIeffRXJFdSEJcNJNG9P7634yIiIicR3cICEOREcZ/3pPKbVf35t/f3corn+4JbN361lOBG6yPngX5r8KzI+G9hYGLyDZX7hK4bEDgEhq1DpVXsmbzAe5N60tsdGTLfUMiIiJSR+UsTEVFRvDLqSO5eVhPfvLOZrJ8tbsyE/rCpGdgQT6kTguUrF+lwvuPw/GDTXvyg1ug8FNImw0RZ39EXs8poqrGMV13BBAREQkalbMwFh0ZwfPTR3HjkCR+vHojb+YVn/1i4uVw+7PwaB6MuAc+fylQ0tb8GE4cuvAT5y2FyBgYObNuVFVdQ5avkOsHd2dA905B+o5ERERE5SzMdYiK5Dczx3DdoO4sfHM9b6/b23CBy/rDlBfg0Vy46tuQ/etASfvz/4KTh89/wtMnYP3rMOwO6HT2tkwffHGIA+WVzNSJACIiIkGlctYGxEZH8tKsNNIHdOUHK9fz3sZGjjHrOhC+/Rt4OAeunAyfPQ+/HAH//VM4dRQ2rIRnhsPPk+F0OXQd0GD1zOwCeifEMmFoj9b5pkRERNoplbM2Ii4mksX3j2VUSiILXlvLnzcfaHzB7oPgzpfgu77AZTL+8kv4v0Nh9XegrN4dsz59NlDYgN2HT/LJl4eZNu5yoiL1IyMiIhJM+k3bhnTqEMXS2WMZnpzAw1n5fLTtAseWJf0d3L0YvpsdOOi/pqrh1/0V8MHPAFiRXUBUhDF1bEojTyQiIiItSeWsjekSG80rc8YxpFcX/ml5Hn/5spHjyurrMRT8lY1/rayYSn81b+QVM/GqXvSIj235wCIiItKAylkblBAXzfI56Qzs3ol5r+aQvevIRVbo+5XzP6zfR1mFnxkZl7d8UBERETmPylkbdVmnGDLnpZNyWUfmLMshd8/Rr154wpMQHddwFh0HE54k01fIFUmdGD+wW+PrioiISItSOWvDunfuwIp56fSKj+WBpTmsKyptfMER98LkZyEhBbDAx8nPsrHrRNYXlTIzox+196cXERGRIFM5a+N6xMeSNT+Drp1iuG+xj017yxpfcMS98P1N8NPSwMcR95KZXUBcdCR3jv6K3Z4iIiLS4lTO2oFeCbFkzU+nS2w0Mxf72Lq//KLrlFX4eXv9XqaM7ENCXHQrpBQRERFQOWs3+l7WkdfmZxAbFcnMl318efD4BZf/fV4xlf4a3RFARESklamctSOXd+tI1vx0IiKM6S/72FVyotHlnHNk+goYmZLI8OSEVk4pIiLSvqmctTMDkzqTNS+dmhrH9EU+Co+cOm+Zz3YeYVfJSWZpq5mIiEirUzlrhwb37ELmvHQqq6qZtiib4mMNC1qmr4DEjtHcNqK3RwlFRETaL5WzdurK3vFkzk3neKWf6Yt8HCgL3CXgYHklazYf5N60FGKjIz1OKSIi0v5EeR1AvDM8OYFX56Yz82Ufk5/7hMiICA6UB0paUucOHqcTERFpn7TlrJ0bmZLInOv6U3LiTF0xA3j6v7azeu1eD5OJiIi0Typnwu/zzi9hFf5qnlqzzYM0IiIi7ZvKmbCvtKJZcxEREQkelTOhT2Jcs+YiIiISPCpnwsKJQ4g758zMuOhIFk4c4lEiERGR9ktnawp3jEoG4Kk129hXWkGfxDgWThxSNxcREZHWo3ImQKCgqYyJiIh4T7s1RUREREKIypmIiIhICFE5ExEREQkhKmciIiIiIUTlTERERCSEqJyJiIiIhBCVMxEREZEQonImIiIiEkJUzkRERERCiMqZiIiISAgx55zXGVqMmZUABUF+me7A4SC/hgSX3sPwpvcv/Ok9DH96D1tGP+dc0rnDNlXOWoOZ5Trn0rzOIZdO72F40/sX/vQehj+9h8Gl3ZoiIiIiIUTlTERERCSEqJw130teB5CvTe9heNP7F/70HoY/vYdBpGPOREREREKItpyJiIiIhBCVsyYys1vMbJuZ7TCzx73OI81jZilm9pGZbTWzzWb2mNeZ5NKYWaSZrTWzP3qdRZrPzBLN7E0z+6L27+N4rzNJ05nZ92v/Dd1kZq+ZWazXmdoilbMmMLNI4AXgVmAYMM3MhnmbSpqpCvihc+5KIAN4WO9h2HoM2Op1CLlkvwL+5JwbCqSi9zJsmFkysABIc84NByKBqd6maptUzppmHLDDObfLOXcGeB2Y4nEmaQbn3H7nXH7t4+MEfiEke5tKmsvM+gK3AS97nUWaz8zigW8AiwGcc2ecc6XeppJmigLizCwK6Ajs8zhPm6Ry1jTJQFG9z4vRL/awZWb9gVGAz9skcgl+CfwLUON1ELkkA4ESYGntrumXzayT16GkaZxze4H/BAqB/UCZc+7P3qZqm1TOmsYamek01zBkZp2B3wPfc86Ve51Hms7MJgGHnHN5XmeRSxYFjAZ+45wbBZwEdAxvmDCzywjsNRoA9AE6mdlMb1O1TSpnTVMMpNT7vC/alBt2zCyaQDFb4Zxb5XUeabZrgdvNbA+BQwu+aWaZ3kaSZioGip1zf9tq/SaBsibh4SZgt3OuxDnnB1YB13icqU1SOWuaHGCwmQ0wsxgCB0C+43EmaQYzMwLHuWx1zj3tdR5pPufcE865vs65/gT+Dn7onNP/2sOIc+4AUGRmQ2pHE4AtHkaS5ikEMsysY+2/qRPQCR1BEeV1gHDgnKsys0eANQTOTlninNvscSxpnmuBWcBGM1tXO/uRc+49DzOJtEePAitq/6O7C5jtcR5pIuecz8zeBPIJnAG/Ft0pICh0hwARERGREKLdmiIiIiIhROVMREREJISonImIiIiEEJUzERERkRCiciYiIiISQlTOREREREKIypmIiIhICFE5ExEREQkh/x/jup45cUqxUQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,6))\n",
    "plt.plot(solver.dice_history, marker='o', label=\"training dice\")\n",
    "plt.plot(solver.val_dice_history, marker='o', label=\"validation dice\")\n",
    "plt.ylabel('dice')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "papermill": {
   "duration": 2206.911082,
   "end_time": "2020-11-03T09:48:37.559012",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2020-11-03T09:11:50.647930",
   "version": "2.1.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "1bcd00e9e22542fcb9744e364f028184": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_a640fcda900248bcb7895b080b2dcf84",
       "placeholder": "​",
       "style": "IPY_MODEL_3d84b1a0a2724999af6a1ca12f9ac7ed",
       "value": " 97.8M/97.8M [00:23&lt;00:00, 4.34MB/s]"
      }
     },
     "327784b17d6b4b209839eec6c5e56b58": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "100%",
       "description_tooltip": null,
       "layout": "IPY_MODEL_f3158190c92e4f52b91af95a30c89a1e",
       "max": 102502400.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_b7481b20cf4e4226b2c145f52340d170",
       "value": 102502400.0
      }
     },
     "3d84b1a0a2724999af6a1ca12f9ac7ed": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "a640fcda900248bcb7895b080b2dcf84": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "b7481b20cf4e4226b2c145f52340d170": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "deae50eb52024d17a6a0ca3119ca8ac8": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e84e611225a44b148af31f868b88c0d0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_327784b17d6b4b209839eec6c5e56b58",
        "IPY_MODEL_1bcd00e9e22542fcb9744e364f028184"
       ],
       "layout": "IPY_MODEL_deae50eb52024d17a6a0ca3119ca8ac8"
      }
     },
     "f3158190c92e4f52b91af95a30c89a1e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}