cloud-inference.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"3767a329c7ce472b84a7f9c4084a877f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"4ecd529f4da14147b1b0a43904754fa5":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"7aa2f0e562d0445a898473c2290eb382":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_c11056711eff47a3b000c889d2f3b73b","IPY_MODEL_daffa3587c154b6b88ec8e3a094b1b38"],"layout":"IPY_MODEL_7b1ad5123f424f32be8cc8812e59795c"}},"7b1ad5123f424f32be8cc8812e59795c":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a4fb4db19d664e43a4a7c514ef8c47fc":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b96328e61ff54400b4cbbcb4f25499b2":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c11056711eff47a3b000c889d2f3b73b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_a4fb4db19d664e43a4a7c514ef8c47fc","max":116,"min":0,"orientation":"horizontal","style":"IPY_MODEL_3767a329c7ce472b84a7f9c4084a877f","value":116}},"daffa3587c154b6b88ec8e3a094b1b38":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_b96328e61ff54400b4cbbcb4f25499b2","placeholder":"​","style":"IPY_MODEL_4ecd529f4da14147b1b0a43904754fa5","value":"100% 116/116 [03:08&lt;00:00,  1.42s/it]"}}},"version_major":2,"version_minor":0}},"colab":{"name":"cloud-inference.ipynb","provenance":[],"collapsed_sections":[]}},"cells":[{"cell_type":"markdown","metadata":{"id":"57-24i0K8pkc","colab_type":"text"},"source":["Reference:\n","\n","https://www.kaggle.com/artgor/segmentation-in-pytorch-using-convenient-tools  \n","https://www.kaggle.com/ryches/turbo-charging-andrew-s-pytorch"]},{"cell_type":"code","metadata":{"_kg_hide-output":true,"id":"5eQbT6b_8pkf","colab_type":"code","colab":{},"outputId":"b24692a4-811e-427f-f73f-14312272c5b3"},"source":["!pip install git+https://github.com/qubvel/segmentation_models.pytorch\n","# !pip install git+https://github.com/qubvel/ttach"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Collecting git+https://github.com/qubvel/segmentation_models.pytorch\r\n","  Cloning https://github.com/qubvel/segmentation_models.pytorch to /tmp/pip-req-build-p6jn2qi6\r\n","  Running command git clone -q https://github.com/qubvel/segmentation_models.pytorch /tmp/pip-req-build-p6jn2qi6\r\n","Requirement already satisfied: torchvision<=0.4.0,>=0.2.2 in /opt/conda/lib/python3.6/site-packages (from segmentation-models-pytorch==0.0.3) (0.4.0a0+6b959ee)\r\n","Collecting pretrainedmodels==0.7.4 (from segmentation-models-pytorch==0.0.3)\r\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/84/0e/be6a0e58447ac16c938799d49bfb5fb7a80ac35e137547fc6cee2c08c4cf/pretrainedmodels-0.7.4.tar.gz (58kB)\r\n","\u001b[K     |████████████████████████████████| 61kB 2.3MB/s \r\n","\u001b[?25hCollecting efficientnet-pytorch==0.4.0 (from segmentation-models-pytorch==0.0.3)\r\n","  Downloading https://files.pythonhosted.org/packages/12/f8/35453605c6c471fc406a137a894fb381b05ae9f174b2ca4956592512374e/efficientnet_pytorch-0.4.0.tar.gz\r\n","Requirement already satisfied: numpy in /opt/conda/lib/python3.6/site-packages (from torchvision<=0.4.0,>=0.2.2->segmentation-models-pytorch==0.0.3) (1.16.4)\r\n","Requirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from torchvision<=0.4.0,>=0.2.2->segmentation-models-pytorch==0.0.3) (1.12.0)\r\n","Requirement already satisfied: torch in /opt/conda/lib/python3.6/site-packages (from torchvision<=0.4.0,>=0.2.2->segmentation-models-pytorch==0.0.3) (1.2.0)\r\n","Requirement already satisfied: pillow>=4.1.1 in /opt/conda/lib/python3.6/site-packages (from torchvision<=0.4.0,>=0.2.2->segmentation-models-pytorch==0.0.3) (5.4.1)\r\n","Requirement already satisfied: munch in /opt/conda/lib/python3.6/site-packages (from pretrainedmodels==0.7.4->segmentation-models-pytorch==0.0.3) (2.3.2)\r\n","Requirement already satisfied: tqdm in /opt/conda/lib/python3.6/site-packages (from pretrainedmodels==0.7.4->segmentation-models-pytorch==0.0.3) (4.32.1)\r\n","Building wheels for collected packages: segmentation-models-pytorch, pretrainedmodels, efficientnet-pytorch\r\n","  Building wheel for segmentation-models-pytorch (setup.py) ... \u001b[?25l-\b \b\\\b \bdone\r\n","\u001b[?25h  Created wheel for segmentation-models-pytorch: filename=segmentation_models_pytorch-0.0.3-cp36-none-any.whl size=29990 sha256=109e8566afa97ccff67ea73074d0886768f5fd4ec3c9b325259558417bb5f695\r\n","  Stored in directory: /tmp/pip-ephem-wheel-cache-rw3at33p/wheels/79/3f/09/1587a252e0314d26ad242d6d2e165622ab95c95e5cfe4b942c\r\n","  Building wheel for pretrainedmodels (setup.py) ... \u001b[?25l-\b \b\\\b \bdone\r\n","\u001b[?25h  Created wheel for pretrainedmodels: filename=pretrainedmodels-0.7.4-cp36-none-any.whl size=60963 sha256=9e5b385724dea08b928d97048a33c684a00ffa89b0ce8982c226e9e0aa781d8e\r\n","  Stored in directory: /tmp/.cache/pip/wheels/69/df/63/62583c096289713f22db605aa2334de5b591d59861a02c2ecd\r\n","  Building wheel for efficientnet-pytorch (setup.py) ... \u001b[?25l-\b \b\\\b \bdone\r\n","\u001b[?25h  Created wheel for efficientnet-pytorch: filename=efficientnet_pytorch-0.4.0-cp36-none-any.whl size=11149 sha256=855a21735e44de7b6e2226e93af34dbf8f3cc0f4d749d721af661a07efaeae32\r\n","  Stored in directory: /tmp/.cache/pip/wheels/27/56/13/5bdaa98ca8bd7d5da65cc741987dd14391b87fa1a09081d17a\r\n","Successfully built segmentation-models-pytorch pretrainedmodels efficientnet-pytorch\r\n","Installing collected packages: pretrainedmodels, efficientnet-pytorch, segmentation-models-pytorch\r\n","Successfully installed efficientnet-pytorch-0.4.0 pretrainedmodels-0.7.4 segmentation-models-pytorch-0.0.3\r\n","\u001b[33mWARNING: You are using pip version 19.2.3, however version 19.3.1 is available.\r\n","You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\r\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"3gppMke28pkm","colab_type":"text"},"source":["## Importing and parameters"]},{"cell_type":"code","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","id":"Bnb0xaAs8pkn","colab_type":"code","colab":{}},"source":["import os, gc, time, json, copy\n","import pickle, random, itertools, collections\n","from tqdm import tqdm, tqdm_notebook\n","from functools import partial\n","\n","import numpy as np\n","import pandas as pd\n","from PIL import Image\n","import cv2\n","\n","from sklearn.model_selection import KFold, StratifiedKFold, train_test_split\n","from sklearn.metrics import roc_auc_score\n","\n","import torch\n","import torch.nn as nn\n","import torch.nn.functional as F\n","import torch.optim as optim\n","from torch.optim import lr_scheduler\n","from torch.utils.data import TensorDataset, DataLoader, Dataset\n","from torch.utils.data.sampler import SubsetRandomSampler\n","from torch.optim.lr_scheduler import StepLR, ReduceLROnPlateau, CosineAnnealingLR\n","\n","import albumentations as albu\n","from albumentations import pytorch as AT\n","import segmentation_models_pytorch as smp\n","# import ttach as tta\n","\n","import matplotlib.pyplot as plt\n","import seaborn as sns\n","%matplotlib inline"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"nspf-xOW8pkq","colab_type":"code","colab":{}},"source":["device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n","SEED = 2019\n","\n","label_classes = [\"Fish\", \"Flower\", \"Gravel\", \"Sugar\"]\n","\n","path = '../input/understanding_cloud_organization'\n","img_paths = '../input/understanding-clouds-resized'\n","model_path = '../input/cloud-shrt-warm-e18/'\n","# model_path = '../input/cloud-im-small/'\n","# model_path = '../input/cloud-ckpt-rn50-0/'\n","n_splits = 5\n","which_fold = 0  # should be int in [0, n_splits-1]\n","\n","batch_size = 32"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"5OLVSXDK8pkt","colab_type":"text"},"source":["## Helper functions and classes"]},{"cell_type":"code","metadata":{"_kg_hide-input":true,"id":"lvWIeBq18pku","colab_type":"code","colab":{}},"source":["sigmoid = lambda x: 1 / (1 + np.exp(-x))\n","\n","def seed_torch(seed=SEED):\n","    torch.manual_seed(seed)\n","    torch.cuda.manual_seed(seed)\n","    torch.backends.cudnn.deterministic = True\n","\n","def get_img(x, folder: str='train_images_525/train_images_525'):\n","    \"\"\"\n","    Return image based on image name and folder.\n","    \"\"\"\n","    data_folder = f\"{img_paths}/{folder}\"\n","    image_path = os.path.join(data_folder, x)\n","    img = cv2.imread(image_path)\n","    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n","    return img\n","\n","def rle_decode(mask_rle: str = '', shape: tuple = (1400, 2100)):\n","    \"\"\"\n","    Decode rle encoded mask.\n","    \n","    :param mask_rle: run-length as string formatted (start length)\n","    :param shape: (height, width) of array to return \n","    Returns numpy array, 1 - mask, 0 - background\n","    \"\"\"\n","    s = mask_rle.split()\n","    starts, lengths = [np.asarray(x, dtype=int) for x in (s[0:][::2], s[1:][::2])]\n","    starts -= 1\n","    ends = starts + lengths\n","    img = np.zeros(shape[0] * shape[1], dtype=np.uint8)\n","    for lo, hi in zip(starts, ends):\n","        img[lo:hi] = 1\n","    return img.reshape(shape, order='F')\n","\n","def make_mask(image_name: str='img.jpg', shape: tuple=(350, 525)):\n","    \"Create mask (ndarray, values between 0-1) based on image name and shape.\"\n","    masks = np.zeros((shape[0], shape[1], 4), dtype=np.float32)\n","    for classidx, classid in enumerate(label_classes):\n","        mask = cv2.imread(\"../input/understanding-clouds-resized/train_masks_525/train_masks_525/\" + classid + image_name)\n","        if mask is None:\n","            continue\n","        if mask[:,:,0].shape != (350,525):\n","            mask = cv2.resize(mask, (525,350))\n","        masks[:, :, classidx] = mask[:,:,0]\n","    masks = masks / 255\n","    return masks\n","\n","def mask2rle(img):\n","    \"\"\"\n","    Convert mask to rle.\n","    img: numpy array, 1 - mask, 0 - background\n","    Returns run length as string formated\n","    \"\"\"\n","    pixels= img.T.flatten()\n","    pixels = np.concatenate([[0], pixels, [0]])\n","    runs = np.where(pixels[1:] != pixels[:-1])[0] + 1\n","    runs[1::2] -= runs[::2]\n","    return ' '.join(str(x) for x in runs)\n","\n","\n","def visualize(image, mask, original_image=None, original_mask=None):\n","    \"\"\"\n","    Plot images and masks.\n","    \n","    Input\n","    - image: ndarray image loaded from cv2\n","    - mask: ndarray output from make_mask\n","    - original ones: If given, show them all\n","    \"\"\"\n","    fontsize = 14\n","    class_dict = {0: 'Fish', 1: 'Flower', 2: 'Gravel', 3: 'Sugar'}\n","    \n","    if original_image is None and original_mask is None:\n","        f, ax = plt.subplots(1, 5, figsize=(24, 24))\n","\n","        ax[0].imshow(image)\n","        for i in range(4):\n","            ax[i + 1].imshow(mask[:, :, i])\n","            ax[i + 1].set_title(f'Mask {class_dict[i]}', fontsize=fontsize)\n","    else:\n","        f, ax = plt.subplots(2, 5, figsize=(24, 12))\n","\n","        ax[0, 0].imshow(original_image)\n","        ax[0, 0].set_title('Original image', fontsize=fontsize)\n","                \n","        for i in range(4):\n","            ax[0, i + 1].imshow(original_mask[:, :, i])\n","            ax[0, i + 1].set_title(f'Original mask {class_dict[i]}', fontsize=fontsize)\n","        \n","        ax[1, 0].imshow(image)\n","        ax[1, 0].set_title('Transformed image', fontsize=fontsize)        \n","        \n","        for i in range(4):\n","            ax[1, i + 1].imshow(mask[:, :, i])\n","            ax[1, i + 1].set_title(f'Transformed mask {class_dict[i]}', fontsize=fontsize)\n","            \n","            \n","def visualize_with_raw(image, mask, original_image=None, original_mask=None, raw_image=None, raw_mak=None):\n","    \"\"\"\n","    Plot images and masks.\n","    \n","    Input\n","    - image: ndarray image loaded from cv2\n","    - mask: ndarray output from make_mask\n","    - original ones / raw ones: If given, show them all\n","    \"\"\"\n","    fontsize = 14\n","    class_dict = {0: 'Fish', 1: 'Flower', 2: 'Gravel', 3: 'Sugar'}\n","\n","    f, ax = plt.subplots(3, 5, figsize=(24, 12))\n","\n","    ax[0, 0].imshow(original_image)\n","    ax[0, 0].set_title('Original image', fontsize=fontsize)\n","\n","    for i in range(4):\n","        ax[0, i + 1].imshow(original_mask[:, :, i])\n","        ax[0, i + 1].set_title(f'Original mask {class_dict[i]}', fontsize=fontsize)\n","\n","    ax[1, 0].imshow(raw_image)\n","    ax[1, 0].set_title('Original image', fontsize=fontsize)\n","\n","    for i in range(4):\n","        ax[1, i + 1].imshow(raw_mak[:, :, i])\n","        ax[1, i + 1].set_title(f'Raw predicted mask {class_dict[i]}', fontsize=fontsize)\n","        \n","    ax[2, 0].imshow(image)\n","    ax[2, 0].set_title('Transformed image', fontsize=fontsize)\n","\n","    for i in range(4):\n","        ax[2, i + 1].imshow(mask[:, :, i])\n","        ax[2, i + 1].set_title(f'Predicted mask with processing {class_dict[i]}', fontsize=fontsize)\n","            \n","            \n","def plot_with_augmentation(image, mask, augment):\n","    \"\"\"\n","    Plot images, masks and augmentation results.\n","    \n","    Input\n","    - image: ndarray image loaded from cv2\n","    - mask: ndarray output from make_mask\n","    - augment: transformation from albumentations\n","    \"\"\"\n","    augmented = augment(image=image, mask=mask)\n","    image_flipped = augmented['image']\n","    mask_flipped = augmented['mask']\n","    visualize(image_flipped, mask_flipped, original_image=image, original_mask=mask)\n","\n","\n","def post_process(probability, threshold, min_size):\n","    \"\"\"\n","    Post processing of each predicted mask, components with lesser number of pixels\n","    than `min_size` are ignored\n","    \n","    Input\n","    - probability: predicted probability mask, ndarray (350, 525)\n","    - threshold: value to binarize probability mask\n","    - min_size: ??\n","    \"\"\"\n","    mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]\n","    num_component, component = cv2.connectedComponents(mask.astype(np.uint8))\n","    predictions = np.zeros((350, 525), np.float32)\n","    num = 0\n","    for c in range(1, num_component):\n","        p = (component == c)\n","        if p.sum() > min_size:\n","            predictions[p] = 1\n","            num += 1\n","    return predictions, num\n","\n","\n","def get_validation_augmentation():\n","    \"\"\"Add paddings to make image shape divisible by 32\"\"\"\n","    test_transform = [\n","        albu.Resize(256, 384)\n","#         albu.Resize(320, 640)\n","    ]\n","    return albu.Compose(test_transform)\n","\n","def to_tensor(x, **kwargs):\n","    \"\"\"\n","    Convert image or mask.\n","    \"\"\"\n","    return x.transpose(2, 0, 1).astype('float32')\n","\n","def get_preprocessing(preprocessing_fn):\n","    \"\"\"Construct preprocessing transform\n","    \n","    Args:\n","        preprocessing_fn (callbale): data normalization function \n","            (can be specific for each pretrained neural network)\n","    Return:\n","        transform: albumentations.Compose\n","    \n","    \"\"\"\n","    _transform = [\n","        albu.Lambda(image=preprocessing_fn),\n","        albu.Lambda(image=to_tensor, mask=to_tensor),\n","    ]\n","    return albu.Compose(_transform)\n","\n","\n","def compute_dice(img1, img2):\n","    img1 = np.asarray(img1).astype(np.bool)\n","    img2 = np.asarray(img2).astype(np.bool)\n","\n","    intersection = np.logical_and(img1, img2)\n","\n","    return 2. * intersection.sum() / (img1.sum() + img2.sum())"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"PojbgiT88pkw","colab_type":"text"},"source":["## Data overview\n","\n","Let's have a look at the data first.\n","We have folders with train and test images, file with train image ids and masks and sample submission."]},{"cell_type":"code","metadata":{"id":"GJTqWOWa8pkx","colab_type":"code","colab":{}},"source":["train = pd.read_csv(f'{path}/train.csv')\n","sub = pd.read_csv(f'{path}/sample_submission.csv')"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"nIsYO_qO8pk0","colab_type":"code","colab":{},"outputId":"e54969a4-f856-447a-c362-1c9d47ed3838"},"source":["n_train = len(os.listdir(f'{img_paths}/train_images_525/train_images_525'))\n","n_test = len(os.listdir(f'{img_paths}/test_images_525/test_images_525'))\n","print(f'There are {n_train} images in train dataset')\n","print(f'There are {n_test} images in test dataset')"],"execution_count":0,"outputs":[{"output_type":"stream","text":["There are 5546 images in train dataset\n","There are 3698 images in test dataset\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"MXV9DmZ38pk2","colab_type":"text"},"source":["## Preparing data for modelling\n","\n","At first, let's create a list of unique image ids and the count of masks for images. This will allow us to make a stratified split based on this count."]},{"cell_type":"code","metadata":{"id":"NmaL6cVE8pk3","colab_type":"code","colab":{}},"source":["def train_val_split(id_mask_count, n_splits=None, which_fold=None):\n","    kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n","    cv_indices = [(tr_idx, val_idx) for tr_idx, val_idx in kf.split(id_mask_count['img_id'], id_mask_count['count'])]\n","    if which_fold is not None:\n","        return cv_indices[which_fold]\n","    return cv_indices"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"POfVCF-T8pk6","colab_type":"code","colab":{}},"source":["id_mask_count = (train.loc[~train['EncodedPixels'].isnull(), 'Image_Label']\n","                    .apply(lambda x: x.split('_')[0])\n","                    .value_counts()\n","                    .reset_index()\n","                    .rename(columns={'index': 'img_id', 'Image_Label': 'count'}))\n","\n","trn_idx, val_idx = train_val_split(id_mask_count, n_splits, which_fold)\n","train_ids, valid_ids = id_mask_count['img_id'].values[trn_idx], id_mask_count['img_id'].values[val_idx]\n","test_ids = sub['Image_Label'].apply(lambda x: x.split('_')[0]).drop_duplicates().values"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"_FZWTFGJ8pk8","colab_type":"code","colab":{},"outputId":"7b38666b-fc67-43b0-ffc5-f5f7284f85f2"},"source":["len(train_ids), len(valid_ids)"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(4435, 1111)"]},"metadata":{"tags":[]},"execution_count":9}]},{"cell_type":"code","metadata":{"id":"myvKVpfU8pk_","colab_type":"code","colab":{},"outputId":"e7999bdb-1573-45b9-872a-fcb55ec6f279"},"source":["id_mask_count"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>img_id</th>\n","      <th>count</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <td>0</td>\n","      <td>23d1c12.jpg</td>\n","      <td>4</td>\n","    </tr>\n","    <tr>\n","      <td>1</td>\n","      <td>2f79ff9.jpg</td>\n","      <td>4</td>\n","    </tr>\n","    <tr>\n","      <td>2</td>\n","      <td>6815a8b.jpg</td>\n","      <td>4</td>\n","    </tr>\n","    <tr>\n","      <td>3</td>\n","      <td>04d92ec.jpg</td>\n","      <td>4</td>\n","    </tr>\n","    <tr>\n","      <td>4</td>\n","      <td>e9c15bb.jpg</td>\n","      <td>4</td>\n","    </tr>\n","    <tr>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <td>5541</td>\n","      <td>67e8608.jpg</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <td>5542</td>\n","      <td>c7bfd1b.jpg</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <td>5543</td>\n","      <td>63e8002.jpg</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <td>5544</td>\n","      <td>a7fdbc3.jpg</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <td>5545</td>\n","      <td>dd5038a.jpg</td>\n","      <td>1</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>5546 rows × 2 columns</p>\n","</div>"],"text/plain":["           img_id  count\n","0     23d1c12.jpg      4\n","1     2f79ff9.jpg      4\n","2     6815a8b.jpg      4\n","3     04d92ec.jpg      4\n","4     e9c15bb.jpg      4\n","...           ...    ...\n","5541  67e8608.jpg      1\n","5542  c7bfd1b.jpg      1\n","5543  63e8002.jpg      1\n","5544  a7fdbc3.jpg      1\n","5545  dd5038a.jpg      1\n","\n","[5546 rows x 2 columns]"]},"metadata":{"tags":[]},"execution_count":10}]},{"cell_type":"code","metadata":{"id":"pAyR-0Cm8plC","colab_type":"code","colab":{},"outputId":"386e33a7-ab19-4209-da36-8823f3865eab"},"source":["id_mask_count['count'].value_counts()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["2    2372\n","3    1560\n","1    1348\n","4     266\n","Name: count, dtype: int64"]},"metadata":{"tags":[]},"execution_count":11}]},{"cell_type":"markdown","metadata":{"id":"USkkJtCQ8plE","colab_type":"text"},"source":["There are a lot of empty masks. In fact only 266 images have all four masks. It is important to remember this."]},{"cell_type":"markdown","metadata":{"id":"CXZH2hEG8plF","colab_type":"text"},"source":["## Setting up data loader, model, solver"]},{"cell_type":"code","metadata":{"id":"N1FGyZM98plG","colab_type":"code","colab":{}},"source":["class CloudDataset(Dataset):\n","    def __init__(self, dataset_type: str='train', img_ids: np.array=None,\n","                 transforms=albu.Compose([albu.HorizontalFlip(), AT.ToTensor()]),\n","                 preprocessing=None, label_smooth=0):\n","        self.dataset_type = dataset_type\n","        if dataset_type != 'test':\n","            self.data_folder = f\"{img_paths}/train_images_525/train_images_525\"\n","        else:\n","            self.data_folder = f\"{img_paths}/test_images_525/test_images_525\"\n","        self.img_ids = img_ids\n","        self.transforms = transforms\n","        self.preprocessing = preprocessing\n","        self.label_smooth = label_smooth\n","\n","    def __getitem__(self, idx):\n","        image_name = self.img_ids[idx]\n","        mask = make_mask(image_name)\n","        img = cv2.imread(os.path.join(self.data_folder, image_name))\n","        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n","        augmented = self.transforms(image=img, mask=mask)\n","        img = augmented['image']\n","        mask = augmented['mask']\n","        if self.label_smooth > 0:\n","            mask = np.where(mask==1, 1-label_smooth, label_smooth).astype('float32')\n","        if self.preprocessing:\n","            preprocessed = self.preprocessing(image=img, mask=mask)\n","            img = preprocessed['image']\n","            mask = preprocessed['mask']\n","        return img, mask\n","\n","    def __len__(self):\n","        return len(self.img_ids)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"clOn6PjA8plI","colab_type":"code","colab":{},"outputId":"79e6a16d-fd36-469c-ddba-419b281e3954"},"source":["ENCODER = 'resnet50'\n","ENCODER_WEIGHTS = None\n","ACTIVATION = None\n","\n","preprocessing_fn = smp.encoders.get_preprocessing_fn(ENCODER)\n","\n","model = smp.Unet(\n","    encoder_name=ENCODER, \n","    encoder_weights=ENCODER_WEIGHTS, \n","    classes=4, \n","    activation=ACTIVATION\n",")\n","model = model.to(device)\n","model.load_state_dict(torch.load(model_path+'best_model_optim.pt')['model'])"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<All keys matched successfully>"]},"metadata":{"tags":[]},"execution_count":13}]},{"cell_type":"code","metadata":{"id":"YcfTe0bE8plL","colab_type":"code","colab":{},"outputId":"22a8c016-c8da-491a-fee8-52e2d34a6607"},"source":["print(str(torch.cuda.memory_allocated(device)/1e6 ) + 'M')\n","print(str(torch.cuda.memory_cached(device)/1e6 ) + 'M')\n","torch.cuda.empty_cache()"],"execution_count":0,"outputs":[{"output_type":"stream","text":["131.664384M\n","555.74528M\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"4dY4bZAX8plO","colab_type":"code","colab":{},"outputId":"cfd2768f-ae91-4083-e6ea-ad699e1ed4f7"},"source":["test_dataset = CloudDataset(dataset_type='test', img_ids=test_ids,\n","                             transforms=get_validation_augmentation(),\n","                             preprocessing=get_preprocessing(preprocessing_fn))\n","\n","test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["/opt/conda/lib/python3.6/site-packages/albumentations/augmentations/transforms.py:2567: UserWarning: Using lambda is incompatible with multiprocessing. Consider using regular functions or partial().\n","  warnings.warn('Using lambda is incompatible with multiprocessing. '\n"],"name":"stderr"}]},{"cell_type":"code","metadata":{"id":"1nuLkRRm8plQ","colab_type":"code","colab":{}},"source":["# tta_transforms = tta.Compose(\n","#     [\n","#         tta.HorizontalFlip(),\n","#         tta.VerticalFlip(),\n","#         tta.Rotate90(angles=[0, 90, 180, 270]),\n","#     ]\n","# )\n","\n","# # model = tta.SegmentationTTAWrapper(model, tta_transforms)\n","# model = tta.SegmentationTTAWrapper(model, tta.aliases.d4_transform(), merge_mode='mean')"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"fRhyNrnY8plS","colab_type":"text"},"source":["## Model inference"]},{"cell_type":"code","metadata":{"id":"KAvIuXYf8plT","colab_type":"code","colab":{}},"source":["def get_preds(loader, model, dataset_type='test'):\n","    \"\"\"\n","    Inference for testset.\n","\n","    :param loader: DataLoader for testing data\n","    :param model: model\n","    :param dataset_type: dataset type\n","\n","    :return: list of batch predictions\n","    \"\"\"\n","    assert dataset_type in ('val', 'test')\n","    model.eval()  # eval mode disables dropout\n","\n","    if dataset_type=='val':\n","        label_masks = []\n","    predictions = []\n","    with torch.no_grad():\n","        # Batches\n","        for images, masks in tqdm_notebook(loader):\n","            if dataset_type=='val':\n","                for m in masks.view(-1, 256, 384).numpy():\n","#                 for m in masks.view(-1, 320, 640).numpy():\n","                    m = cv2.resize(m, dsize=(525, 350), interpolation=cv2.INTER_LINEAR)\n","                    label_masks.append(m)\n","            \n","            # Move to default device\n","            images = images.to(device)  # (N, 3, 320, 640)\n","            \n","            # Forward prop.\n","            pred_scores = model(images)  # (N, 4, 320, 640)\n","            pred_scores = pred_scores.view(-1, 256, 384)\n","#             pred_scores = pred_scores.view(-1, 320, 640)  # (N*4, 320, 640)\n","            \n","            # Resize to required size\n","            for m in torch.sigmoid(pred_scores).cpu().numpy():\n","                m = cv2.resize(m, dsize=(525, 350), interpolation=cv2.INTER_LINEAR)\n","                predictions.append(m)\n","    torch.cuda.empty_cache()\n","\n","    if dataset_type=='val':\n","        return predictions, label_masks\n","    return predictions"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"ubn6WJAT8plW","colab_type":"code","colab":{}},"source":["class Rot90(object):\n","    \"\"\"rotate batch of images by 90 degrees k times\"\"\"\n","    def __init__(self, k=1):\n","        self.k = k\n","    def transform(self, x):\n","        return torch.rot90(x, self.k, (2, 3))\n","    def inverse(self, x):\n","        return torch.rot90(x, -self.k, (2, 3))\n","\n","class Hflip(object):\n","    \"\"\"flip batch of images horizontally\"\"\"\n","    def transform(self, x):\n","        return x.flip(3)\n","    def inverse(self, x):\n","        return x.flip(3)\n","\n","class Vflip(object):\n","    \"\"\"flip batch of images vertically\"\"\"\n","    def transform(self, x):\n","        return x.flip(2)\n","    def inverse(self, x):\n","        return x.flip(2)\n","\n","def TTA(loader, model, dataset_type='test', beta=0.4):\n","    \"\"\"\n","    Inference for testset with test-time augmentation (TTA).\n","\n","    :param loader: DataLoader for testing data\n","    :param model: model\n","    :param dataset_type: dataset type\n","    :param beta: ratio of prediction from original dataset in final results\n","\n","    :return: list of batch predictions\n","    \"\"\"\n","    assert dataset_type in ('val', 'test')\n","    model.eval()  # eval mode disables dropout\n","    tsfms = [Vflip(), Hflip(), Rot90(k=1), Rot90(k=2), Rot90(k=3)]\n","\n","    if dataset_type=='val':\n","        label_masks = []\n","    predictions = []\n","    with torch.no_grad():\n","        # Batches\n","        for images, masks in tqdm_notebook(loader):\n","            if dataset_type=='val':\n","                for m in masks.view(-1, 256, 384).numpy():\n","#                 for m in masks.view(-1, 320, 640).numpy():\n","                    m = cv2.resize(m, dsize=(525, 350), interpolation=cv2.INTER_LINEAR)\n","                    label_masks.append(m)\n","            \n","            # Move to default device\n","            images = images.to(device)  # (N, 3, 320, 640)\n","            \n","            # Forward prop.\n","            pred_scores = beta * model(images)  # (N, 4, 320, 640)\n","            \n","            # TTA\n","            for tsfm in tsfms:\n","                pred_scores += (1-beta) * tsfm.inverse(model(tsfm.transform(images))) / len(tsfms)\n","            \n","            # Resize to required size\n","            pred_scores = pred_scores.view(-1, 256, 384)\n","#             pred_scores = pred_scores.view(-1, 320, 640)  # (N*4, 320, 640)\n","            for m in torch.sigmoid(pred_scores).cpu().numpy():\n","                m = cv2.resize(m, dsize=(525, 350), interpolation=cv2.INTER_LINEAR)\n","                predictions.append(m)\n","    torch.cuda.empty_cache()\n","\n","    if dataset_type=='val':\n","        return predictions, label_masks\n","    return predictions"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"jBKgzhV58plb","colab_type":"code","colab":{}},"source":["def TTA_inference(loader, model, class_params, beta=0.4):\n","    \"\"\"\n","    Inference for testset with test-time augmentation (TTA).\n","\n","    :param loader: DataLoader for testing data\n","    :param model: model\n","    :param dataset_type: dataset type\n","    :param beta: ratio of prediction from original dataset in final results\n","\n","    :return: list of batch predictions\n","    \"\"\"\n","    model.eval()  # eval mode disables dropout\n","    tsfms = [Vflip(), Hflip(), Rot90(k=1), Rot90(k=2), Rot90(k=3)]\n","\n","    encoded_pixels = []\n","    image_id = 0\n","    \n","    with torch.no_grad():\n","        # Batches\n","        for images, masks in tqdm_notebook(loader):\n","            \n","            # Move to default device\n","            images = images.to(device)  # (N, 3, 320, 640)\n","            \n","            # Forward prop.\n","            pred_scores = beta * model(images)  # (N, 4, 320, 640)\n","            \n","            # TTA\n","            for tsfm in tsfms:\n","                pred_scores += (1-beta) * tsfm.inverse(model(tsfm.transform(images))) / len(tsfms)\n","            \n","            # Resize back and encode to rle\n","#             pred_scores = pred_scores.view(-1, 320, 640)  # (N*4, 320, 640)\n","            pred_scores = pred_scores.view(-1, 256, 384)  # (N*4, 256, 384)\n","            for prob_mask in torch.sigmoid(pred_scores).cpu().numpy():\n","                prob_mask = cv2.resize(prob_mask, dsize=(525, 350), interpolation=cv2.INTER_LINEAR)\n","                pred, num_pred = post_process(prob_mask, *class_params[image_id%4])\n","                if num_pred == 0:\n","                    encoded_pixels.append('')\n","                else:\n","                    r = mask2rle(pred)\n","                    encoded_pixels.append(r)\n","                image_id += 1\n","    \n","    torch.cuda.empty_cache()\n","\n","    return encoded_pixels"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"dref70cb8plf","colab_type":"code","colab":{}},"source":["# class_params = [(0.45, 22500), (0.3, 22500), (0.5, 15000), (0.4, 10000)]\n","# class_params = [(0.45, 22500), (0.5, 22500), (0.45, 22500), (0.4, 10000)]\n","# class_params = [(0.45, 22500), (0.45, 22500), (0.5, 15000), (0.4, 12500)]\n","# class_params = [(0.45, 22500), (0.45, 22500), (0.45, 22500), (0.45, 20000)]\n","# class_params = [(0.4, 25000), (0.2, 22500), (0.2, 22500), (0.5, 10000)]\n","# class_params = [(0.35, 22500), (0.35, 20000), (0.4, 25000), (0.4, 10000)]\n","class_params = [(0.3, 15000), (0.5, 10000), (0.45, 20000), (0.35, 15000)]"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"DW40NR1G8plk","colab_type":"text"},"source":["### Prediction from testset"]},{"cell_type":"code","metadata":{"id":"qnJCjaRS8plk","colab_type":"code","colab":{},"outputId":"c0848f37-fc70-4353-de73-18aef5aa684c"},"source":["# predictions = get_preds(test_loader, model) # list\n","# predictions = TTA(test_loader, model) # list\n","encoded_pixels = TTA_inference(test_loader, model, class_params)\n","\n","print(str(torch.cuda.memory_allocated(device)/1e6 ) + 'M')\n","print(str(torch.cuda.memory_cached(device)/1e6 ) + 'M')\n","torch.cuda.empty_cache()"],"execution_count":0,"outputs":[{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"7aa2f0e562d0445a898473c2290eb382","version_major":2,"version_minor":0},"text/plain":["HBox(children=(IntProgress(value=0, max=116), HTML(value='')))"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n","131.664384M\n","394.264576M\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"b_3X-5CE8pln","colab_type":"code","colab":{}},"source":["# encoded_pixels = []\n","# image_id = 0\n","# for prob_mask in predictions:\n","#     pred, num_pred = post_process(prob_mask, *class_params[image_id%4])\n","#     if num_pred == 0:\n","#         encoded_pixels.append('')\n","#     else:\n","#         r = mask2rle(pred)\n","#         encoded_pixels.append(r)\n","#     image_id += 1"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"r4dFMEzi8plr","colab_type":"code","colab":{}},"source":["sub['EncodedPixels'] = encoded_pixels\n","sub.to_csv('submission.csv', columns=['Image_Label', 'EncodedPixels'], index=False)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"HOlw6Xxh8plt","colab_type":"code","colab":{}},"source":[""],"execution_count":0,"outputs":[]}]}