Plotting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch \n",
    "import time\n",
    "import numpy as np\n",
    "from torchsummary import summary\n",
    "\n",
    "from sde import VPSDE\n",
    "from configs.config import CFGS\n",
    "from models.unet import UNet\n",
    "from sampling import get_sampler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_config = CFGS[\"model\"]\n",
    "model = UNet(**model_config).to(\"cuda\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==========================================================================================\n",
      "Layer (type:depth-idx)                   Output Shape              Param #\n",
      "==========================================================================================\n",
      "├─Sequential: 1-1                        [-1, 256]                 --\n",
      "|    └─Linear: 2-1                       [-1, 256]                 16,640\n",
      "|    └─SiLU: 2-2                         [-1, 256]                 --\n",
      "|    └─Linear: 2-3                       [-1, 256]                 65,792\n",
      "├─ModuleList: 1                          []                        --\n",
      "|    └─EmbedSequential: 2-4              [-1, 64, 64, 64]          --\n",
      "|    |    └─Conv2d: 3-1                  [-1, 64, 64, 64]          3,520\n",
      "|    └─EmbedSequential: 2-5              [-1, 64, 64, 64]          --\n",
      "|    |    └─ResBlock: 3-2                [-1, 64, 64, 64]          107,008\n",
      "|    └─EmbedSequential: 2-6              [-1, 64, 64, 64]          --\n",
      "|    |    └─ResBlock: 3-3                [-1, 64, 64, 64]          107,008\n",
      "|    └─EmbedSequential: 2-7              [-1, 64, 32, 32]          --\n",
      "|    |    └─ResBlock: 3-4                [-1, 64, 32, 32]          107,008\n",
      "|    └─EmbedSequential: 2-8              [-1, 128, 32, 32]         --\n",
      "|    |    └─ResBlock: 3-5                [-1, 128, 32, 32]         295,936\n",
      "|    └─EmbedSequential: 2-9              [-1, 128, 32, 32]         --\n",
      "|    |    └─ResBlock: 3-6                [-1, 128, 32, 32]         361,472\n",
      "|    └─EmbedSequential: 2-10             [-1, 128, 16, 16]         --\n",
      "|    |    └─ResBlock: 3-7                [-1, 128, 16, 16]         361,472\n",
      "|    └─EmbedSequential: 2-11             [-1, 256, 16, 16]         --\n",
      "|    |    └─ResBlock: 3-8                [-1, 256, 16, 16]         1,050,624\n",
      "|    └─EmbedSequential: 2-12             [-1, 256, 16, 16]         --\n",
      "|    |    └─ResBlock: 3-9                [-1, 256, 16, 16]         1,312,768\n",
      "|    └─EmbedSequential: 2-13             [-1, 256, 8, 8]           --\n",
      "|    |    └─ResBlock: 3-10               [-1, 256, 8, 8]           1,312,768\n",
      "|    └─EmbedSequential: 2-14             [-1, 512, 8, 8]           --\n",
      "|    |    └─ResBlock: 3-11               [-1, 512, 8, 8]           3,936,256\n",
      "|    └─EmbedSequential: 2-15             [-1, 512, 8, 8]           --\n",
      "|    |    └─ResBlock: 3-12               [-1, 512, 8, 8]           4,984,832\n",
      "├─EmbedSequential: 1-2                   [-1, 512, 8, 8]           --\n",
      "|    └─ResBlock: 2-16                    [-1, 512, 8, 8]           --\n",
      "|    |    └─Sequential: 3-13             [-1, 512, 8, 8]           2,360,832\n",
      "|    |    └─Sequential: 3-14             [-1, 1024]                263,168\n",
      "|    |    └─Identity: 3-15               [-1, 512, 8, 8]           --\n",
      "|    └─AttentionBlock: 2-17              [-1, 512, 8, 8]           --\n",
      "|    |    └─GroupNorm32: 3-16            [-1, 512, 64]             1,024\n",
      "|    |    └─Conv1d: 3-17                 [-1, 1536, 64]            787,968\n",
      "|    |    └─QKVAttentionLegacy: 3-18     [-1, 512, 64]             --\n",
      "|    |    └─Conv1d: 3-19                 [-1, 512, 64]             262,656\n",
      "|    └─ResBlock: 2-18                    [-1, 512, 8, 8]           --\n",
      "|    |    └─Sequential: 3-20             [-1, 512, 8, 8]           2,360,832\n",
      "|    |    └─Sequential: 3-21             [-1, 1024]                263,168\n",
      "|    |    └─Identity: 3-22               [-1, 512, 8, 8]           --\n",
      "├─ModuleList: 1                          []                        --\n",
      "|    └─EmbedSequential: 2-19             [-1, 512, 8, 8]           --\n",
      "|    |    └─ResBlock: 3-23               [-1, 512, 8, 8]           7,869,952\n",
      "|    └─EmbedSequential: 2-20             [-1, 512, 8, 8]           --\n",
      "|    |    └─ResBlock: 3-24               [-1, 512, 8, 8]           7,869,952\n",
      "|    └─EmbedSequential: 2-21             [-1, 512, 16, 16]         --\n",
      "|    |    └─ResBlock: 3-25               [-1, 512, 8, 8]           6,558,720\n",
      "|    |    └─ResBlock: 3-26               [-1, 512, 16, 16]         4,984,832\n",
      "|    └─EmbedSequential: 2-22             [-1, 256, 16, 16]         --\n",
      "|    |    └─ResBlock: 3-27               [-1, 256, 16, 16]         2,690,304\n",
      "|    └─EmbedSequential: 2-23             [-1, 256, 16, 16]         --\n",
      "|    |    └─ResBlock: 3-28               [-1, 256, 16, 16]         2,034,432\n",
      "|    └─EmbedSequential: 2-24             [-1, 256, 32, 32]         --\n",
      "|    |    └─ResBlock: 3-29               [-1, 256, 16, 16]         1,706,496\n",
      "|    |    └─ResBlock: 3-30               [-1, 256, 32, 32]         1,312,768\n",
      "|    └─EmbedSequential: 2-25             [-1, 128, 32, 32]         --\n",
      "|    |    └─ResBlock: 3-31               [-1, 128, 32, 32]         706,176\n",
      "|    └─EmbedSequential: 2-26             [-1, 128, 32, 32]         --\n",
      "|    |    └─ResBlock: 3-32               [-1, 128, 32, 32]         542,080\n",
      "|    └─EmbedSequential: 2-27             [-1, 128, 64, 64]         --\n",
      "|    |    └─ResBlock: 3-33               [-1, 128, 32, 32]         460,032\n",
      "|    |    └─ResBlock: 3-34               [-1, 128, 64, 64]         361,472\n",
      "|    └─EmbedSequential: 2-28             [-1, 64, 64, 64]          --\n",
      "|    |    └─ResBlock: 3-35               [-1, 64, 64, 64]          193,344\n",
      "|    └─EmbedSequential: 2-29             [-1, 64, 64, 64]          --\n",
      "|    |    └─ResBlock: 3-36               [-1, 64, 64, 64]          152,256\n",
      "|    └─EmbedSequential: 2-30             [-1, 64, 64, 64]          --\n",
      "|    |    └─ResBlock: 3-37               [-1, 64, 64, 64]          152,256\n",
      "├─Sequential: 1-3                        [-1, 3, 64, 64]           --\n",
      "|    └─GroupNorm32: 2-31                 [-1, 64, 64, 64]          128\n",
      "|    └─SiLU: 2-32                        [-1, 64, 64, 64]          --\n",
      "|    └─Conv2d: 2-33                      [-1, 3, 64, 64]           1,731\n",
      "==========================================================================================\n",
      "Total params: 57,919,683\n",
      "Trainable params: 57,919,683\n",
      "Non-trainable params: 0\n",
      "Total mult-adds (G): 1.30\n",
      "==========================================================================================\n",
      "Input size (MB): 0.09\n",
      "Forward/backward pass size (MB): 20.36\n",
      "Params size (MB): 220.95\n",
      "Estimated Total Size (MB): 241.40\n",
      "==========================================================================================\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "==========================================================================================\n",
       "Layer (type:depth-idx)                   Output Shape              Param #\n",
       "==========================================================================================\n",
       "├─Sequential: 1-1                        [-1, 256]                 --\n",
       "|    └─Linear: 2-1                       [-1, 256]                 16,640\n",
       "|    └─SiLU: 2-2                         [-1, 256]                 --\n",
       "|    └─Linear: 2-3                       [-1, 256]                 65,792\n",
       "├─ModuleList: 1                          []                        --\n",
       "|    └─EmbedSequential: 2-4              [-1, 64, 64, 64]          --\n",
       "|    |    └─Conv2d: 3-1                  [-1, 64, 64, 64]          3,520\n",
       "|    └─EmbedSequential: 2-5              [-1, 64, 64, 64]          --\n",
       "|    |    └─ResBlock: 3-2                [-1, 64, 64, 64]          107,008\n",
       "|    └─EmbedSequential: 2-6              [-1, 64, 64, 64]          --\n",
       "|    |    └─ResBlock: 3-3                [-1, 64, 64, 64]          107,008\n",
       "|    └─EmbedSequential: 2-7              [-1, 64, 32, 32]          --\n",
       "|    |    └─ResBlock: 3-4                [-1, 64, 32, 32]          107,008\n",
       "|    └─EmbedSequential: 2-8              [-1, 128, 32, 32]         --\n",
       "|    |    └─ResBlock: 3-5                [-1, 128, 32, 32]         295,936\n",
       "|    └─EmbedSequential: 2-9              [-1, 128, 32, 32]         --\n",
       "|    |    └─ResBlock: 3-6                [-1, 128, 32, 32]         361,472\n",
       "|    └─EmbedSequential: 2-10             [-1, 128, 16, 16]         --\n",
       "|    |    └─ResBlock: 3-7                [-1, 128, 16, 16]         361,472\n",
       "|    └─EmbedSequential: 2-11             [-1, 256, 16, 16]         --\n",
       "|    |    └─ResBlock: 3-8                [-1, 256, 16, 16]         1,050,624\n",
       "|    └─EmbedSequential: 2-12             [-1, 256, 16, 16]         --\n",
       "|    |    └─ResBlock: 3-9                [-1, 256, 16, 16]         1,312,768\n",
       "|    └─EmbedSequential: 2-13             [-1, 256, 8, 8]           --\n",
       "|    |    └─ResBlock: 3-10               [-1, 256, 8, 8]           1,312,768\n",
       "|    └─EmbedSequential: 2-14             [-1, 512, 8, 8]           --\n",
       "|    |    └─ResBlock: 3-11               [-1, 512, 8, 8]           3,936,256\n",
       "|    └─EmbedSequential: 2-15             [-1, 512, 8, 8]           --\n",
       "|    |    └─ResBlock: 3-12               [-1, 512, 8, 8]           4,984,832\n",
       "├─EmbedSequential: 1-2                   [-1, 512, 8, 8]           --\n",
       "|    └─ResBlock: 2-16                    [-1, 512, 8, 8]           --\n",
       "|    |    └─Sequential: 3-13             [-1, 512, 8, 8]           2,360,832\n",
       "|    |    └─Sequential: 3-14             [-1, 1024]                263,168\n",
       "|    |    └─Identity: 3-15               [-1, 512, 8, 8]           --\n",
       "|    └─AttentionBlock: 2-17              [-1, 512, 8, 8]           --\n",
       "|    |    └─GroupNorm32: 3-16            [-1, 512, 64]             1,024\n",
       "|    |    └─Conv1d: 3-17                 [-1, 1536, 64]            787,968\n",
       "|    |    └─QKVAttentionLegacy: 3-18     [-1, 512, 64]             --\n",
       "|    |    └─Conv1d: 3-19                 [-1, 512, 64]             262,656\n",
       "|    └─ResBlock: 2-18                    [-1, 512, 8, 8]           --\n",
       "|    |    └─Sequential: 3-20             [-1, 512, 8, 8]           2,360,832\n",
       "|    |    └─Sequential: 3-21             [-1, 1024]                263,168\n",
       "|    |    └─Identity: 3-22               [-1, 512, 8, 8]           --\n",
       "├─ModuleList: 1                          []                        --\n",
       "|    └─EmbedSequential: 2-19             [-1, 512, 8, 8]           --\n",
       "|    |    └─ResBlock: 3-23               [-1, 512, 8, 8]           7,869,952\n",
       "|    └─EmbedSequential: 2-20             [-1, 512, 8, 8]           --\n",
       "|    |    └─ResBlock: 3-24               [-1, 512, 8, 8]           7,869,952\n",
       "|    └─EmbedSequential: 2-21             [-1, 512, 16, 16]         --\n",
       "|    |    └─ResBlock: 3-25               [-1, 512, 8, 8]           6,558,720\n",
       "|    |    └─ResBlock: 3-26               [-1, 512, 16, 16]         4,984,832\n",
       "|    └─EmbedSequential: 2-22             [-1, 256, 16, 16]         --\n",
       "|    |    └─ResBlock: 3-27               [-1, 256, 16, 16]         2,690,304\n",
       "|    └─EmbedSequential: 2-23             [-1, 256, 16, 16]         --\n",
       "|    |    └─ResBlock: 3-28               [-1, 256, 16, 16]         2,034,432\n",
       "|    └─EmbedSequential: 2-24             [-1, 256, 32, 32]         --\n",
       "|    |    └─ResBlock: 3-29               [-1, 256, 16, 16]         1,706,496\n",
       "|    |    └─ResBlock: 3-30               [-1, 256, 32, 32]         1,312,768\n",
       "|    └─EmbedSequential: 2-25             [-1, 128, 32, 32]         --\n",
       "|    |    └─ResBlock: 3-31               [-1, 128, 32, 32]         706,176\n",
       "|    └─EmbedSequential: 2-26             [-1, 128, 32, 32]         --\n",
       "|    |    └─ResBlock: 3-32               [-1, 128, 32, 32]         542,080\n",
       "|    └─EmbedSequential: 2-27             [-1, 128, 64, 64]         --\n",
       "|    |    └─ResBlock: 3-33               [-1, 128, 32, 32]         460,032\n",
       "|    |    └─ResBlock: 3-34               [-1, 128, 64, 64]         361,472\n",
       "|    └─EmbedSequential: 2-28             [-1, 64, 64, 64]          --\n",
       "|    |    └─ResBlock: 3-35               [-1, 64, 64, 64]          193,344\n",
       "|    └─EmbedSequential: 2-29             [-1, 64, 64, 64]          --\n",
       "|    |    └─ResBlock: 3-36               [-1, 64, 64, 64]          152,256\n",
       "|    └─EmbedSequential: 2-30             [-1, 64, 64, 64]          --\n",
       "|    |    └─ResBlock: 3-37               [-1, 64, 64, 64]          152,256\n",
       "├─Sequential: 1-3                        [-1, 3, 64, 64]           --\n",
       "|    └─GroupNorm32: 2-31                 [-1, 64, 64, 64]          128\n",
       "|    └─SiLU: 2-32                        [-1, 64, 64, 64]          --\n",
       "|    └─Conv2d: 2-33                      [-1, 3, 64, 64]           1,731\n",
       "==========================================================================================\n",
       "Total params: 57,919,683\n",
       "Trainable params: 57,919,683\n",
       "Non-trainable params: 0\n",
       "Total mult-adds (G): 1.30\n",
       "==========================================================================================\n",
       "Input size (MB): 0.09\n",
       "Forward/backward pass size (MB): 20.36\n",
       "Params size (MB): 220.95\n",
       "Estimated Total Size (MB): 241.40\n",
       "=========================================================================================="
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.randn((1,6,64,64), device=\"cuda\")\n",
    "t = torch.rand(x.shape[0], device=\"cuda\")\n",
    "\n",
    "summary(model, x, t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hardware Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__CUDA Device Name: NVIDIA GeForce RTX 2080 Ti\n",
      "__CUDA Device Total Memory [GB]: 11.554586624\n"
     ]
    }
   ],
   "source": [
    "print('__CUDA Device Name:',torch.cuda.get_device_name(0))\n",
    "print('__CUDA Device Total Memory [GB]:',torch.cuda.get_device_properties(0).total_memory/1e9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sampling time for different image sizes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the sde\n",
    "vpsde = VPSDE(N=10)\n",
    "\n",
    "def sampler_time(model, img_size, total_steps):\n",
    "\n",
    "    condition = torch.randn(1,3,img_size, img_size)\n",
    "    condition = condition.to(\"cuda\")\n",
    "    sampling_fn = get_sampler(sde=vpsde, shape=condition.shape)\n",
    "    times = list()\n",
    "    \n",
    "    for steps in total_steps:\n",
    "\n",
    "        start = time.time()       \n",
    "        with torch.no_grad():\n",
    "            sample = sampling_fn(model, condition, num_steps=steps)\n",
    "        end = time.time()\n",
    "        elapsed = end - start\n",
    "        times.append(elapsed)\n",
    "        \n",
    "    return times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampling_times = dict()\n",
    "\n",
    "total_steps = [int(i) for i in list(np.linspace(10, 2000, 8))] \n",
    "total_steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Config for 32 by 32 Image\n",
    "CFGS[\"model\"][\"image_size\"] = 32\n",
    "model_config = CFGS[\"model\"]\n",
    "model_32 = UNet(**model_config).to(\"cuda\")\n",
    "\n",
    "times_32 = sampler_time(model_32, img_size=32, total_steps=total_steps)\n",
    "sampling_times[\"32\"] = times_32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampling_times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CFGS[\"model\"][\"image_size\"] = 64\n",
    "model_config = CFGS[\"model\"]\n",
    "model_64 = UNet(**model_config).to(\"cuda\")\n",
    "\n",
    "times_64 = sampler_time(model_64, img_size=64, total_steps=total_steps)\n",
    "sampling_times[\"64\"] = times_64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampling_times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CFGS[\"model\"][\"image_size\"] = 128\n",
    "model_config = CFGS[\"model\"]\n",
    "model_128 = UNet(**model_config).to(\"cuda\")\n",
    "\n",
    "times_128 = sampler_time(model_128, img_size=128, total_steps=total_steps)\n",
    "sampling_times[\"128\"] = times_128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CFGS[\"model\"][\"image_size\"] = 256\n",
    "model_config = CFGS[\"model\"]\n",
    "model_256 = UNet(**model_config).to(\"cuda\")\n",
    "\n",
    "times_256 = sampler_time(model_256, img_size=256, total_steps=total_steps)\n",
    "sampling_times[\"256\"] = times_256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampling_times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(7, 5))\n",
    "\n",
    "ax.plot(total_steps, sampling_times[\"32\"], marker='o', color='c', linewidth=2, label='$32~by~32$')\n",
    "ax.plot(total_steps, sampling_times[\"64\"], marker='o', color='y', linewidth=2,  label='$64~by~64$')\n",
    "ax.plot(total_steps, sampling_times[\"128\"], marker='o', linewidth=2,  label='$128~by~128$')\n",
    "ax.plot(total_steps, sampling_times[\"256\"], marker='o', linewidth=2,  label='$256~by~256$')\n",
    "ax.set_title(label=\"Sampling time using EM Sampler\", fontdict={'fontsize':14})\n",
    "plt.xticks(total_steps)\n",
    "ax.set_xlabel('N (Steps)')\n",
    "ax.set_ylabel('Time (in sec)')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Memory consuption of the EM sampler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the sde\n",
    "vpsde = VPSDE(N=10)\n",
    "\n",
    "def sampler_mem(model, img_size, total_steps, device=0, grad_track=True):\n",
    "    \n",
    "    model.to(device)\n",
    "    condition = torch.randn(1,3,img_size, img_size)\n",
    "    condition = condition.to(device)\n",
    "    sampling_fn = get_sampler(sde=vpsde, shape=condition.shape)\n",
    "    \n",
    "    if grad_track:\n",
    "        start = torch.cuda.memory_allocated(device)         \n",
    "        sample = sampling_fn(model, condition, num_steps=total_steps)\n",
    "        end = torch.cuda.memory_allocated(device)\n",
    "    else: \n",
    "        with torch.no_grad():\n",
    "            start = torch.cuda.memory_allocated(device)         \n",
    "            sample = sampling_fn(model, condition, num_steps=total_steps)\n",
    "            end = torch.cuda.memory_allocated(device)\n",
    "        \n",
    "    consumed = (end - start)/1e6\n",
    "    print(f\"Memory Consumption of sampler for img size:{img_size} and N steps:{total_steps} = {consumed}\")\n",
    "        \n",
    "    return consumed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory Consumption of sampler for img size:32 and N steps:10 = 403.887616\n",
      "Memory Consumption of sampler for img size:32 and N steps:50 = 2019.390976\n",
      "Memory Consumption of sampler for img size:32 and N steps:100 = 0.012288\n",
      "Memory Consumption of sampler for img size:32 and N steps:1000 = 0.012288\n",
      "Memory Consumption of sampler for img size:32 and N steps:1500 = 0.012288\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.012288"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CFGS[\"model\"][\"image_size\"] = 32\n",
    "model_config = CFGS[\"model\"]\n",
    "model_32 = UNet(**model_config)\n",
    "\n",
    "sampler_mem(model_32, img_size=32, total_steps=10)\n",
    "sampler_mem(model_32, img_size=32, total_steps=50)\n",
    "sampler_mem(model_32, img_size=32, total_steps=100, grad_track=False)\n",
    "sampler_mem(model_32, img_size=32, total_steps=1000, grad_track=False)\n",
    "sampler_mem(model_32, img_size=32, total_steps=1500, grad_track=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory Consumption of sampler for img size:64 and N steps:10 = 1610.97472\n",
      "Memory Consumption of sampler for img size:64 and N steps:50 = 8060.970496\n",
      "Memory Consumption of sampler for img size:64 and N steps:100 = 0.049152\n",
      "Memory Consumption of sampler for img size:64 and N steps:500 = 0.049152\n",
      "Memory Consumption of sampler for img size:64 and N steps:1500 = 0.049152\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.049152"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CFGS[\"model\"][\"image_size\"] = 64\n",
    "model_config = CFGS[\"model\"]\n",
    "model_64 = UNet(**model_config)\n",
    "\n",
    "sampler_mem(model_64, img_size=64, total_steps=10)\n",
    "sampler_mem(model_64, img_size=64, total_steps=50)\n",
    "sampler_mem(model_64, img_size=64, total_steps=100, grad_track=False)\n",
    "sampler_mem(model_64, img_size=64, total_steps=500, grad_track=False)\n",
    "sampler_mem(model_64, img_size=64, total_steps=1500, grad_track=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory Consumption of sampler for img size:128 and N steps:10 = 6465.668608\n",
      "Memory Consumption of sampler for img size:128 and N steps:10 = 0.196608\n",
      "Memory Consumption of sampler for img size:128 and N steps:50 = 0.196608\n",
      "Memory Consumption of sampler for img size:128 and N steps:100 = 0.196608\n",
      "Memory Consumption of sampler for img size:128 and N steps:500 = 0.196608\n",
      "Memory Consumption of sampler for img size:128 and N steps:1500 = 0.196608\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.196608"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CFGS[\"model\"][\"image_size\"] = 128\n",
    "model_config = CFGS[\"model\"]\n",
    "model_128 = UNet(**model_config)\n",
    "\n",
    "sampler_mem(model_128, img_size=128, total_steps=10)\n",
    "#sampler_mem(model_128, img_size=128, total_steps=50)\n",
    "sampler_mem(model_128, img_size=128, total_steps=10, grad_track=False)\n",
    "sampler_mem(model_128, img_size=128, total_steps=50, grad_track=False)\n",
    "sampler_mem(model_128, img_size=128, total_steps=100, grad_track=False)\n",
    "sampler_mem(model_128, img_size=128, total_steps=500, grad_track=False)\n",
    "sampler_mem(model_128, img_size=128, total_steps=1500, grad_track=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEWCAYAAABsY4yMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAiUElEQVR4nO3deZxcVZ338c+XLCRAIBA6M5AGOgGFsEYJDKggIoEYBURkBlCRzYCDgiM+guCD4sw8oyMgzoiOARFUDAMoIAgRUCKgsiSQBQLIDolAFvaQAAm/549zOqk03dXV3XW7um++79erXt13Pb+6detX5567HEUEZmZWPus0OgAzMyuGE7yZWUk5wZuZlZQTvJlZSTnBm5mVlBO8mVlJOcFb6Uh6QNI+Ba37BEnnF7HuKmW2SApJA/PwjZI+25sxFKXte2szbUtJr0ka0IjYOiPpXEmfb3Qc1TjBV5D0pKQ3JW3aZvx9eSdsaVBo3SZpQ0nnS3o6f1key8Obdr503yfpEkn/VjkuInaIiOkFlDUY+Drw3cpxks6S9LCkpZIW5AS8f73LbxURH4mIS3u6HklHS7qjyvQH8j7zmqSVkpZXDJ/R0/I7ExFPR8QGEbGyiPVLep+kP0h6VdLLkq6TtH0XVnEOcEbeL/okJ/h3egI4onVA0k7Aeo0LZ7X2ajmdzD8Y+D2wAzAR2BDYE1gC7F73AMvvYOChiFhQMe6qPP4oYGNgNPB94KPtraCrn2Ej5R/KDSJiA+B24AutwxHx/1rn60/vqZWkPYGbgGuBzUmf22zgT5LG1LKOiHgWeAg4qKg4eywi/Mov4ElSDe2einHnAGcCAbTkcevm8U8DzwP/AwzN0/YB5gNfBRYCzwIfByYBfwVeAM6oWP+6wPnA3/LrfGDdNus6DXgO+DlwP3BgxfKDgMXAe9p5P8fn+Dao8p7HAtOBl4AHgIMqpl0CXAD8FngVuAvYOk8T8L38Hl8B5gI75mnTgeMr1nM0cEfFcAD/DDyS1/uvwNbAn/O6rgAGt9kGZ+T3+STwqTxtMvAW8CbwGnBdxee4Xxe276kVn9UxVbbVxcDXK4b3A5YBzTXsV6cBc4A3gIHA6cBj+f3PAw6pmH8Aaf9aDDwOnJS32cAOtu+xwIPAi8DvgK3abOsT87Z+KX+eyp/7cmBl3nYvdfIeVpUJtOT1Hkf6DtyWx19J2k9fBm4DdqhYfihwLvBUnn5HHte6rtb3dmjeXju2M206aV/5U95uNwGbVpRxVF7/EuD/Vu4H7byf24EftjP+RuBnte4fpNzw00bnrg4/t0YH0JderTsE8HD+AgzIH/BWrJngvwf8BtgEGAZcB/xHxU6xAjiLlHw/BywCfpnn3YGUFEbn+b8F3AmMBJpISe5f26zrO6RENZT0w/G/FTEfDMzt4P1cDlxa5f0OAh4lJc/BwL75i7Ntnn4Jq2v7A4HLgMvztAOAmcBwVieMzfK06XSe4K8lHVHsQEp6vwfGABuREt5n22yD8/I2+CCwtE2M/9be59iF7futvC0mAa8DG3ewve4BDqsY/jYwvcb9ahawBasrAoeRao7rAP+U31Pr9juRVDPcgrSP3UoHCT5//o/m7T+QVEH5c5ttfX3+nLYk7YsT2/tcOnkPlWW25PX+DFi/4j0dS9rHW39UZ1Usf0FexyjS9+p9eb7WdQ0EjsnvZZs25VS+78eAd5O+C9OBb+dp25N+qD5A2pfPIf34vyPBk47IVwIfamfaMcCzte4fwCeAexuduzr83BodQDsb+GLSr+X9Ncx7IqnmOItUI9i+h2U/SUrwXwf+g9SscXPe+SLvcCJ9GbeuWG5P4ImKnWIZMCAPD8vL/kPF/DOBj+f/HwMmVUw7AHiyYl1vAkMqpm9OSsIb5uGrgK928H5ubv0CdDB9L1KNa52KcVOBb+b/LwEuqpg2idREAenH4K/AHpXL52nT6TzBv7/N9jitYvhc4PyKbbACWL9i+hXA/62IsVqC72z7LiMnkDxuIbBHB9vrEXJyzMMXkX/w8vAmpFryy8DyNvEc28m+Nws4OP//B+DEimn703GCvxE4rmLedUhJaKuKbf2BNtvu9PY+l07iqyyzJa93TJX5h+d5NsoxLQN2aWe+1nV9hfTD3tzOtMr3XXkE9c/AtPz/WcDUimnrkb477SX45rze7dqZNhF4q9b9A5gAPF7LNmzEqy+2wV9C2si1+GVE7BQR44D/JNXy6uHnwJGkL8DP2kxrIu08MyW9JOklYFoe32pJrD4xtCz/fb5i+jJgg/z/5qTDylZP5XGtFkXE8taBiPgb6RD1UEnDgY+QatbtWQJs1sG01rKfiYi325Q/qmL4uYr/X2+NOyL+APyAVDNbKGmKpA2rlNVW2+3R0fYBeDEilraJsXIbVdPZ9l0SESsqhle9x3a8SPrBXrUsFds3Il6IiOHArqTaaaVnKgckHSVpVsU+tCPQeuJ78zbzV8bf1lbA9yvW8wKpEtLpZ1gHq2KUNEDSt/NJ/FdIP2qQ3tOmwBDSj21H/g9wQUTM76TMjt7LGtssIl4nfT7teRF4m/a/G5uRmsZadbZ/DCP9qPdJfS7BR8RtpJ10FUlbS5omaaak2yVtl+d9pWK29Um/yvWI4SnSydZJwK/bTF5MSkA7RMTw/Noo0omo7vgb6Uvaass8blU47SxzKfBp0mH+X2LNk36VbgEOkLR+lbK3kFS5H2wJdLS+NUTEf0XErqTD43eTvqSQjnAqT0z/fS3rq2LjNu+hcht19pl3tn27Yg7pfbb6PbCbpOYall0Vp6StgAuBLwAj8o/C/aTEDKmtd4s2MXfkGeCEin1xeEQMjYg/dyWmbqpc/khSc9F+pFp7Sx4v0ndmOek8S0f2B74u6dBuxvIsqWaeCpWGAiPaDTpVFv5C+v609Y+kz7VWY0knZ/ukPpfgOzAF+GJOJl8Bftg6QdJJkh4j1eBPrmOZxwH7tqk5kmu7FwLfkzQyxzBK0gHdLGcqacduypcungX8opNlrgHeC5zCO48wKv2clAB+JWk7SetIGiHpDEmTSCdNXwe+KmlQvnb8QFLbfVWSdpP0D5IGkRL6clKtCFJzwyckrSdpG9K27Kmz8yWJewEfI53Qg1Tzr3bVQ3e2b0duIJ0DACAibiK1j1+Tt8XgvD326GQ9rZWRRQCSjiHV4FtdAZwsqVnSxqQTsh35H+BrknbI69pIUnuJqz3PA811usxvGOlcyhLSj/uqq2zyd+Zi4DxJm+fa/p6SKo9yHiAduV8gqTtXpVwFHJgvfRwMfJPVP5jtOR34rKSTJQ2TtHG+3HZP4OwulPtBUjNZn9TnE7ykDUgnZK6UNAv4MWseFl8QEVuTrlL4er3KjYjHImJGB5NPI50MujMfjt4CbNvNov4NmEGqHc4F7s3jqsW2DPgV6dKutkcYlfO9QapRPURqj38FuJt0yHxXRLxJSugfIdWyfggcFREP1RD3hqQfuhdZfeVC6/Xh3yO1fz5POtroqAmpVs/lcv6W13ViRYw/AbbPTRTXtLNsl7dvFdcB20mqbOI5hHQS8xekQ/UngE+R2vrbFRHzSOcZ/kLaRjuRmt1aXUi6GmZ2jrfaZ3w16ST85XlfvJ/0edbiD6TE+pykxZ3N3ImfkfaDBaS29DvbTP8KafvfQzpC/w5t8k9EzCb9eF8oqdb30LrsA8AXSZWTZ0knXBeSfnTam/8O0mf0iTz/U8B7SOcrHqmlTEmbkY5er+lKrL1J+URBn5JvKLo+InbM7boPR0S1tmRyM8OLEbFRb8TYaJLOAt4dEZ9udCxFykcVv4iIWppBCidpMulk/pcaHYt1LFcMXwLeFRFPFFTGucBjEfHDTmdukD5/g0JEvCLpCUmHRcSVkgTsHBGzJb2r4tf2o6SrHEpP0iakZo/PNDqWtU1ETGl0DNY+SQeS2s9FukxyLqtP9tZdRJxa1Lrrpc810UiaSjp03VbSfEnHkQ55j5M0m3RIeXCe/Qv5dupZwJeBzzYi5t4k6XOkdvUb8wlpM0sOZvUNbe8CDo++2ETRi/pkE42ZmfVcn6vBm5lZffSpNvhNN900WlpaGh2GmVm/MXPmzMUR0dTetD6V4FtaWpgxo6MrE83MrC1JHd7p7CYaM7OScoI3MyspJ3gzs5LqU23wZmat3nrrLebPn8/y5cs7n3ktMGTIEJqbmxk0aFDNyzjBm1mfNH/+fIYNG0ZLSwvpBva1V0SwZMkS5s+fz+jRo2tezk00ZtYnLV++nBEjRqz1yR1AEiNGjOjy0YwTvJn1WU7uq3VnWzjBm5mVVKEJXtK/5IeB3S9pqqQhRZZnZmarFXaSVdIoUg9L20fEMklXAIeT+lw161cW/ujMwssY+fl/L7wMW7sUfRXNQGCopLdI3Xh1ty9MM1vLjb7uys5n6oInDqytZ8Mf//jHnHjiicybN4+xY8cCMHbsWG644YYuXdECcOyxx3L99dczcuRI7r///jWmTZs2jVNOOYWVK1dy/PHHc/rp1XpqrE1hTTS5I+hzgKdJXWK9nPuwXIOkyZJmSJqxaNGiosIxM+uWuXPnMm7cOH77298C6eqe559/nu48GPHoo49m2rRp7xi/cuVKTjrpJG688UbmzZvH1KlTmTdvXk9DLy7B586CDyb1G7o5sL6kd3QvFxFTImJ8RIxvamr3gWhmZg0zZ84cTjvttFUJft68eWy33Xbduqpl7733ZpNNNnnH+LvvvpttttmGMWPGMHjwYA4//HCuvfbaHsde5EnW/YAnImJRRLxF6jj4fQWWZ2ZWd/PmzePggw9m4cKFvPzyy8ydO5edd955jXn22msvxo0b947XLbfcUlMZCxYsYIsttlg13NzczIIFC3oce5Ft8E8De0haD1gGfJjUu72ZWb/wzDPPMGLECIYOHcqECRP43e9+x5w5c9hpp53WmO/2229vUITVFZbgI+IuSVcB9wIrgPsAd1hsZv3G3LlzVyXzSZMmcdlll/Hss8/y8Y9/fI359tprL1599dV3LH/OOeew3377dVrOqFGjeOaZZ1YNz58/n1GjRvUseAq+iiYivgF8o8gyzMyKUllb/+AHP8gJJ5zAsmXL6l6D32233XjkkUd44oknGDVqFJdffjm//OUve7RO8MPGzKyfqPWyxnqaO3cuhx56KADrrrsuO++8M/fddx/Dhw/v1vqOOOIIpk+fzuLFi2lububss8/muOOOY+DAgfzgBz/ggAMOYOXKlRx77LHssMMOPY7fCd7MrAOXXXbZGsM9vbJl6tSpHU6bNGkSkyZN6tH62/KzaMzMSsoJ3syspJzgzcxKygnezKyknODNzErKCd7MrKSc4M3MSsoJ3syspHyjk5n1C/XuVavWHrTq2eFHS0sLw4YNY8CAAQwcOJAZM1Y/f7GIDj+c4M3Mqqjs8GPs2LE96vAD4NZbb2XTTTddY1xrhx8333wzzc3N7Lbbbhx00EFsv/32PYrdTTRmZlXUs8OPjvTHDj/MzPq9enb4IYn999+fXXfdlSlTVj89vT92+GFm1q/Vu8OPO+64g1GjRrFw4UImTJjAdtttx957711E6IATvJlZh+rd4UdrJx4jR47kkEMO4e6772bvvffufx1+SNoW+N+KUWOAsyLi/KLKNDOrp3p2+LF06VLefvtthg0bxtKlS7nppps466yzgH7Y4UdEPAyMA5A0AFgAXF1UeWZWbrVe1lhP9ezw4/nnn+eQQw4BYMWKFRx55JFMnDgRoN93+PFh4LGIeKqXyjMz67F6dvgxZswYZs+e3eH0/tzhx+FAu12ZSJosaYakGYsWLeqlcMzMyq/wBC9pMHAQcGV70yNiSkSMj4jxTU1NRYdjZrbW6I0a/EeAeyPi+V4oy8xKJCIaHUKf0Z1t0RsJ/gg6aJ4xM+vIkCFDWLJkiZM8KbkvWbKEIUOGdGm5Qk+ySlofmACcUGQ5ZlY+zc3NzJ8/H5+bS4YMGUJzc3OXlik0wUfEUmBEkWWYWTkNGjSoy09rtDX5WTRmZiXlBG9mVlJO8GZmJeUEb2ZWUk7wZmYl5QRvZlZSTvBmZiXlBG9mVlLu0cnM+qXR17X7/MK6e+LAw3qlnCK4Bm9mVlJO8GZmJeUEb2ZWUk7wZmYl5QRvZlZSTvBmZiXlBG9mVlJO8GZmJVVogpc0XNJVkh6S9KCkPYssz8zMViv6TtbvA9Mi4pOSBgPrFVyemZllhSV4SRsBewNHA0TEm8CbRZVnZmZrKrKJZjSwCPippPskXSRp/bYzSZosaYakGe493cysfopM8AOB9wI/ioj3AEuB09vOFBFTImJ8RIxvamoqMBwzs7VLkQl+PjA/Iu7Kw1eREr6ZmfWCwhJ8RDwHPCNp2zzqw8C8osozM7M1FX0VzReBy/IVNI8DxxRcnpmZZYUm+IiYBYwvsgwzM2uf72Q1MyspJ3gzs5JygjczK6ma2uAlbQxsDiwDnoyItwuNyszMeqzDBJ8fNXAScAQwmHRX6hDg7yTdCfwwIm7tlSjNzKzLqtXgrwJ+BuwVES9VTpC0K/AZSWMi4icFxmdmZt3UYYKPiAlVps0EZhYSkZmZ1UXVk6ySBkpS/n8LSZ+U9J7eCc3MzHqiwwQv6XPAQuCp/P/vgU8Cl0s6rZfiMzOzbqrWBv8lYGtgGPAgsFVELJa0HnAP8J3iwzMzs+6qluDfjIgXgRclPRoRiwEi4nVJ7rjDzKyPq5bgh+b29nWAwfl/5deQ3gjOzMy6r1qCfw44r53/W4fNzKwPq3aZ5D69GIeZmdVZtTtZP1FtwYj4df3DMTOzeunsTtZZ+QWp7b1VAE7wZmZ9WLUE/wngcGBn4FpgakQ82pWVS3oSeBVYCayICHf+YWbWS6q1wV8DXCNpfeBg4FxJI4AzI+KPXSjjQ62XWJqZWe+p5Xnwy4GXgVeADfAlkmZm/UK1k6z7kppodgduAb4fETO6uP4AbpIUwI8jYkq3IzUzsy6p1gZ/CzAHuANYFzhK0lGtEyPi5BrW/4GIWCBpJHCzpIci4rbKGSRNBiYDbLnlll2N38zMOlAtwR/T05VHxIL8d6Gkq0lHA7e1mWcKMAVg/Pjx0dMyzcwsqXaS9dKerDifnF0nIl7N/+8PfKsn6zQzs9rV1CdrN/0dcHV+nPxA4JcRMa3A8szMrEJhCT4iHgd2KWr9ZmZWXS2XSZqZWT/UaQ1eUhPwOaClcv6IOLa4sMzMrKdqaaK5FriddNnkymLDMTOzeqklwa8XEe6D1cysn6mlDf56SZMKj8TMzOqqlgR/CinJL5P0iqRXJb1SdGBmZtYznTbRRMSw3gjEzMzqq9rDxraLiIckvbe96RFxb3FhmZlZT1WrwX+Z9BCwc9uZFsC+hURkZmZ1Ue1ZNJPz3w/1XjhmZlYvHZ5klfSBagtK2lDSjvUPyczM6qFaE82hkv4TmAbMBBaRenPaBvgQsBVwauERmplZt1RrovkXSZsAhwKHAZsBy4AHSb0z3dE7IZqZWXdUvUwyIl6QdHFEXNhbAZmZWX3UcqPTI5K+K2ls4dGYmVnd1JLgdwH+CvxE0p2SJkvasOC4zMyshzpN8BHxakRcGBHvA04DvgE8K+lSSdsUHqGZmXVLpwle0gBJB+VOs88n3fg0BrgOuKHG5e+TdH1PgzUzs9rV8rjgR4Bbge9GxJ8rxl8lae8alj+FdOWNm3XMzHpR1Rq8pAHAJRFxXJvkDkBEnNzJ8s3AR4GLehSlmZl1WdUEHxErgY/1YP3nA18F3u5ohnzSdoakGYsWLepBUWZmVqmWq2j+JOkHkvaS9N7WV2cLSfoYsDAiZlabLyKmRMT4iBjf1NRUa9xmZtaJWtrgx+W/36oYV8vTJN8PHJR7gxoCbCjpFxHx6S5HaWZmXVZLhx/deppkRHwN+BqApH2Arzi5m5n1nlouk9xI0nmt7eSSzpW0UW8EZ2Zm3VdLG/zFwKvAP+bXK8BPu1JIREyPiJ6crDUzsy6qpQ1+64g4tGL4bEmzCorHzMzqpJYa/LLKzj8kvZ/02GAzM+vDaqnBfx64NLe7C3gBOLrIoMzMrOdquYpmFrBL6xMkI+KVooMyM7Oe6zTBSxoOHAW0AAMlAZ0/psDMzBqrliaaG4A7gblUeeSAmZn1LbUk+CER8eXCIzEzs7qq5Sqan0v6nKTNJG3S+io8MjMz65FaavBvAt8FziQ9g4b8d0xRQZmZWc/VkuBPBbaJiMVFB2NmZvVTSxPNo8DrRQdiZmb1VUsNfikwS9KtwButI32ZpJlZ31ZLgr8mv8zMrB+p5U7WSyUNBbaMiId7ISYzM6uDWp4HfyAwC5iWh8dJ+k3BcZmZWQ/VcpL1m8DuwEuw6tk0vkTSzKyPqyXBvxURL7cZ50cWmJn1cbUk+AckHQkMkPQuSf8N/LmzhSQNkXS3pNmSHpB0do+jNTOzmtWS4L8I7EC6RHIqqcu+L9Ww3BvAvhGxCzAOmChpj+6FaWZmXVXLVTSvkx5TcGZXVhwRAbyWBwflV3S8hJmZ1VOHCb6zK2Ui4qDOVi5pADAT2Aa4ICLuameeycBkgC233LKzVZqZWY2q1eD3BJ4hNcvcRequr0siYiUwLncacrWkHSPi/jbzTAGmAIwfP941fDOzOqnWBv/3wBnAjsD3gQnA4oj4Y0T8sSuFRMRLwK3AxG7GaWZmXdRhgo+IlRExLSI+C+xBeujYdElfqGXFkppyzZ18J+wE4KGeh2xmZrWoepJV0rrAR4EjSH2y/hdwdY3r3gy4NLfDrwNcERHXdz9UMzPrimonWX9Gap65ATi7bdt5ZyJiDvCenoVnZmbdVa0G/2nSo4JPAU6WVp1jFekqyA0Ljs3MzHqgwwQfEbXcBGVmZn2Uk7iZWUk5wZuZlZQTvJlZSTnBm5mVlBO8mVlJOcGbmZWUE7yZWUk5wZuZlZQTvJlZSTnBm5mVlBO8mVlJOcGbmZWUE7yZWUk5wZuZlVRhCV7SFpJulTRP0gOSTimqLDMze6eqXfb10Arg1Ii4V9IwYKakmyNiXoFlmplZVlgNPiKejYh78/+vAg8Co4oqz8zM1lRkDX4VSS2k/lnv6o3yirLwR2f2SjkjP//vvVKOmZVb4SdZJW0A/Ar4UkS80s70yZJmSJqxaNGiosMxM1trFJrgJQ0iJffLIuLX7c0TEVMiYnxEjG9qaioyHDOztUqRV9EI+AnwYEScV1Q5ZmbWviJr8O8HPgPsK2lWfk0qsDwzM6tQ2EnWiLgDUFHrNzOz6nwnq5lZSTnBm5mVlBO8mVlJOcGbmZVUr9zJalak0dddWXgZ/foWbFtruQZvZlZSTvBmZiXlBG9mVlJO8GZmJeUEb2ZWUk7wZmYl5QRvZlZSTvBmZiXlBG9mVlJO8GZmJeUEb2ZWUk7wZmYlVWSfrBdLWijp/qLKMDOzjhVZg78EmFjg+s3MrIrCEnxE3Aa8UNT6zcysuoa3wUuaLGmGpBmLFi1qdDhmZqXR8AQfEVMiYnxEjG9qamp0OGZmpdHwBG9mZsVwgjczK6kiL5OcCvwF2FbSfEnHFVWWmZm9U2GdbkfEEUWt28zMOucmGjOzknKCNzMrKSd4M7OScoI3MyspJ3gzs5JygjczKykneDOzknKCNzMrKSd4M7OSKuxO1t42+rorCy/jrsJLMDOrH9fgzcxKygnezKyknODNzErKCd7MrKSc4M3MSsoJ3syspJzgzcxKqtAEL2mipIclPSrp9CLLMjOzNRXZJ+sA4ALgI8D2wBGSti+qPDMzW1ORNfjdgUcj4vGIeBO4HDi4wPLMzKyCIqKYFUufBCZGxPF5+DPAP0TEF9rMNxmYnAe3BR4uJKD62BRY3OggrCH82a+9+vpnv1VENLU3oeHPoomIKcCURsdRC0kzImJ8o+Ow3ufPfu3Vnz/7IptoFgBbVAw353FmZtYLikzw9wDvkjRa0mDgcOA3BZZnZmYVCmuiiYgVkr4A/A4YAFwcEQ8UVV4v6RdNSVYIf/Zrr3772Rd2ktXMzBrLd7KamZWUE7yZWUk5wXdA0hBJd0uaLekBSWfn8Zflxy/cL+liSYMaHavVl6Thkq6S9JCkByXtWTHtVEkhadNGxmj1kb/DCyXdXzHuu/mznyPpaknD8/hBki6VNDfvF19rWOA1coLv2BvAvhGxCzAOmChpD+AyYDtgJ2AocHzDIrSifB+YFhHbAbsADwJI2gLYH3i6gbFZfV0CTGwz7mZgx4jYGfgr0JrIDwPWjYidgF2BEyS19FKc3eIE34FIXsuDg/IrIuKGPC2Au0nX91tJSNoI2Bv4CUBEvBkRL+XJ3wO+CvjKhJKIiNuAF9qMuykiVuTBO1n9HQ9gfUkDSZW7N4FXeivW7nCCr0LSAEmzgIXAzRFxV8W0QcBngGkNCs+KMRpYBPxU0n2SLpK0vqSDgQURMbvB8VnvOha4Mf9/FbAUeJZ0FHdORLzQ0YJ9gRN8FRGxMiLGkX7Bd5e0Y8XkHwK3RcTtDQnOijIQeC/wo4h4D+kL/U3gDOCsBsZlvUzSmcAKUrMspAcorgQ2J1UETpU0pkHh1cQJvgb5EP1WcludpG8ATcCXGxiWFWM+ML/iaO0qUsIfDcyW9CTpB/9eSX/fmBCtaJKOBj4GfCpW3yx0JOnczFsRsRD4E9Cnn1HjBN8BSU0VZ8+HAhOAhyQdDxwAHBERbzcwRCtARDwHPCNp2zzqw8C9ETEyIloiooX0I/DePK+VjKSJpHMtB0XE6xWTngb2zfOsD+wBPNT7Edau4U+T7MM2Ay7NHZesA1wREddLWgE8BfxFEsCvI+JbDYzT6u+LwGX5GUqPA8c0OB4riKSpwD7AppLmA98gXTWzLnBz/o7fGREnkjow+qmkBwABP42IOQ0JvEZ+VIGZWUm5icbMrKSc4M3MSsoJ3syspJzgzcxKygnezKyknOCt35H0WudzFU/Sx/LjDGZLmifphDz+RElHNTo+M18maf2OpNciYoMGxzCIdD/E7hExX9K6QEtEPNzIuMwquQZv/ZakfST9UdK1kh6X9G1Jn8rP8Z8raes834GS7sq17Vsk/V0e3yTp5vy8/4skPdX6nHdJn87rmSXpx/mGt0rDSDcKLgGIiDdak7ukb0r6iqTN8/Ktr5WStsrl/krSPfn1/rzcByvmvU/SsF7alFZSTvDW3+0CnAiMJT3d890RsTtwEemOVIA7gD3yw8MuJ92GDumuxT9ExA6kZ85sCSBpLPBPwPvzw+ZWAp+qLDQ/RfA3wFOSpuYflnXazPO3iBiX13Eh8KuIeIr0vPnvRcRuwKE5VoCvACfl+fcClvVw29hazo8qsP7unoh4FkDSY8BNefxc4EP5/2bgfyVtBgwGnsjjPwAcAhAR0yS9mMd/mNShwz35VvWhpEdGryEijpe0E7AfKTlPAI5uO1+uoX8ul0eef/u8boANJW1AenjVeZIuIz0CY36XtoRZG07w1t+9UfH/2xXDb7N6//5v4LyI+I2kfUiP/61GwKUR0WmXbBExF5gr6eekH46j11hR+lH5CenBVa0nh9chHVEsb7O6b0v6LTAJ+JOkAyKiTz/Myvo2N9HY2mAjYEH+/7MV4/8E/COApP2BjfP43wOflDQyT9tE0laVK5S0Qf6xaDWOdNK1cp5BwJXAaRHx14pJN7G6+QhJ4/LfrSNibkR8B7iH1DWkWbc5wdva4JvAlZJmAosrxp8N7K/U4fJhwHPAqxExD/g6cJOkOaQ+Ojdrs04BX1XqgH1WXtfRbeZ5H+l54WdXnDzdHDgZGK/UqfM80jkEgC8pdeY+B3iL1T0JmXWLL5O0tVa+tHFlRKyQtCepF6dxDQ7LrG7cBm9rsy2BK/LVL2+SToSalYZr8GZmJeU2eDOzknKCNzMrKSd4M7OScoI3MyspJ3gzs5L6/xliRQNOKJ25AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Import Library\n",
    "\n",
    "import numpy as np \n",
    "import matplotlib.pyplot as plt \n",
    "\n",
    "# Define Data\n",
    "\n",
    "img_sizes = ['32', '64','128']\n",
    "step_10 = [403.887616, 1622.050304, 6460.294656]\n",
    "step_50 = [2019.390976, 8078.337536, 0]\n",
    "width = 0.2\n",
    "\n",
    "x_axis = np.arange(len(img_sizes))\n",
    "\n",
    "# Multi bar Chart\n",
    "\n",
    "plt.bar(x_axis - 0.1, step_10, width=width, label = '$N=10$', color='lightseagreen')\n",
    "plt.bar(x_axis +0.1, step_50, width=width, label = '$N=50$', color='darksalmon')\n",
    "\n",
    "# Xticks\n",
    "\n",
    "plt.xticks(x_axis, img_sizes)\n",
    "\n",
    "# Add labels\n",
    "\n",
    "plt.xlabel(\"Image Sizes\")\n",
    "plt.ylabel(\"Memory(in MB)\")\n",
    "\n",
    "# Add legend\n",
    "plt.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0,0))\n",
    "\n",
    "plt.legend()\n",
    "plt.title(\"Memory Consumption (Gradient Tracking On)\")\n",
    "\n",
    "# Display\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAnnUlEQVR4nO3deZwdVZn/8c+XLAQhrAkKaSAJoIQ1SEAQWVQIISMERGcCKrsBBhRHHcHlx6Yzo4OKOqASNQIOBgVEg4ZAFBhABBI0JBBAwmY6IgkJ+57w/P44p0mlU337dnKrb5P+vl+v++pb59Ty3LrV96mqU1VHEYGZmVl7azU7ADMz65mcIMzMrJQThJmZlXKCMDOzUk4QZmZWygnCzMxKOUGYFUi6T9L+Fc37JEnfqWLeNZY5VFJI6puHr5N0THfGUJX2n61d3ZaSXpDUpxmx5RhOkfRkjmMTSXtLeigPHybpakkHNyu+ukSEXw14AY8BrwGD2pX/BQhgaLNjXIXPtD7wHeBvwAvAw3l4ULNja9DnuwT4Wjctqz8wHxjSruws4EHgRWABcB0wuoHLHZq3v74N/jzHArfVqL8vbzMvAMuAVwrDX+rJn63OZb8XuBF4HngWuBbYvlDfD3gZ2KVQ9gfg9MLwHsDd3R17V14+gmisR4Ej2wYk7QS8rXnhLFe2l9XJ+P1JG/QOwBhSstgLWEzasK1rxgEPRMSCQtlVufxoYCNgGPBd4J/KZtDV77CZImKHiFgvItYDbgVOaxuOiP9sG++t9JnaSNoLuAH4DbA56Xu7B/ijpOF5tLcDA0iJss1WxeGIuAtYX9Ko7oh7lTQ7Q60pL9IRxFeAGYWybwJfpnAEAaydy/8GPAn8EFgn1+0PtAJfABYCTwCHAWOBvwJLKOx95Xl9B/h7fn0HWLvdvM4A/gH8DLgXOKQwfT/gKWDXks9zYo5vvRqfeQRwM/AMacM/tFB3CXAR8DvSXtadwNa5TsAF+TM+B8wBdsx1NwMnFuZzLIU91bwu/xV4KM/3q8DWwO15Xr8E+rdbB1/Kn/Mx4GO5bgLwOumo7wXg2sL3eEAX1u/nCt/VcTXW1STgK4XhA0h7mC11bFdnALOBV4G+wJmko7nngbnA4YXx+5C2r6eAR4BTKexll6zf44H7gaeB64Gt2q3rk/O6fiZ/n8rf+yukI4MXgGc6+QxvLpPle/0nkP4HbsnlV5K202eBW4AdCtOvA3wLeDzX35bL2ubV9tmOyOtrx5K6m0nbyh/zeruBwpEwKUk/TtoB+n/F7aDk89wKfL+k/DrgMuCdpCPCyOvnxvx9vZG/8xcK29GPgLOb/fvV4XfX7ADWlFfbBkU6XTAi/6O2kvYaigniAmAKsDEwkHRo+l+5bn9gKem0Qz/gk8Ai4Od53B3yBjYsj38ecAewKTCY9CP51Xbz+gbph24dUuL5RSHmccCcDj7PFcClNT5vP2Ae6ce3P/CB/I/3rlx/CcuPNvoClwNX5LqDgLuBDVn+g7NZrruZzhPEb0hHNDuQfjT/AAwHNiD9YB7Tbh18O6+D/fI/bjHGr5V9j11Yv+fldTEWeAnYqIP1NQP4aGH468DNdW5Xs4AtWL4j8VHSnutawL/kz9S2/k4GHsjjbwzcRAcJIn//8/L670vawbm93br+bf6etiRti2PKvpdOPkNxmUPzfC8D1i18puNJ23hbUp5VmP6iPI8hpP+r9+bx2ubVFzguf5Zt2i2n+LkfJv14r5OHv57rtif9aL+PtC1/k7TzsFKCIJ0RWAa8v6TuOOCJsuW337YKZZ8FftXs368Ov7tmB9DwD5T21BYC9zZoftNIe0+/7WS8x0gJ4ivAf5FOy0zPG2/kDUakf+atC9PtBTya3+9PSgB98vDAPO17CuPfDRyW3z8MjC3UHQQ8VpjXa8CAQv3mpB/x9fPwVcAXOvg809v+gTqo34e0x7dWoWwycE5+fwnw40LdWNIpFkjJ5K/AnsXpc93NdJ4g9m63Ps4oDH8L+E5hHSwF1i3U/xL4f4UYayWIztbvy6z4A7AQ2LOD9fUQ+cc1D/+YnDDz8MZ5O3sWeKVdPMd3su3NAsbl9zcCJxfqRtNxgrgOOKEw7lqkJLdVYV2/r926O7Pse+kkvuIyh+b5Dq8x/oZ5nA1yTCucyy+M1zavz5N2DFpK6oqfu3gE96/AtPz+LGByoe5tpP+dsgTRkue7XUndGOD1suW337YKZZ8EbqxnPTbjtSa2QVxC+qIa5XzgE10Y/2fAUaR/oMva1Q0mbXx3S3pG0jOkBDS4MM7iiFiW37+c/z5ZqH8ZWC+/35x0WNzm8VzWZlFEvNI2EBF/Jx1iHyFpQ+Bg0p59mcXAZh3UtS17fkS80W75QwrD/yi8f6kt7oi4EbiQtGe4UNJESevXWFZ77ddHR+sH4OmIeLFdjMV1VEtn63dxRCwtDL/5GUs8TUr4b05LYf1GxJKI2BDYjbR3XDS/OCDpaEmzCtvQjsCgQszF8Yvxt7cV8N3CfJaQdmI6/Q4b4M0YJfWR9HVJD0t6jvRDCukzDSKdy3+4xrz+HbgoIlo7WWZHn2WFdRYRL5G+nzJPk04Vlf1vbEY6tdcVA0k7Bj3SGpcgIuIW0ob+JklbS5om6W5Jt0rargvz+wNpr7ve8R8nNVaPBX7Vrvop0g/YDhGxYX5tEKkhb1X8nfRP3mbLXPZmOCXTXAp8nHSa4k+xYqNp0e+BgyStW2PZW0gqbkNbkq7E6VREfC8idiMd3r+T9E8O6Qir2LD/jnrmV8NG7T5DcR2VrZ+iztZvV8wmfc42fwB2l9RSx7RvxilpK9J569OATXJSuZf0ww6pLWSLdjF3ZD5wUmFb3DAi1omI27sS0yoqTn8U6XTXAaSjhqG5XKT/mVdI7UwdGQ18RdIRqxjLE6Qjg7RQaR1gk9Kg087Gn0j/P+39M+l77YoRpAbuHmmNSxAdmAh8Kv8gfR74fsXLOwH4QLs9V/Le9o+ACyRtCiBpiKSDVnE5k0n/GIMlDSIdKv9vJ9P8Gng3cDorH+EU/Yz0A3K1pO0krZWv5f6SpLGkRueXgC9I6pfvHTiE1HZRk6TdJb1HUj9SQniFtFcG6XTJhyW9TdI2pHW5us6V1F/SPsCHSA2ikI48hnc82Sqt345MJbWBABARN5DaB36d10X/vD727GQ+65J+XBcBSDqOdATR5pfApyW1SNqI1KDdkR8CX5S0Q57XBpLKfvjKPAm05KvdVtdAUlvSYtLOwZtXOeX/mUnAtyVtno829pJUPMq6j3TW4CJJh67C8q8CDpH03vx5zmF5wi1zJnCMpE9LGihpI0lfI50uPreLy96PdKqvR1rjE4Sk9UiNWldKmgVcTD48lPRhSfeWvK5fnWVGxMMRMbOD6jNIjWl35MPp3wPvWsVFfQ2YSdo7nQP8OZfViu1l4GrSpXntj3CK471K2qN7gNQe8RxwF+mQ/86IeI2UEA4m7eV9Hzg6Ih6oI+71SYnyaZZfOXJ+rruAdP73SdLRTkenwOr1j7ycv+d5nVyI8SfA9vkUy69Lpu3y+q3hWmA7ScVTVIeTGoH/l3Sa4VHgY6S2jlIRMZfUzvIn0jraiXTasM2PSFcj3ZPjrfUdX0O6iOGKvC3eS/o+63Ej6Yf5H5K6elqlvctI28ECUlvCHe3qP09a/zNIZwe+Qbvfroi4h5T8f9TVm88i4j7gU6SdmydIDdYLSUmrbPzbSN/Rh/P4jwO7ktprHqp3uZJ2B16IdLlrj6TcULJGkTSU1Ki8Yz63/WBE1Dqf3tn89gc+HxEfakyEzSXpLOCdEfHxZsdSpfy9/W9E1HMap3KSJpBupvpMs2OxjuWdymeAbSPi0QqXczXwk4iYWtUyVtdb7iaVroqI5yQ9KumjEXGlJAE75z2OXkfSxqTTNl1peLcGiIiJzY7Bykk6hNR+INJlrnNY3lheiYhY1TaTbrPGnWKSNJl0+P0uSa2STiAdtp8g6R7SYfG4LszvVtI56w/m+a1qe0HTSfokqV3hutyYb2bJOJbfELktMD7WxNMrXbRGnmIyM7PVt8YdQZiZWWOsUW0QgwYNiqFDhzY7DDOzt4y77777qYgYXFa3RiWIoUOHMnNmR1eXmplZe5I6vNvep5jMzKyUE4SZmZWqLEFI2kLSTZLmKnXjeHrJOJL0PUnzJM2W9O5C3TFK3fM9pDWki0Qzs7eSKtsglgKfi4g/SxpIeoLp9PyogDYHk6453hZ4D/AD4D35Zq6zgVGk587cLWlKRDzd1SBef/11WltbeeWVVzofuRcYMGAALS0t9OvXr9mhmFkPV1mCiIgnSM8pISKel3Q/6THCxQQxDrgs35Byh6QNJW1Getb+9IhYAiBpOulhXJO7GkdraysDBw5k6NChpJuoe6+IYPHixbS2tjJs2LBmh2NmPVy3tEHkZyPtSnoCaNEQVnx2fWsu66i8bN4TJM2UNHPRokUr1b/yyitssskmvT45AEhik0028dGUmdWl8gSRH3x1NfCZiHiu0fOPiIkRMSoiRg0eXHopr5NDgdeFmdWr0gSRn29/NXB5RJQ9dngBK3Zu0pLLOio3M7NuUuVVTCI9b//+iPh2B6NNAY7OVzPtCTyb2y6uB0bnjjg2IvUYtVp9NJiZWddUeRXT3qRHSs/JHfUAfIncBWJE/JDUy9ZYUgc6LwHH5bolkr5K6iAE4Ly2Bmszs0YYdu2VnY/UAHe2zqp8GZue8h+VzLfKq5huo3a3feSrl07toG4SqavBhmr0RvHoIfX10HjxxRdz8sknM3fuXEaMGAHAiBEjmDp1apevKDr++OP57W9/y6abbsq99967Qt20adM4/fTTWbZsGSeeeCJnnlmrx0kzs475TupuMmfOHEaOHMnvfvc7IF1d9eSTT7IqDxc89thjmTZt2krly5Yt49RTT+W6665j7ty5TJ48mblz55bMwcysc04Q3WT27NmcccYZbyaIuXPnst12263SVUX77rsvG2+88Urld911F9tssw3Dhw+nf//+jB8/nt/85jerHbuZ9U5OEN1k7ty5jBs3joULF/Lss88yZ84cdt555xXG2WeffRg5cuRKr9///vd1LWPBggVsscXyi79aWlpYsMAXf5nZqlmjHvfdU82fP59NNtmEddZZhwMPPJDrr7+e2bNns9NOO60w3q233tqkCM3MVuYE0Q3mzJnzZjIYO3Ysl19+OU888QSHHXbYCuPts88+PP/88ytN/81vfpMDDjig0+UMGTKE+fOX34De2trKkCGlN6CbmXXKCaIbFI8W9ttvP0466SRefvnlhh9B7L777jz00EM8+uijDBkyhCuuuIKf//znqzVPM+u9el2CqPey1EaaM2cORxxxBABrr702O++8M3/5y1/YcMMNV2l+Rx55JDfffDNPPfUULS0tnHvuuZxwwgn07duXCy+8kIMOOohly5Zx/PHHs8MOOzTwk5hZb9LrEkQzXH755SsMr+6VRZMnd/xQ27FjxzJ27NjVmr+ZGfgqJjMz64AThJmZlXKCMDOzUk4QZmZWygnCzMxKOUGYmVkpJwgzMyvlBGFmZqV63Y1yC3/w5YbOr96enBrZYdDQoUMZOHAgffr0oW/fvsycOfPNOncYZGaNUlmCkDQJ+BCwMCJ2LKn/d+BjhThGAINzd6OPAc8Dy4ClETGqqji7S7HDoBEjRqxWh0EAN910E4MGDVqhrK3DoOnTp9PS0sLuu+/OoYceyvbbb9+AT2BmvU2Vp5guAcZ0VBkR50fEyIgYCXwR+L92/U6/P9e/5ZMDNLbDoI64wyAza6TKEkRE3AIs6XTE5Eig4wcMrQEa2WGQJEaPHs1uu+3GxIkT3yx3h0Fm1khNb4OQ9DbSkcZpheIAbpAUwMURMbF04jT9BGACwJZbblllqKus0R0G3XbbbQwZMoSFCxdy4IEHst1227HvvvtWEbqZ9WJNTxDAIcAf251eel9ELJC0KTBd0gP5iGQlOXlMBBg1alRUH27XNbrDoLZOgDbddFMOP/xw7rrrLvbdd193GGRmDdUTEsR42p1eiogF+e9CSdcAewClCeKtoJEdBr344ou88cYbDBw4kBdffJEbbriBs846C3CHQWbWWE1NEJI2APYDPl4oWxdYKyKez+9HA+c1apn1XpbaSI3sMOjJJ5/k8MMPB2Dp0qUcddRRjBmTrgVwh0Fm1khVXuY6GdgfGCSpFTgb6AcQET/Mox0O3BARLxYmfTtwTb66py/w84iYVlWc3aGRHQYNHz6ce+65p8N6dxhkZo1SWYKIiCPrGOcS0uWwxbJHgF2qicrMzOrlR22YmVmpXpEgInrkxU1N4XVhZvVa4xPEgAEDWLx4sX8YSclh8eLFDBgwoNmhmNlbQE+4zLVSLS0ttLa2smjRomaH0iMMGDCAlpaWZodhZm8Ba3yC6NevX5eflmpmZr3gFJOZma0aJwgzMyvlBGFmZqWcIMzMrJQThJmZlXKCMDOzUk4QZmZWygnCzMxKOUGYmVkpJwgzMyvlBGFmZqWcIMzMrJQThJmZlaosQUiaJGmhpHs7qN9f0rOSZuXXWYW6MZIelDRP0plVxWhmZh2r8gjiEmBMJ+PcGhEj8+s8AEl9gIuAg4HtgSMlbV9hnGZmVqKyBBERtwBLVmHSPYB5EfFIRLwGXAGMa2hwZmbWqWa3Qewl6R5J10naIZcNAeYXxmnNZaUkTZA0U9JM9xpnZtY4zUwQfwa2iohdgP8Bfr0qM4mIiRExKiJGDR48uJHxmZn1ak1LEBHxXES8kN9PBfpJGgQsALYojNqSy8zMrBs1LUFIeock5fd75FgWAzOAbSUNk9QfGA9MaVacZma9Vd+qZixpMrA/MEhSK3A20A8gIn4IfAQ4RdJS4GVgfEQEsFTSacD1QB9gUkTcV1WcZmZWrrIEERFHdlJ/IXBhB3VTgalVxGVmZvVp9lVMZmbWQzlBmJlZKScIMzMr5QRhZmalnCDMzKyUE4SZmZVygjAzs1J13QchaSNgc9INbY9FxBuVRmVmZk3XYYKQtAFwKnAk0B9YBAwA3i7pDuD7EXFTt0RpZmbdrtYRxFXAZcA+EfFMsULSbsAnJA2PiJ9UGJ+ZmTVJhwkiIg6sUXc3cHclEZmZWY9Qs5FaUt/CE1e3kPQRSbt2T2hmZtZMHSYISZ8EFgKP5/d/ID2B9QpJZ3RTfGZm1iS12iA+A2wNDATuJ/X+9pSkt5H6bPhG9eGZmVmz1EoQr0XE08DTkuZFxFMAEfGSpNe6JzwzM2uWWglindzesBbQP79Xfg3ojuDMzKx5aiWIfwDfLnnfNmxmZmuwWpe57r86M5Y0CfgQsDAidiyp/xhwBumI5HnglIi4J9c9lsuWAUsjYtTqxGJmZl1X607qD9eaMCJ+1cm8LyF1KXpZB/WPAvtFxNOSDgYmAu8p1L+/rd3DzMy6X2d3Us/KL0h7+m0CqJkgIuIWSUNr1N9eGLwDaKk1PzMz6161EsSHgfHAzsBvgMkRMa+iOE4ArisMB3CDpAAujoiJHU0oaQIwAWDLLbesKDwzs96nwxvlIuLXETEe2A94GPiWpNsk7dfIACS9n5QgijffvS8i3g0cDJwqad8acU6MiFERMWrw4MGNDM3MrFerpz+IV4BngeeA9WjgJa6SdgZ+DIyLiMVt5RGxIP9dCFwD7NGoZZqZWX1qPWrjA5Imkh7K937guxExMiKub8SCJW1Jasf4RET8tVC+rqSBbe+B0cC9jVimmZnVr1YbxO+B2cBtwNrA0ZKObquMiE/XmrGkycD+wCBJrcDZQL887Q+Bs4BNgO/n5wG2Xc76duCaXNYX+HlETFuVD2dmZquuVoI4bnVmHBFHdlJ/InBiSfkjwC6rs2wzM1t9tW6Uu7Q7AzEzs56lnkZqMzPrhZwgzMyslBOEmZmVqtVIDYCkwcAngaHF8SPi+OrCMjOzZus0QZAes3Er6bLXZdWGY2ZmPUU9CeJtEeE+qM3Mepl62iB+K2ls5ZGYmVmPUk+COJ2UJF6W9Jyk5yU9V3VgZmbWXJ2eYoqIgd0RiJmZ9Sy1epTbLiIekPTusvqI+HN1YZmZWbPVOoL4LKkjnm+V1AXwgUoiMjOzHqHWs5gm5L/v775wzMysp6jVH8T7ak0oaX1JOzY+JDMz6wlqnWI6QtJ/A9NInQYtIvUmtw2pA6GtgM9VHqGZmTVFrVNM/yZpY+AI4KPAZsDLwP3AxRFxW/eEaGZmzVDzMteIWCJpUkT8qLsCMjOznqGeG+UeknS+pBFdnbmkSZIWSirtU1rJ9yTNkzS7eEmtpGMkPZRfx3R12WZmtnrqSRC7AH8FfiLpDkkTJK1f5/wvAcbUqD8Y2Da/JgA/AMints4G3gPsAZwtaaM6l2lmZg3QaYKIiOcj4kcR8V7gDNIP9xOSLpW0TSfT3gIsqTHKOOCySO4ANpS0GXAQMD0ilkTE08B0aicaMzNrsE4ThKQ+kg6VdA3wHdKNc8OBa4Gpq7n8IcD8wnBrLuuovCy+CZJmSpq5aNGi1QzHzMza1PO474eAm4DzI+L2QvlVkvatJqz6RcREYCLAqFGjosnhmJmtMWoeQUjqA1wSESe0Sw4ARMSnV3P5C4AtCsMtuayjcjMz6yY1E0RELAM+VOHypwBH56uZ9gSejYgngOuB0ZI2yo3To3OZmZl1k3pOMf1R0oXAL4AX2wrreZqrpMnA/sAgSa2kBu5+efofktowxgLzgJeA43LdEklfBWbkWZ0XEbUau83MrMHqSRAj89/zCmV1Pc01Io7spD6AUzuomwRMqiM+MzOrQD0dBvlprmZmvVA9l7luIOnbbZeSSvqWpA26IzgzM2ueeu6kngQ8D/xzfj0H/LTKoMzMrPnqaYPYOiKOKAyfK2lWRfGYmVkPUc8RxMvFzoMk7U167LeZma3B6jmCOAW4NLc7iPRspWOrDMrMzJqvnquYZgG7tD3BNSKeqzooMzNrvk4ThKQNgaOBoUBfSUBDHrNhZmY9WD2nmKYCdwBzgDeqDcfMzHqKehLEgIj4bOWRmJlZj1LPVUw/k/RJSZtJ2rjtVXlkZmbWVPUcQbwGnA98mfQMJvLf4VUFZWZmzVdPgvgcsE1EPFV1MGZm1nPUc4qp7VHcZmbWi9RzBPEiMEvSTcCrbYW+zNXMbM1WT4L4dX6ZmVkvUs+d1JdKWgfYMiIe7IaYzMysB6inP4hDgFnAtDw8UtKUiuMyM7Mmq6eR+hxgD+AZePPZTHVd4ippjKQHJc2TdGZJ/QWSZuXXXyU9U6hbVqhzQjIz62b1tEG8HhHPtj2DKev0kRuS+gAXAQcCrcAMSVMiYm7bOBHxb4XxPwXsWpjFyxExso74zMysAvUcQdwn6Sigj6RtJf0PcHsd0+0BzIuIRyLiNeAKYFyN8Y8EJtcxXzMz6wb1JIhPATuQLnGdTOpy9DN1TDcEmF8Ybs1lK5G0FTAMuLFQPCD3gX2HpMM6WoikCW39ZS9atKiOsMzMrB71XMX0EukxG1+uMI7xwFURsaxQtlVELJA0HLhR0pyIeLgkvonARIBRo0ZF+3ozM1s1HSaIzhqGI+LQTua9ANiiMNySy8qMB05tN/8F+e8jkm4mtU+slCDMzKwatY4g9iKdIpoM3EnqbrQrZgDbShpGSgzjgaPajyRpO2Aj4E+Fso2AlyLiVUmDgL2B/+7i8s3MbDXUShDvIF2BdCTph/13wOSIuK+eGUfEUkmnAdcDfYBJEXGfpPOAmRHRdoQyHrgiIoqnh0YAF0t6g9RO8vXi1U9mZla9DhNEbg+YBkyTtDYpUdws6dyIuLCemUfEVFKPdMWys9oNn1My3e3ATvUsw8zMqlGzkTonhn8iJYehwPeAa6oPy8zMmq1WI/VlwI6kI4BzI+LebovKzMyartYRxMdJj/o+Hfh04U5qARER61ccm5mZNVGtNoh6bqIzM7M1lJOAmZmVcoIwM7NSThBmZlbKCcLMzEo5QZiZWSknCDMzK+UEYWZmpZwgzMyslBOEmZmVcoIwM7NSThBmZlbKCcLMzEo5QZiZWalKE4SkMZIelDRP0pkl9cdKWiRpVn6dWKg7RtJD+XVMlXGamdnKavYotzok9QEuIvVr3QrMkDSlpG/pX0TEae2m3Rg4GxgFBHB3nvbpquI1M7MVVXkEsQcwLyIeiYjXgCuAcXVOexAwPSKW5KQwHRhTUZxmZlaiygQxBJhfGG7NZe0dIWm2pKskbdHFaZE0QdJMSTMXLVrUiLjNzIwKTzHV6VpgckS8Kukk4FLgA12ZQURMBCYCjBo1Khofoq3phl17ZeXLuLN1VuXL2PSU/6h8Gda7VHkEsQDYojDcksveFBGLI+LVPPhjYLd6pzUzs2pVmSBmANtKGiapPzAemFIcQdJmhcFDgfvz++uB0ZI2krQRMDqXmZlZN6nsFFNELJV0GumHvQ8wKSLuk3QeMDMipgCflnQosBRYAhybp10i6aukJANwXkQsqSpWMzNbWaVtEBExFZjaruyswvsvAl/sYNpJwKQq4zMzs475TmozMyvlBGFmZqWcIMzMrJQThJmZlXKCMDOzUk4QZmZWygnCzMxKOUGYmVkpJwgzMyvlBGFmZqWcIMzMrJQThJmZlXKCMDOzUk4QZmZWygnCzMxKOUGYmVkpJwgzMytVaYKQNEbSg5LmSTqzpP6zkuZKmi3pD5K2KtQtkzQrv6a0n9bMzKpVWZejkvoAFwEHAq3ADElTImJuYbS/AKMi4iVJpwD/DfxLrns5IkZWFZ+ZmdVW5RHEHsC8iHgkIl4DrgDGFUeIiJsi4qU8eAfQUmE8ZmbWBVUmiCHA/MJway7ryAnAdYXhAZJmSrpD0mEdTSRpQh5v5qJFi1YrYDMzW66yU0xdIenjwChgv0LxVhGxQNJw4EZJcyLi4fbTRsREYCLAqFGjolsCNjPrBao8glgAbFEYbsllK5B0APBl4NCIeLWtPCIW5L+PADcDu1YYq5mZtVNlgpgBbCtpmKT+wHhghauRJO0KXExKDgsL5RtJWju/HwTsDRQbt83MrGKVnWKKiKWSTgOuB/oAkyLiPknnATMjYgpwPrAecKUkgL9FxKHACOBiSW+QktjX2139ZGZmFau0DSIipgJT25WdVXh/QAfT3Q7sVGVsZmZWm++kNjOzUk4QZmZWygnCzMxKOUGYmVkpJwgzMyvlBGFmZqWcIMzMrJQThJmZleoRD+vrCYZde2Xly7izdVblywDY9JT/6JblmNmazUcQZmZWygnCzMxKOUGYmVkpJwgzMyvlBGFmZqWcIMzMrJQThJmZlXKCMDOzUk4QZmZWqtIEIWmMpAclzZN0Zkn92pJ+kevvlDS0UPfFXP6gpIOqjNPMzFZWWYKQ1Ae4CDgY2B44UtL27UY7AXg6IrYBLgC+kafdHhgP7ACMAb6f52dmZt2kyiOIPYB5EfFIRLwGXAGMazfOOODS/P4q4IOSlMuviIhXI+JRYF6en5mZdZMqH9Y3BJhfGG4F3tPROBGxVNKzwCa5/I520w4pW4ikCcCEPPiCpAdXP/RqvB0GAU9VvqB//c/KF2Fd0y3fvb/3Hukt8N1v1VHFW/5prhExEZjY7DjqIWlmRIxqdhzW/fzd915v5e++ylNMC4AtCsMtuax0HEl9gQ2AxXVOa2ZmFaoyQcwAtpU0TFJ/UqPzlHbjTAGOye8/AtwYEZHLx+ernIYB2wJ3VRirmZm1U9kpptymcBpwPdAHmBQR90k6D5gZEVOAnwA/kzQPWEJKIuTxfgnMBZYCp0bEsqpi7UZviVNhVgl/973XW/a7V9phNzMzW5HvpDYzs1JOEGZmVsoJoiKSBki6S9I9ku6TdG4uvzw/PuReSZMk9Wt2rNZYkjaUdJWkByTdL2mvQt3nJIWkQc2M0Roj/w8vlHRvoez8/N3PlnSNpA1zeT9Jl0qak7eLLzYt8Do5QVTnVeADEbELMBIYI2lP4HJgO2AnYB3gxKZFaFX5LjAtIrYDdgHuB5C0BTAa+FsTY7PGuoT0OKCi6cCOEbEz8FegLRF8FFg7InYCdgNOKj5/ridygqhIJC/kwX75FRExNdcF6dLdlqYFaQ0naQNgX9IVekTEaxHxTK6+APgC4CtD1hARcQvpCsxi2Q0RsTQP3sHy//EA1s33fK0DvAY8112xrgoniApJ6iNpFrAQmB4Rdxbq+gGfAKY1KTyrxjBgEfBTSX+R9GNJ60oaByyIiHuaHJ91r+OB6/L7q4AXgSdIR5HfjIglHU3YEzhBVCgilkXESNIexB6SdixUfx+4JSJubUpwVpW+wLuBH0TErqQfhHOALwFnNTEu62aSvky6j+vyXLQHsAzYnLQj8TlJw5sUXl2cILpBPsVwE/lcpaSzgcHAZ5sYllWjFWgtHC1eRUoYw4B7JD1G2mH4s6R3NCdEq5qkY4EPAR+L5TebHUVqm3o9IhYCfwR69DOanCAqImlw4eqFdYADgQcknQgcBBwZEW80MUSrQET8A5gv6V256IPAnyNi04gYGhFDSUnk3XlcW8NIGkNqazo0Il4qVP0N+EAeZ11gT+CB7o+wfm/5p7n2YJsBl+aOjtYCfhkRv5W0FHgc+FPq+oJfRcR5TYzTGu9TwOX5GWSPAMc1OR6riKTJwP7AIEmtwNmkq5bWBqbn//E7IuJkUgdqP5V0HyDgpxExuymB18mP2jAzs1I+xWRmZqWcIMzMrJQThJmZlXKCMDOzUk4QZmZWygnCeh1JL3Q+VvUkfSg/juMeSXMlnZTLT5Z0dLPjM/NlrtbrSHohItZrcgz9SPfD7BERrZLWBoZGxIPNjMusyEcQ1mtJ2l/S/0n6jaRHJH1d0sdyPx5zJG2dxztE0p15b//3kt6eywdLmp77+/ixpMfb+nmQ9PE8n1mSLs43TBYNJN2ouhggIl5tSw6SzpH0eUmb5+nbXsskbZWXe7WkGfm1d55uv8K4f5E0sJtWpa2hnCCst9sFOBkYQXq67jsjYg/gx6Q7ogFuA/bMD9+7gvQYBUh3zd4YETuQnrm0JYCkEcC/AHvnhzUuAz5WXGh+iucU4HFJk3NiWqvdOH+PiJF5Hj8Cro6Ix0n9TVwQEbsDR+RYAT4PnJrH3wd4eTXXjfVyftSG9XYzIuIJAEkPAzfk8jnA+/P7FuAXkjYD+gOP5vL3AYcDRMQ0SU/n8g+SOoSZkR+1sA7pke8riIgTJe0EHED6cT8QOLb9ePkI4ZN5eeTxt8/zBlhf0nqkh799W9LlpEe4tHZpTZi14wRhvd2rhfdvFIbfYPn/x/8A346IKZL2Jz2+uxYBl0ZEp11KRsQcYI6kn5ESz7ErzCglpZ+QHvzW1ri+FumI5pV2s/u6pN8BY4E/SjooInr0w+CsZ/MpJrPObQAsyO+PKZT/EfhnAEmjgY1y+R+Aj0jaNNdtLGmr4gwlrZeTTZuRpEbr4jj9gCuBMyLir4WqG1h++gtJI/PfrSNiTkR8A5hB6trWbJU5QZh17hzgSkl3A08Vys8FRit1WP9R4B/A8xExF/gKcIOk2aQ+ijdrN08BX5D0YO518FxWPr30XlJ/AecWGp83Bz4NjJI0W9JcUhsKwGck3ZuX+TrLezIzWyW+zNVsFeVLU5dFxFJJe5F6kRvZ5LDMGsZtEGarbkvgl/nqo9dIDclmawwfQZiZWSm3QZiZWSknCDMzK+UEYWZmpZwgzMyslBOEmZmV+v+RzgSQllWn+AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Define Data\n",
    "\n",
    "img_sizes = ['32', '64','128']\n",
    "step_10 = [0.012288, 0.049152, 0.196608]\n",
    "step_50 = [0.012288, 0.049152, 0.196608]\n",
    "width = 0.2\n",
    "\n",
    "x_axis = np.arange(len(img_sizes))\n",
    "\n",
    "# Multi bar Chart\n",
    "\n",
    "plt.bar(x_axis - 0.1, step_10, width=width, label = '$N=10$', color='lightseagreen')\n",
    "plt.bar(x_axis +0.1, step_50, width=width, label = '$N=50$', color='darksalmon')\n",
    "\n",
    "# Xticks\n",
    "\n",
    "plt.xticks(x_axis, img_sizes)\n",
    "\n",
    "# Add labels\n",
    "\n",
    "plt.xlabel(\"Image Sizes\")\n",
    "plt.ylabel(\"Memory(in MB)\")\n",
    "\n",
    "# Add legend\n",
    "plt.ticklabel_format(axis=\"y\", style=\"sci\", scilimits=(0,0))\n",
    "\n",
    "plt.legend()\n",
    "plt.title(\"Memory Consumption (Gradient Tracking Off)\")\n",
    "\n",
    "# Display \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mykernel",
   "language": "python",
   "name": "mykernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}