notebooks/pytorch_basics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d2accc44-fe81-49ab-9593-f22922d7fd82",
   "metadata": {},
   "source": [
    "# Pytorch Basics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "995b0071-0edd-4b25-b3e3-8a0e75458b1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import packages\n",
    "import torch\n",
    "import torchvision\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "import torchvision.transforms as transforms"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84034c33-2b74-446d-812d-3cc06ef17527",
   "metadata": {},
   "source": [
    "## Basic Autograd Example 1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c7fcd288-62a9-4135-bdad-c5b2e63fd091",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create tensors\n",
    "x = torch.tensor(1., requires_grad=True)\n",
    "w = torch.tensor(2., requires_grad=True)\n",
    "b = torch.tensor(3., requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fc688ab0-8799-45e0-bef0-8e41b192fa21",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build a computational graph.\n",
    "y = w * x + b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f0924232-c24b-48c1-9fc0-61f7e5d35769",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute gradients.\n",
    "y.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b347b83d-b7cd-4485-a0f0-953d55c342ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(2.)\n",
      "tensor(1.)\n",
      "tensor(1.)\n"
     ]
    }
   ],
   "source": [
    "# Print out the gradients.\n",
    "print(x.grad)\n",
    "print(w.grad)\n",
    "print(b.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fbfab706-b19f-463d-ae59-31c881c96f28",
   "metadata": {},
   "source": [
    "## Basic Autograd Example 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "29e47134-1c47-49b9-897f-d22eb36dc7f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create tensors of shape (10, 3) and (10, 2).\n",
    "x = torch.randn(10, 3)\n",
    "y = torch.randn(10, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8ce83424-a87c-4312-8c09-ffdbe29fb4da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "w:  Parameter containing:\n",
      "tensor([[ 0.4277, -0.3905,  0.0272],\n",
      "        [ 0.5755,  0.5655,  0.0907]], requires_grad=True)\n",
      "b:  Parameter containing:\n",
      "tensor([-0.0968,  0.1185], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "# Build a fully connected layer.\n",
    "linear = nn.Linear(3, 2)\n",
    "print('w: ', linear.weight)\n",
    "print('b: ', linear.bias)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6967a327-1c22-4396-a305-cd51895c3616",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build loss function and optimizer.\n",
    "criterion = nn.MSELoss()\n",
    "optimizer = torch.optim.SGD(linear.parameters(), lr=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3f7b1535-47c8-4551-990f-6625bcec3ef4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Forward pass.\n",
    "pred = linear(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "57f6a899-e0e6-40dd-bdfc-d9f4a5b9a9ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:  1.0568517446517944\n"
     ]
    }
   ],
   "source": [
    "# Compute loss\n",
    "loss = criterion(pred, y)\n",
    "print('loss: ', loss.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4fd1663a-dda6-451d-a9c9-8305cc730f51",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Backward pass.\n",
    "loss.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2eeeb7e8-14ae-4231-af5e-2cb22ae94dd6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dL/dw:  tensor([[ 0.2306, -0.3402,  0.2334],\n",
      "        [ 0.0816,  0.2783, -0.0115]])\n",
      "dL/db:  tensor([-0.0753,  0.0586])\n"
     ]
    }
   ],
   "source": [
    "# Print out the gradients.\n",
    "print('dL/dw: ', linear.weight.grad)\n",
    "print('dL/db: ', linear.bias.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6db07e46-d93a-4ce7-a22c-f10ab247a9d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1-step gradient descent.\n",
    "optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "837ae33c-d62c-48c3-9b6a-824b8fbd2c4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Perform gradient descent at the low level.\n",
    "# linear.weight.data.sub_(0.01 * linear.weight.grad.data)\n",
    "# linear.bias.data.sub_(0.01 * linear.bias.grad.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "15135021-e329-4e15-b2a7-8a6bb3259111",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss after 1 step optimization:  1.0536946058273315\n"
     ]
    }
   ],
   "source": [
    "# Print out the loss after 1-step gradient descent.\n",
    "pred = linear(x) # Optimizer make it change.\n",
    "loss = criterion(pred, y)\n",
    "print('loss after 1 step optimization: ', loss.item())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c85df356-fa6d-421e-a352-305f0e207b9f",
   "metadata": {},
   "source": [
    "## Loading Data from Numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ad180f15-2d0d-4801-a220-6d85c2ef30c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a numpy array.\n",
    "x = np.array([[1, 2], [3, 4]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "10654b8e-c55d-4202-b4c2-1b2f4fff8de3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert the numpy array to a torch tensor.\n",
    "y = torch.from_numpy(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5a2f2893-a197-4e16-995e-337aeacc9cfb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert the torch tensor to a numpy array.\n",
    "z = y.numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1030f11f-bfca-4f81-b79d-a0fa6c158a9d",
   "metadata": {},
   "source": [
    "## Input Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "30816c38-8f33-4905-80d3-8185a95b0420",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download and construct CIFAR-10 dataset.\n",
    "train_dataset = torchvision.datasets.CIFAR10(root='../data/CIFAR10/', train=True, transform=transforms.ToTensor(), download=False) # '../' 为向上跳一级"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d40e9abc-b951-46a2-a533-5424603e0db6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 32, 32])\n",
      "6\n"
     ]
    }
   ],
   "source": [
    "# Fetch one data pair (read data from disk).\n",
    "image, label = train_dataset[0]\n",
    "print(image.size())\n",
    "print(label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "00c61d16-a00b-4f59-986c-ca98a7028002",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data loader (this provides quenes and threads in a very simple way).\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=64, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "2eca6b56-b526-41e7-8731-e268271fdec7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# When iteration starts, quene and thread start to load data from files.\n",
    "data_iter = iter(train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "8932812f-6340-4170-8839-0202c9e20a21",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Mini-batch images and labels.\n",
    "images, labels = data_iter.next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d7b15fe8-0e1e-4003-9813-addcad7909ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Actual usage of the data loader is as below.\n",
    "for images, labels in train_loader:\n",
    "    # Training code should be written here\n",
    "    pass"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6bb2f75a-887c-4fb6-95e2-5e97c9216719",
   "metadata": {},
   "source": [
    "## Input Pipeline for Custom Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1a20fe50-398f-4725-903e-a2691bc25323",
   "metadata": {},
   "outputs": [],
   "source": [
    "# You should build your custom dataset as below.\n",
    "class CustomDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self):\n",
    "        # Initialize file paths or a list of file names.\n",
    "        pass\n",
    "    def __getitem__(self, index):\n",
    "        # TODO\n",
    "        # 1. Read one data from file (e.g. using numpy.fromfile, PIL.Image.open).\n",
    "        # 2. Preprocess the data (e.g. torchvision.Transform).\n",
    "        # 3. Return a data pair (e.g. image and label).\n",
    "        pass\n",
    "    def __len__(self):\n",
    "        # You should change 0 to the total size of your dataset.\n",
    "        return 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "3cd1ea85-3841-46bb-8c15-3a5854870109",
   "metadata": {},
   "outputs": [],
   "source": [
    "# You can then use the prebuilt data loader.\n",
    "custom_dataset = CustomDataset()\n",
    "train_loader = torch.utils.data.DataLoader(dataset=custom_dataset, batch_size=64, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c6ae821-3ffd-4262-87ee-24a4ecdee53f",
   "metadata": {},
   "source": [
    "## Pretrained Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "57255099-4306-4660-873f-8238c32360e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"https://download.pytorch.org/models/resnet18-5c106cde.pth\" to C:\\Users\\林景/.cache\\torch\\hub\\checkpoints\\resnet18-5c106cde.pth\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36298e5c44e9447cba12c2316cbf48d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0.00/44.7M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Download and load the pretrained ResNet-18.\n",
    "resnet = torchvision.models.resnet18(pretrained=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d752617f-d0c1-43b3-84bb-263d28390945",
   "metadata": {},
   "outputs": [],
   "source": [
    "# If you want to finetune only the top layer of the model, set as below.\n",
    "for param in resnet.parameters():\n",
    "    param.requires_grad = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ae01943f-cd5a-47c2-b7bf-69ad51f1160e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replace the top layer for finetuning.\n",
    "resnet.fc = nn.Linear(resnet.fc.in_features, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "2d4e241a-904c-44a9-866b-413360a2a182",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([64, 100])\n"
     ]
    }
   ],
   "source": [
    "# Forward pass.\n",
    "images = torch.randn(64, 3, 224, 224)\n",
    "outputs = resnet(images)\n",
    "print (outputs.size()) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c60e6c0-9df3-4708-bd09-e1396c63bfbf",
   "metadata": {},
   "source": [
    "## Save and Load the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "3bcf5658-e360-42b2-85c8-533d379df36d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save and load the entire model.\n",
    "torch.save(resnet, 'model.ckpt')\n",
    "model = torch.load('model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "1dbf22c8-f094-4b5e-a1bb-66c5e43c60a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save and load only the model parameters (recommended).\n",
    "torch.save(resnet.state_dict(), 'params.ckpt')\n",
    "resnet.load_state_dict(torch.load('params.ckpt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76c606e4-52a8-4a1f-aed4-c94261139802",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "0c3b8f75513a4b88917614713a3cd247": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "1a4ba392b6954977b928f641a67d0f00": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "2021d5defbd34dd9bf42e535a7487d57": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "2366dc9840424344ab3f8f257187cef5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_593c0a156dce45f7be4c833811037770",
       "style": "IPY_MODEL_0c3b8f75513a4b88917614713a3cd247",
       "value": " 44.7M/44.7M [00:09&lt;00:00, 5.30MB/s]"
      }
     },
     "36298e5c44e9447cba12c2316cbf48d4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "children": [
        "IPY_MODEL_f7d08b3b149b479daee97cd4d45c6808",
        "IPY_MODEL_94b8de7561c04b9fac3b33f834431f49",
        "IPY_MODEL_2366dc9840424344ab3f8f257187cef5"
       ],
       "layout": "IPY_MODEL_914b90a9a0f34a34882c01facfef01c5"
      }
     },
     "593c0a156dce45f7be4c833811037770": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "8e15404b53574c7faa6b5d933527c132": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "914b90a9a0f34a34882c01facfef01c5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {}
     },
     "94b8de7561c04b9fac3b33f834431f49": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "bar_style": "success",
       "layout": "IPY_MODEL_8e15404b53574c7faa6b5d933527c132",
       "max": 46827520,
       "style": "IPY_MODEL_b53f7401f47747f8bea2b1f94b6e4aac",
       "value": 46827520
      }
     },
     "b53f7401f47747f8bea2b1f94b6e4aac": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "description_width": ""
      }
     },
     "f7d08b3b149b479daee97cd4d45c6808": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "layout": "IPY_MODEL_1a4ba392b6954977b928f641a67d0f00",
       "style": "IPY_MODEL_2021d5defbd34dd9bf42e535a7487d57",
       "value": "100%"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}