usage.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyPC/KxIybNegHt/pUMQxfKb",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/anarlavrenov/ExpressNet/blob/master/usage.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##ExpressNet Usage Example 😀"
      ],
      "metadata": {
        "id": "MFzFR482aZdU"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Import necessary libraries - we use only torch & torchtext"
      ],
      "metadata": {
        "id": "GPVPfVwramWe"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "v6WX925qpTn2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "outputId": "123120a2-53c2-4ce0-d85f-c7497b0ee3db"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'ExpressNet'...\n",
            "remote: Enumerating objects: 40, done.\u001b[K\n",
            "remote: Counting objects: 100% (40/40), done.\u001b[K\n",
            "remote: Compressing objects: 100% (38/38), done.\u001b[K\n",
            "remote: Total 40 (delta 12), reused 12 (delta 0), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (40/40), 134.79 KiB | 5.39 MiB/s, done.\n",
            "Resolving deltas: 100% (12/12), done.\n",
            "/content/ExpressNet\n",
            "Collecting torchdata\n",
            "  Downloading torchdata-0.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (13 kB)\n",
            "Requirement already satisfied: urllib3>=1.25 in /usr/local/lib/python3.10/dist-packages (from torchdata) (2.0.7)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from torchdata) (2.31.0)\n",
            "Requirement already satisfied: torch>=2 in /usr/local/lib/python3.10/dist-packages (from torchdata) (2.3.1+cu121)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=2->torchdata) (3.15.4)\n",
            "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch>=2->torchdata) (4.12.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=2->torchdata) (1.13.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=2->torchdata) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=2->torchdata) (3.1.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=2->torchdata) (2024.6.1)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=2->torchdata)\n",
            "  Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=2->torchdata)\n",
            "  Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=2->torchdata)\n",
            "  Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=2->torchdata)\n",
            "  Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch>=2->torchdata)\n",
            "  Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch>=2->torchdata)\n",
            "  Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-curand-cu12==10.3.2.106 (from torch>=2->torchdata)\n",
            "  Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=2->torchdata)\n",
            "  Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=2->torchdata)\n",
            "  Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-nccl-cu12==2.20.5 (from torch>=2->torchdata)\n",
            "  Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl.metadata (1.8 kB)\n",
            "Collecting nvidia-nvtx-cu12==12.1.105 (from torch>=2->torchdata)\n",
            "  Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\n",
            "Requirement already satisfied: triton==2.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=2->torchdata) (2.3.1)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=2->torchdata)\n",
            "  Downloading nvidia_nvjitlink_cu12-12.5.82-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->torchdata) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->torchdata) (3.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->torchdata) (2024.7.4)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=2->torchdata) (2.1.5)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=2->torchdata) (1.3.0)\n",
            "Downloading torchdata-0.7.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.7 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.7/4.7 MB\u001b[0m \u001b[31m35.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hUsing cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n",
            "Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "Downloading nvidia_nvjitlink_cu12-12.5.82-py3-none-manylinux2014_x86_64.whl (21.3 MB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.3/21.3 MB\u001b[0m \u001b[31m47.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, torchdata\n",
            "Successfully installed nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.5.82 nvidia-nvtx-cu12-12.1.105 torchdata-0.7.1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torchtext/data/__init__.py:4: UserWarning: \n",
            "/!\\ IMPORTANT WARNING ABOUT TORCHTEXT STATUS /!\\ \n",
            "Torchtext is deprecated and the last released version will be 0.18 (this one). You can silence this warning by calling the following at the beginnign of your scripts: `import torchtext; torchtext.disable_torchtext_deprecation_warning()`\n",
            "  warnings.warn(torchtext._TORCHTEXT_DEPRECATION_MSG)\n",
            "/usr/local/lib/python3.10/dist-packages/torchtext/vocab/__init__.py:4: UserWarning: \n",
            "/!\\ IMPORTANT WARNING ABOUT TORCHTEXT STATUS /!\\ \n",
            "Torchtext is deprecated and the last released version will be 0.18 (this one). You can silence this warning by calling the following at the beginnign of your scripts: `import torchtext; torchtext.disable_torchtext_deprecation_warning()`\n",
            "  warnings.warn(torchtext._TORCHTEXT_DEPRECATION_MSG)\n",
            "/usr/local/lib/python3.10/dist-packages/torchtext/utils.py:4: UserWarning: \n",
            "/!\\ IMPORTANT WARNING ABOUT TORCHTEXT STATUS /!\\ \n",
            "Torchtext is deprecated and the last released version will be 0.18 (this one). You can silence this warning by calling the following at the beginnign of your scripts: `import torchtext; torchtext.disable_torchtext_deprecation_warning()`\n",
            "  warnings.warn(torchtext._TORCHTEXT_DEPRECATION_MSG)\n",
            "/usr/local/lib/python3.10/dist-packages/torchtext/datasets/__init__.py:4: UserWarning: \n",
            "/!\\ IMPORTANT WARNING ABOUT TORCHTEXT STATUS /!\\ \n",
            "Torchtext is deprecated and the last released version will be 0.18 (this one). You can silence this warning by calling the following at the beginnign of your scripts: `import torchtext; torchtext.disable_torchtext_deprecation_warning()`\n",
            "  warnings.warn(torchtext._TORCHTEXT_DEPRECATION_MSG)\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/anarlavrenov/ExpressNet\n",
        "%cd ExpressNet\n",
        "\n",
        "!pip install torchdata\n",
        "!pip install portalocker>=2.0.0\n",
        "\n",
        "from ExpressNet.model import ExpressNet\n",
        "\n",
        "import torch\n",
        "import torchtext; torchtext.disable_torchtext_deprecation_warning\n",
        "from torchtext.data.utils import get_tokenizer\n",
        "from torchtext.vocab import build_vocab_from_iterator\n",
        "from torchtext.datasets import AG_NEWS\n",
        "\n",
        "from tqdm import tqdm\n",
        "\n",
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\""
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Download data & create simple vocab"
      ],
      "metadata": {
        "id": "ovObT-VtavrO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_iter = AG_NEWS(split=\"train\")\n",
        "valid_iter = AG_NEWS(split=\"test\")\n",
        "\n",
        "def yield_tokens(data_iter, tokenizer):\n",
        "  for label, text in data_iter:\n",
        "    yield tokenizer(text)\n",
        "\n",
        "tokenizer = get_tokenizer(tokenizer=\"basic_english\")\n",
        "vocab = build_vocab_from_iterator(iterator=yield_tokens(train_iter, tokenizer), specials=[\"<unk>\"])\n",
        "vocab.set_default_index(vocab[\"<unk>\"])"
      ],
      "metadata": {
        "id": "lz3ufvCuQeul"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Create torch loader"
      ],
      "metadata": {
        "id": "cqj26qVza5O2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def collate_fn(batch):\n",
        "\n",
        "  labels, text = zip(*batch)\n",
        "\n",
        "  tokens_ = []\n",
        "  labels_ = []\n",
        "\n",
        "  for label, t in zip(labels, text):\n",
        "    tokens = torch.IntTensor([vocab[word.lower()] for word in t.split()])\n",
        "    tokens = torch.nn.functional.pad(tokens, (0, 96-len(tokens)))\n",
        "\n",
        "    tokens_.append(tokens)\n",
        "    if label == 1:\n",
        "      labels_.append(0)\n",
        "    elif label == 2:\n",
        "      labels_.append(1)\n",
        "    elif label == 3:\n",
        "      labels_.append(2)\n",
        "    elif label == 4:\n",
        "      labels_.append(3)\n",
        "\n",
        "  tokens_ = torch.stack(tokens_, dim=0)\n",
        "  labels_ = torch.tensor(labels_, dtype=torch.long)\n",
        "\n",
        "  return tokens_, labels_"
      ],
      "metadata": {
        "id": "q8_SQBNAQ4wG"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_loader = torch.utils.data.DataLoader(\n",
        "    list(train_iter),\n",
        "    batch_size=64,\n",
        "    shuffle=True,\n",
        "    num_workers=2,\n",
        "    drop_last=True,\n",
        "    collate_fn=collate_fn\n",
        ")\n",
        "\n",
        "valid_loader = torch.utils.data.DataLoader(\n",
        "    list(valid_iter),\n",
        "    batch_size=32,\n",
        "    shuffle=True,\n",
        "    num_workers=2,\n",
        "    drop_last=True,\n",
        "    collate_fn=collate_fn\n",
        ")"
      ],
      "metadata": {
        "id": "FGx2bNQVQpF4"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Initialize ExpressNet 🤖"
      ],
      "metadata": {
        "id": "aQKe6-yEb0kD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = ExpressNet(\n",
        "    d_model=256,\n",
        "    vocab_size=len(vocab),\n",
        "    classification_type=\"multiclass\",\n",
        "    n_classes=4\n",
        ").to(device)\n",
        "\n",
        "criterion = torch.nn.CrossEntropyLoss()\n",
        "optimizer = torch.optim.Adam(model.parameters())"
      ],
      "metadata": {
        "id": "0kN3PkzYRYu8"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Run training and validation. See accuracy metric"
      ],
      "metadata": {
        "id": "L9veJBTqb6CO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def train(loader):\n",
        "  model.train()\n",
        "  total_loss = 0\n",
        "  total_accuracy = 0\n",
        "\n",
        "  for batch in tqdm(loader):\n",
        "    optimizer.zero_grad()\n",
        "    inputs, targets = batch[0].to(device), batch[1].to(device)\n",
        "\n",
        "    outputs = model(inputs)\n",
        "\n",
        "    loss = criterion(outputs, targets)\n",
        "    total_loss += loss.item()\n",
        "    accuracy = torch.sum((torch.argmax(outputs, dim=1) == targets).int()) / len(targets)\n",
        "    total_accuracy += accuracy\n",
        "\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "\n",
        "  return total_loss / len(loader), total_accuracy / len(loader)\n",
        "\n",
        "\n",
        "def evaluation(loader):\n",
        "  model.eval()\n",
        "  total_loss = 0\n",
        "  total_accuracy = 0\n",
        "\n",
        "  with torch.no_grad():\n",
        "    for batch in tqdm(loader):\n",
        "\n",
        "      inputs, targets = batch[0].to(device), batch[1].to(device)\n",
        "\n",
        "      outputs = model(inputs)\n",
        "\n",
        "      loss = criterion(outputs, targets)\n",
        "      total_loss += loss.item()\n",
        "      accuracy = torch.sum((torch.argmax(outputs, dim=1) == targets).int()) / len(targets)\n",
        "      total_accuracy += accuracy\n",
        "\n",
        "  return total_loss / len(loader), total_accuracy / len(loader)"
      ],
      "metadata": {
        "id": "KkO9KH5eRAeR"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "epochs = 3\n",
        "\n",
        "for epoch in range(epochs):\n",
        "  loss, acc = train(train_loader)\n",
        "  val_loss, val_acc = evaluation(valid_loader)\n",
        "\n",
        "  print(f\"Epoch: {epoch + 1} | loss: {loss:.3f} | acc: {acc:.3f} | val_loss: {val_loss:.3f} | val_acc: {val_acc:.3f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6uIlHN5lRRp4",
        "outputId": "8f43f0d6-1a8f-4856-b110-7a4165b7803a"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1875/1875 [02:23<00:00, 13.03it/s]\n",
            "100%|██████████| 237/237 [00:03<00:00, 65.32it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 1 | loss: 0.534 | acc: 0.801 | val_loss: 0.330 | val_acc: 0.889\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1875/1875 [02:28<00:00, 12.60it/s]\n",
            "100%|██████████| 237/237 [00:03<00:00, 65.21it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 2 | loss: 0.313 | acc: 0.892 | val_loss: 0.280 | val_acc: 0.908\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1875/1875 [02:28<00:00, 12.61it/s]\n",
            "100%|██████████| 237/237 [00:03<00:00, 65.89it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 3 | loss: 0.255 | acc: 0.911 | val_loss: 0.272 | val_acc: 0.912\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    }
  ]
}