pseudolabeling-training-single-label.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "3cbaf220",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import json\n",
    "import random\n",
    "import torch\n",
    "import numpy as np\n",
    "from transformers import AutoTokenizer, AutoConfig, AutoModel\n",
    "from classifier import MistralForSequenceClassification\n",
    "import os\n",
    "\n",
    "# os.environ['CUDA_VISIBLE_DEVICES'] = '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "83fff176",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "texts, labels = [], []\n",
    "texts_, labels_ = [],[]\n",
    "\n",
    "\n",
    "with open('sfw-dataset.jsonl') as fopen:\n",
    "    \n",
    "    for x in fopen:\n",
    "        data.append(json.loads(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "c0ce88cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "porn = []\n",
    "\n",
    "with open('porn-sfw-dataset.jsonl') as fopen:\n",
    "    \n",
    "    for x in fopen:\n",
    "        texts.append(json.loads(x)['Content'])\n",
    "        labels.append(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "4d1e1ca7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(571, 104889)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "labels_sentiment = ['racist', 'religion insult','psychiatric or mental illness', 'sexist','harassment','informative','safe for work','hate']\n",
    "\n",
    "for l in data:\n",
    "    try:\n",
    "        if eval(l['output'])['label'] != 'porn':\n",
    "            texts_.append(l['left'])\n",
    "            labels_.append(0)\n",
    "        else:\n",
    "            continue\n",
    "    except SyntaxError as e:\n",
    "        if eval(l['output']+ '}')['label'] != 'porn':\n",
    "            texts_.append(l['left'])\n",
    "            labels_.append(0)\n",
    "        elif eval(l['output']+ '}')['label'] == 'porn':\n",
    "            continue\n",
    "    except Exception as e: \n",
    "        print(e)\n",
    "        continue\n",
    "len(texts),len(texts_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "aecd2d49",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1142"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random.shuffle(texts_)\n",
    "sample_non_sexists = random.sample(texts_, len(texts)*2)\n",
    "label_non_sexists = labels_[:len(sample_non_sexists)]\n",
    "\n",
    "len(sample_non_sexists)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "f03359b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "texts.extend(sample_non_sexists)\n",
    "labels.extend(label_non_sexists)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "8061b12f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1713"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "cfe78473",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_X, test_X, train_Y, test_Y = train_test_split(\n",
    "    texts, labels, test_size = 0.1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "9f891192",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1]), array([1142,  571]))"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(labels, return_counts = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "ce2396bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "id2label = {'0':'non porn','1':'porn'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "0fd65cb9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/.local/lib/python3.10/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "config = AutoConfig.from_pretrained('mesolitica/malaysian-mistral-191M-MLM-512')\n",
    "config.num_labels = len(set(labels))\n",
    "config.vocab = ['non porn', 'porn']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "2f0a2a9e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of MistralForSequenceClassification were not initialized from the model checkpoint at mesolitica/malaysian-mistral-191M-MLM-512 and are newly initialized: ['score.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = MistralForSequenceClassification.from_pretrained('mesolitica/malaysian-mistral-191M-MLM-512', config = config)\n",
    "_ = model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "c6604d3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained('mesolitica/malaysian-mistral-191M-MLM-512')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "41f5e2ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainable_parameters = [param for param in model.parameters() if param.requires_grad]\n",
    "trainer = torch.optim.AdamW(trainable_parameters, lr = 2e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "462dd7dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [00:16<00:00,  4.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, loss: 0.22593505177885675, dev_predicted: 0.9462962962962962\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [00:16<00:00,  4.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1, loss: 0.02160757591068082, dev_predicted: 0.9555555555555555\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [00:16<00:00,  4.84it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2, loss: 0.009253401805101301, dev_predicted: 0.9351851851851851\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [00:16<00:00,  4.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 3, loss: 0.0006005912231534566, dev_predicted: 0.9296296296296295\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [00:16<00:00,  4.82it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 4, loss: 5.9047970773043256e-05, dev_predicted: 0.9296296296296295\n"
     ]
    }
   ],
   "source": [
    "batch_size = 20\n",
    "epoch = 100\n",
    "\n",
    "\n",
    "best_dev_acc = -np.inf\n",
    "patient = 3\n",
    "current_patient = 0\n",
    "\n",
    "for e in range(epoch):\n",
    "    pbar = tqdm(range(0, len(train_X), batch_size))\n",
    "    losses = []\n",
    "    for i in pbar:\n",
    "        trainer.zero_grad()\n",
    "        x = train_X[i: i + batch_size]\n",
    "        y = np.array(train_Y[i: i + batch_size])\n",
    "        \n",
    "        padded = tokenizer(x, padding = 'longest', return_tensors = 'pt')\n",
    "\n",
    "        padded['labels'] = torch.from_numpy(y)\n",
    "        for k in padded.keys():\n",
    "            padded[k] = padded[k].cuda()\n",
    "        \n",
    "        padded.pop('token_type_ids', None)\n",
    "\n",
    "        loss, pred = model(**padded, return_dict = False)\n",
    "        loss.backward()\n",
    "        \n",
    "        grad_norm = torch.nn.utils.clip_grad_norm_(trainable_parameters, 5.0)\n",
    "        trainer.step()\n",
    "        losses.append(float(loss))\n",
    "        \n",
    "    dev_predicted = []\n",
    "    for i in range(0, len(test_X), batch_size):\n",
    "        x = test_X[i: i + batch_size]\n",
    "        y = np.array(test_Y[i: i + batch_size])\n",
    "        padded = tokenizer(x, padding = 'longest', return_tensors = 'pt')\n",
    "        padded['labels'] = torch.from_numpy(y)\n",
    "        for k in padded.keys():\n",
    "            padded[k] = padded[k].cuda()\n",
    "        \n",
    "        padded.pop('token_type_ids', None)\n",
    "\n",
    "        loss, pred = model(**padded, return_dict = False)\n",
    "        dev_predicted.append((pred.argmax(axis = 1).detach().cpu().numpy() == y).mean())\n",
    "        \n",
    "    dev_predicted = np.mean(dev_predicted)\n",
    "    \n",
    "    print(f'epoch: {e}, loss: {np.mean(losses)}, dev_predicted: {dev_predicted}')\n",
    "    \n",
    "    if dev_predicted >= best_dev_acc:\n",
    "        best_dev_acc = dev_predicted\n",
    "        current_patient = 0\n",
    "        model.save_pretrained('porn-mistral-mlm')\n",
    "    else:\n",
    "        current_patient += 1\n",
    "    \n",
    "    if current_patient >= patient:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "d182b765",
   "metadata": {},
   "outputs": [],
   "source": [
    "padded = tokenizer(texts[:10], padding = True, return_tensors = 'pt')\n",
    "for k in padded.keys():\n",
    "    padded[k] = padded[k].cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "549e5416",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9/9 [00:00<00:00, 14.74it/s]\n"
     ]
    }
   ],
   "source": [
    "batch_size = 20\n",
    "epoch = 100\n",
    "\n",
    "real_Y = []\n",
    "for i in tqdm(range(0, len(test_X), batch_size)):\n",
    "    x = test_X[i: i + batch_size]\n",
    "    y = np.array(test_Y[i: i + batch_size])\n",
    "    padded = tokenizer(x, padding = 'longest', return_tensors = 'pt')\n",
    "    padded['labels'] = torch.from_numpy(y)\n",
    "    for k in padded.keys():\n",
    "        padded[k] = padded[k].cuda()\n",
    "    padded.pop('token_type_ids', None)\n",
    "\n",
    "    loss, pred = model(**padded,return_dict=False)\n",
    "    real_Y.extend(pred.argmax(axis = 1).detach().cpu().numpy().tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "111e7288",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "    non porn    0.93023   0.97561   0.95238       123\n",
      "        porn    0.93023   0.81633   0.86957        49\n",
      "\n",
      "    accuracy                        0.93023       172\n",
      "   macro avg    0.93023   0.89597   0.91097       172\n",
      "weighted avg    0.93023   0.93023   0.92879       172\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "\n",
    "print(\n",
    "    metrics.classification_report(\n",
    "        real_Y, test_Y, target_names = config.vocab,\n",
    "        digits = 5\n",
    "    )\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}