generation_with_resampling_v2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a2fc0576-c246-44da-b3e6-e5086715a467",
   "metadata": {},
   "source": [
    "## In this version, sentence length is sampled from normal distribution."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "656a78f6-358f-4a18-9a5c-bb6d1cb3b72c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8826915ee42246c39532064d3c32bf2c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "from sentence_transformers import SentenceTransformer\n",
    "from sklearn.model_selection import train_test_split\n",
    "import torch\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "from huggingface_hub import login\n",
    "import datasets\n",
    "login()\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.metrics import silhouette_score\n",
    "\n",
    "from huggingface_hub.hf_api import HfFolder"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "528516b5-2b47-4116-8d0a-c985e8343d70",
   "metadata": {},
   "source": [
    "### Generating clusters for valid and blocked regions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9ce5105b-0cd8-459b-a5e3-580503a3f809",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4588"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent_essays = datasets.load_dataset(\"AnkushRaut216/test_essays\")\n",
    "sent_essays_pd = sent_essays['train'].to_pandas()\n",
    "sentences = sum([sent_tokenize(sent_essays_pd['text'][a]) for a in range(len(sent_essays_pd))], [])\n",
    "len(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ed297b03-93e8-414d-9336-2641cb6c577e",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_encoder = SentenceTransformer('/scratch/alpine/anra7539/contrastive_learning_model/best_model')\n",
    "sentence_embeddings = np.asarray([sentence_encoder.encode(a) for a in sentences])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a1340fab-a197-423b-a273-685011af18f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.05063439"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kclust = KMeans(n_clusters = 8, random_state = 2024)\n",
    "silhouette_score(sentence_embeddings, kclust.fit_predict(sentence_embeddings))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6a912b4b-2520-40b3-b3bb-d2005e339017",
   "metadata": {},
   "outputs": [],
   "source": [
    "name = \"meta-llama/Llama-2-7b-chat-hf\"\n",
    "access_token = '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' # Use a different access token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1ffd4b41-5d9c-408c-b240-a825b5797c3b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec7a42dd090e40af8564e9eaf0167931",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "device = \"cuda\"\n",
    "\n",
    "HfFolder.save_token(access_token)\n",
    "model = AutoModelForCausalLM.from_pretrained(name, \n",
    "                                             device_map = device,\n",
    "                                             cache_dir='/scratch/alpine/anra7539').to(device)\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(name, padding_side=\"left\")\n",
    "tokenizer.pad_token = tokenizer.eos_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7c2bb3ab-c256-41b7-8e34-4bd98bb144bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_sentences(text):\n",
    "    prompt = f\"Generate only one sentence following the given sentence:{text}\"\n",
    "    input_tokens = tokenizer(prompt, return_tensors = \"pt\").to(\"cuda\")\n",
    "\n",
    "    outputs = model.generate(**input_tokens)\n",
    "\n",
    "    return tokenizer.decode(outputs[0], skip_special_tokens = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca717028-b936-492f-8ea5-b76daf772344",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|██▉       | 11/38 [27:17<1:03:53, 141.99s/it]"
     ]
    }
   ],
   "source": [
    "watermarked_essays = []\n",
    "for essay in tqdm(sent_essays_pd.text[135:]):\n",
    "    sampled_essay_length = int(np.abs(np.random.normal(18,15)))\n",
    "    generated_essay = [sent_tokenize(essay)[0]]\n",
    "    cluster_assignment = [kclust.predict(sentence_encoder.encode(generated_essay[-1]).reshape(1,-1))]\n",
    "    for i in range(sampled_essay_length):\n",
    "        np.random.seed(cluster_assignment[-1])\n",
    "        valid_clusters = np.random.choice(range(10),2)\n",
    "        generated_sentence = generate_sentences(generated_essay[-1]).split(\"\\n\")[-1]\n",
    "        tries = 0\n",
    "        while (kclust.predict(sentence_encoder.encode(generated_sentence).reshape(1,-1)) not in valid_clusters) and (tries < 5):\n",
    "            generated_sentence = generate_sentences(generated_essay[-1]).split(\"\\n\")[-1]\n",
    "            tries+=1\n",
    "        generated_essay.append(generated_sentence)\n",
    "    watermarked_essays.append(generated_essay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c94d3e9-ff23-41c1-bcd2-26fb4bb73af6",
   "metadata": {},
   "outputs": [],
   "source": [
    "watermarked_essays_df = pd.DataFrame({\"text\":watermarked_essays})\n",
    "watermarked_essays_df.to_csv(\"/projects/anra7539/projects/nlp_project/watermarked_essays_v2/watermarked_essays_135+.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "365e32c4-37f0-455d-a252-16a70400c7d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# filenames = [ 'watermarked_essays_v2/watermarked_essays_15.csv', 'watermarked_essays_v2/watermarked_essays_15_30.csv', \n",
    "#              'watermarked_essays_v2/watermarked_essays_30_45.csv', 'watermarked_essays_v2/watermarked_essays_45_90.csv',\n",
    "#              'watermarked_essays_v2/watermarked_essays_90_135.csv', 'watermarked_essays_v2/watermarked_essays_135+.csv']\n",
    "\n",
    "# dfs = []\n",
    "# for file in filenames:\n",
    "#     dfs.append(pd.read_csv(file))\n",
    "\n",
    "# watermarked_essays = pd.concat(dfs, ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "287b84c6-913e-4257-925d-abc98fc17a38",
   "metadata": {},
   "outputs": [],
   "source": [
    "# watermarked_essays.to_csv(\"all_watermarked_essays_v2.csv\", index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}