03_chatglm_feats.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bcad3cf0-102e-40cc-b18a-ba97b865aa1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import warnings\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import StratifiedKFold,GroupKFold\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from tqdm import tqdm\n",
    "import gc\n",
    "import re\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import os\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "import polars as pl\n",
    "from pathlib import Path\n",
    "from glob import glob\n",
    "import json\n",
    "import joblib\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5578928a-4815-4afa-ac90-b8fec06e85d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.makedirs('abstract')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90287b4c-5f49-44cd-ad21-5ba1c9afdb35",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards: 100%|██████████| 7/7 [00:04<00:00,  1.56it/s]\n",
      "Setting eos_token is not supported, use the default one.\n",
      "Setting pad_token is not supported, use the default one.\n",
      "Setting unk_token is not supported, use the default one.\n",
      "  0%|          | 0/2957 [00:00<?, ?it/s]Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n",
      "100%|██████████| 2957/2957 [01:25<00:00, 34.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 3000\n",
      "1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards: 100%|██████████| 7/7 [00:03<00:00,  1.96it/s]\n",
      "Setting eos_token is not supported, use the default one.\n",
      "Setting pad_token is not supported, use the default one.\n",
      "Setting unk_token is not supported, use the default one.\n",
      "0it [00:00, ?it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3000 6000\n",
      "2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards:  71%|███████▏  | 5/7 [00:02<00:00,  2.07it/s]\n",
      "Traceback (most recent call last):\n",
      "  File \"chatglm_abstract.py\", line 83, in <module>\n",
      "    model = AutoModel.from_pretrained(\"THUDM/chatglm3-6b\",trust_remote_code=True)\n",
      "  File \"/root/miniconda3/lib/python3.8/site-packages/transformers/models/auto/auto_factory.py\", line 556, in from_pretrained\n",
      "    return model_class.from_pretrained(\n",
      "  File \"/root/miniconda3/lib/python3.8/site-packages/transformers/modeling_utils.py\", line 3502, in from_pretrained\n",
      "    ) = cls._load_pretrained_model(\n",
      "  File \"/root/miniconda3/lib/python3.8/site-packages/transformers/modeling_utils.py\", line 3948, in _load_pretrained_model\n",
      "    del state_dict\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards:  86%|████████▌ | 6/7 [00:02<00:00,  2.19it/s]\n",
      "Traceback (most recent call last):\n",
      "  File \"chatglm_abstract.py\", line 83, in <module>\n",
      "    model = AutoModel.from_pretrained(\"THUDM/chatglm3-6b\",trust_remote_code=True)\n",
      "  File \"/root/miniconda3/lib/python3.8/site-packages/transformers/models/auto/auto_factory.py\", line 556, in from_pretrained\n",
      "    return model_class.from_pretrained(\n",
      "  File \"/root/miniconda3/lib/python3.8/site-packages/transformers/modeling_utils.py\", line 3502, in from_pretrained\n",
      "    ) = cls._load_pretrained_model(\n",
      "  File \"/root/miniconda3/lib/python3.8/site-packages/transformers/modeling_utils.py\", line 3949, in _load_pretrained_model\n",
      "    gc.collect()\n",
      "KeyboardInterrupt\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "for i in range(106):\n",
    "    os.system(f'python3 chatglm_abstract.py {i}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "68044afd-b196-4499-bea2-024679c17c79",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 2/106 [00:00<00:07, 14.01it/s]\n"
     ]
    },
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'abstract/t_dict_abstract_2.pkl'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [2]\u001b[0m, in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m t_dict \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mdict\u001b[39m()\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m tqdm(\u001b[38;5;28mrange\u001b[39m(\u001b[38;5;241m106\u001b[39m)):\n\u001b[0;32m----> 3\u001b[0m     a \u001b[38;5;241m=\u001b[39m \u001b[43mjoblib\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mabstract/t_dict_abstract_\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mi\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m.pkl\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      4\u001b[0m     t_dict\u001b[38;5;241m.\u001b[39mupdate(a)\n",
      "File \u001b[0;32m~/miniconda3/lib/python3.8/site-packages/joblib/numpy_pickle.py:579\u001b[0m, in \u001b[0;36mload\u001b[0;34m(filename, mmap_mode)\u001b[0m\n\u001b[1;32m    577\u001b[0m         obj \u001b[38;5;241m=\u001b[39m _unpickle(fobj)\n\u001b[1;32m    578\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 579\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mfilename\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mrb\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[1;32m    580\u001b[0m         \u001b[38;5;28;01mwith\u001b[39;00m _read_fileobject(f, filename, mmap_mode) \u001b[38;5;28;01mas\u001b[39;00m fobj:\n\u001b[1;32m    581\u001b[0m             \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(fobj, \u001b[38;5;28mstr\u001b[39m):\n\u001b[1;32m    582\u001b[0m                 \u001b[38;5;66;03m# if the returned file object is a string, this means we\u001b[39;00m\n\u001b[1;32m    583\u001b[0m                 \u001b[38;5;66;03m# try to load a pickle file generated with an version of\u001b[39;00m\n\u001b[1;32m    584\u001b[0m                 \u001b[38;5;66;03m# Joblib so we load it with joblib compatibility function.\u001b[39;00m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'abstract/t_dict_abstract_2.pkl'"
     ]
    }
   ],
   "source": [
    "t_dict = dict()\n",
    "for i in tqdm(range(106)):\n",
    "    a = joblib.load(f'abstract/t_dict_abstract_{i}.pkl')\n",
    "    t_dict.update(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "11c6115d-4979-43f8-8478-2b4f1197aafd",
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf abastract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "18b52ebc-9377-4c0d-9494-d41a7aa8dc94",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['t_dict_abstract_all.pkl']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(t_dict,'t_dict_abstract_all.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5ea874be-c0c6-4e0d-bfe0-3c58b38e5faa",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "train = pd.read_feather('data/train.feather')\n",
    "valid = pd.read_feather('data/valid.feather')\n",
    "test = pd.read_feather('data/test.feather')\n",
    "\n",
    "piddf = joblib.load('data/pid_df.pkl')\n",
    "\n",
    "data = pd.concat([train,valid,test]).reset_index(drop = True)\n",
    "piddf = piddf[piddf['id'].isin(set(data['PID']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "80cddd5a-1c46-4367-90f5-0b6a765f2e81",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 15/15 [00:35<00:00,  2.37s/it]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def cos_similarity(target, embedding):\n",
    "    numerator = np.sum(target * embedding, axis=1)\n",
    "    denominator = np.sqrt(np.sum(np.square(target)) * np.sum(np.square(embedding),axis=1))\n",
    "    return numerator / denominator\n",
    "\n",
    "ans = []\n",
    "for autherID in tqdm(data['autherID'].unique()):\n",
    "    pid_list = data[data['autherID'] == autherID]['PID'].to_list()\n",
    "    for f1 in pid_list:\n",
    "        x1 = t_dict.get(f1)\n",
    "        x2 = [f for f in pid_list if f != f1]\n",
    "        x2 = [t_dict.get(f) for f in x2]\n",
    "        x1 = np.array(x1 , dtype = np.float32)\n",
    "        x2 = np.array(x2 , dtype = np.float32)\n",
    "\n",
    "        ans.append(list(cos_similarity(x1, x2)))\n",
    "\n",
    "    \n",
    "data['chatglm_pid_abstract_sim'] = ans\n",
    "\n",
    "for f in ['chatglm_pid_abstract_sim']:\n",
    "    data[f + '_mean'] = data[f].apply(lambda x:np.mean(x))\n",
    "    data[f + '_max'] = data[f].apply(lambda x:np.max(x))\n",
    "    data[f + '_min'] = data[f].apply(lambda x:np.min(x))\n",
    "    data[f + '_std'] = data[f].apply(lambda x:np.std(x))\n",
    "    data[f + '_median'] = data[f].apply(lambda x:np.median(x))\n",
    "    \n",
    "data = data.drop(['chatglm_pid_abstract_sim'],axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d10167fa-4e8f-48bb-bb33-612b86bdd8a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "data[['PID',  'autherID',  'chatglm_pid_abstract_sim_mean', 'chatglm_pid_abstract_sim_max',\n",
    "       'chatglm_pid_abstract_sim_min', 'chatglm_pid_abstract_sim_std',\n",
    "       'chatglm_pid_abstract_sim_median']].to_feather('chatglm_pid_abstract_sim.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "28a9204d-130f-406d-a853-d67211a19da9",
   "metadata": {},
   "outputs": [],
   "source": [
    "t_dict = joblib.load(f't_dict_abstract_all.pkl')\n",
    "abstract_emb = pd.DataFrame(t_dict.values()).values\n",
    "svd_tmp = TruncatedSVD(n_components=16, n_iter=20, random_state=2024)\n",
    "abstract_emb_svd = svd_tmp.fit_transform(abstract_emb)\n",
    "abstract_emb_svd = pd.DataFrame(abstract_emb_svd,columns = [f'chatglm_abstract_emb_svd_{i}' for i in range(16)])\n",
    "abstract_emb_svd = pd.concat([pd.DataFrame(t_dict.keys(),columns = ['PID']),abstract_emb_svd],axis = 1)\n",
    "data = data.merge(abstract_emb_svd,on = ['PID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2e53b88e-de33-4ded-8a2a-d101a0026c1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "data[['PID', 'autherID', 'chatglm_abstract_emb_svd_0',\n",
    "       'chatglm_abstract_emb_svd_1', 'chatglm_abstract_emb_svd_2',\n",
    "       'chatglm_abstract_emb_svd_3', 'chatglm_abstract_emb_svd_4',\n",
    "       'chatglm_abstract_emb_svd_5', 'chatglm_abstract_emb_svd_6',\n",
    "       'chatglm_abstract_emb_svd_7', 'chatglm_abstract_emb_svd_8',\n",
    "       'chatglm_abstract_emb_svd_9', 'chatglm_abstract_emb_svd_10',\n",
    "       'chatglm_abstract_emb_svd_11', 'chatglm_abstract_emb_svd_12',\n",
    "       'chatglm_abstract_emb_svd_13', 'chatglm_abstract_emb_svd_14',\n",
    "       'chatglm_abstract_emb_svd_15']].to_feather('chatglm_pid_abstract_svd.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1fa9aaef-68fc-4e8c-890e-2337f098dc27",
   "metadata": {},
   "outputs": [
    {
     "ename": "FileExistsError",
     "evalue": "[Errno 17] File exists: 'title'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileExistsError\u001b[0m                           Traceback (most recent call last)",
      "Input \u001b[0;32mIn [2]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mos\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmakedirs\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mtitle\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/miniconda3/lib/python3.8/os.py:223\u001b[0m, in \u001b[0;36mmakedirs\u001b[0;34m(name, mode, exist_ok)\u001b[0m\n\u001b[1;32m    221\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[1;32m    222\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 223\u001b[0m     \u001b[43mmkdir\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    224\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m:\n\u001b[1;32m    225\u001b[0m     \u001b[38;5;66;03m# Cannot rely on checking for EEXIST, since the operating system\u001b[39;00m\n\u001b[1;32m    226\u001b[0m     \u001b[38;5;66;03m# could give priority to other errors like EACCES or EROFS\u001b[39;00m\n\u001b[1;32m    227\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m exist_ok \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m path\u001b[38;5;241m.\u001b[39misdir(name):\n",
      "\u001b[0;31mFileExistsError\u001b[0m: [Errno 17] File exists: 'title'"
     ]
    }
   ],
   "source": [
    "os.makedirs('title')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9101772f-6a97-44e6-8bfc-1ba0a45a8032",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards: 100%|██████████| 7/7 [00:02<00:00,  2.58it/s]\n",
      "Setting eos_token is not supported, use the default one.\n",
      "Setting pad_token is not supported, use the default one.\n",
      "Setting unk_token is not supported, use the default one.\n",
      "  0%|          | 0/2957 [00:00<?, ?it/s]Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n",
      "100%|██████████| 2957/2957 [00:55<00:00, 53.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 3000\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "for i in range(106):\n",
    "    os.system(f'python3 chatglm_title.py {i}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cadb18a5-5b44-4632-bfd3-9a7a4253015e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00,  5.40it/s]\n"
     ]
    }
   ],
   "source": [
    "t_dict = dict()\n",
    "for i in tqdm(range(106)):\n",
    "    a = joblib.load(f'title/t_dict_title_{i}.pkl')\n",
    "    t_dict.update(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "35c1c8cd-8daf-4b32-88f4-d0d2a2d31d10",
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "99700622-b596-4fbf-9666-41cfcc9d716f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['t_dict_title_all.pkl']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(t_dict,'t_dict_title_all.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c0620f52-3548-45f8-9d93-bf5c150f153b",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "train = pd.read_feather('data/train.feather')\n",
    "valid = pd.read_feather('data/valid.feather')\n",
    "test = pd.read_feather('data/test.feather')\n",
    "\n",
    "piddf = joblib.load('data/pid_df.pkl')\n",
    "\n",
    "data = pd.concat([train,valid,test]).reset_index(drop = True)\n",
    "piddf = piddf[piddf['id'].isin(set(data['PID']))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d7f0f856-b2ab-4816-8752-1a402b76cceb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 15/15 [00:38<00:00,  2.58s/it]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def cos_similarity(target, embedding):\n",
    "    numerator = np.sum(target * embedding, axis=1)\n",
    "    denominator = np.sqrt(np.sum(np.square(target)) * np.sum(np.square(embedding),axis=1))\n",
    "    return numerator / denominator\n",
    "\n",
    "ans = []\n",
    "for autherID in tqdm(data['autherID'].unique()):\n",
    "    pid_list = data[data['autherID'] == autherID]['PID'].to_list()\n",
    "    for f1 in pid_list:\n",
    "        x1 = t_dict.get(f1)\n",
    "        x2 = [f for f in pid_list if f != f1]\n",
    "        x2 = [t_dict.get(f) for f in x2]\n",
    "        x1 = np.array(x1 , dtype = np.float32)\n",
    "        x2 = np.array(x2 , dtype = np.float32)\n",
    "\n",
    "        ans.append(list(cos_similarity(x1, x2)))\n",
    "\n",
    "    \n",
    "data['chatglm_pid_title_sim'] = ans\n",
    "\n",
    "for f in ['chatglm_pid_title_sim']:\n",
    "    data[f + '_mean'] = data[f].apply(lambda x:np.mean(x))\n",
    "    data[f + '_max'] = data[f].apply(lambda x:np.max(x))\n",
    "    data[f + '_min'] = data[f].apply(lambda x:np.min(x))\n",
    "    data[f + '_std'] = data[f].apply(lambda x:np.std(x))\n",
    "    data[f + '_median'] = data[f].apply(lambda x:np.median(x))\n",
    "    \n",
    "data = data.drop(['chatglm_pid_title_sim'],axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9e428277-1f82-476b-9fd2-621c510c1a2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "data[['PID',  'autherID',  'chatglm_pid_title_sim_mean', 'chatglm_pid_title_sim_max',\n",
    "       'chatglm_pid_title_sim_min', 'chatglm_pid_title_sim_std',\n",
    "       'chatglm_pid_title_sim_median']].to_feather('chatglm_pid_title_sim.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4c3268d3-2fae-4636-a722-c23427d2fb26",
   "metadata": {},
   "outputs": [],
   "source": [
    "t_dict = joblib.load(f't_dict_title_all.pkl')\n",
    "abstract_emb = pd.DataFrame(t_dict.values()).values\n",
    "svd_tmp = TruncatedSVD(n_components=16, n_iter=20, random_state=2024)\n",
    "abstract_emb_svd = svd_tmp.fit_transform(abstract_emb)\n",
    "abstract_emb_svd = pd.DataFrame(abstract_emb_svd,columns = [f'chatglm_title_emb_svd_{i}' for i in range(16)])\n",
    "abstract_emb_svd = pd.concat([pd.DataFrame(t_dict.keys(),columns = ['PID']),abstract_emb_svd],axis = 1)\n",
    "data = data.merge(abstract_emb_svd,on = ['PID'],how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f54a6179-5b27-4ecb-9b56-e13d6608f2f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['PID', 'label', 'autherID', 'autherName', 'chatglm_pid_title_sim_mean',\n",
       "       'chatglm_pid_title_sim_max', 'chatglm_pid_title_sim_min',\n",
       "       'chatglm_pid_title_sim_std', 'chatglm_pid_title_sim_median',\n",
       "       'chatglm_title_emb_svd_0', 'chatglm_title_emb_svd_1',\n",
       "       'chatglm_title_emb_svd_2', 'chatglm_title_emb_svd_3',\n",
       "       'chatglm_title_emb_svd_4', 'chatglm_title_emb_svd_5',\n",
       "       'chatglm_title_emb_svd_6', 'chatglm_title_emb_svd_7',\n",
       "       'chatglm_title_emb_svd_8', 'chatglm_title_emb_svd_9',\n",
       "       'chatglm_title_emb_svd_10', 'chatglm_title_emb_svd_11',\n",
       "       'chatglm_title_emb_svd_12', 'chatglm_title_emb_svd_13',\n",
       "       'chatglm_title_emb_svd_14', 'chatglm_title_emb_svd_15'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "bab20341-5b30-4f5c-bfb2-5edd6e9c3907",
   "metadata": {},
   "outputs": [],
   "source": [
    "data[['PID', 'autherID','chatglm_title_emb_svd_0', 'chatglm_title_emb_svd_1',\n",
    "       'chatglm_title_emb_svd_2', 'chatglm_title_emb_svd_3',\n",
    "       'chatglm_title_emb_svd_4', 'chatglm_title_emb_svd_5',\n",
    "       'chatglm_title_emb_svd_6', 'chatglm_title_emb_svd_7',\n",
    "       'chatglm_title_emb_svd_8', 'chatglm_title_emb_svd_9',\n",
    "       'chatglm_title_emb_svd_10', 'chatglm_title_emb_svd_11',\n",
    "       'chatglm_title_emb_svd_12', 'chatglm_title_emb_svd_13',\n",
    "       'chatglm_title_emb_svd_14', 'chatglm_title_emb_svd_15']].to_feather('chatglm_pid_title_svd.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "953a33f3-f8a5-48a0-9a42-22fea89768b6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}