preproc_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, shutil\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split documents with coordinates into sets\n",
    "\n",
    "## Load PMIDs from articles with coordinates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Documents with Coordinates 18155\n"
     ]
    }
   ],
   "source": [
    "meta = pd.read_csv(\"data/metadata.csv\", encoding=\"latin1\")\n",
    "coord_pmids = list(meta[\"PMID\"].dropna().astype(int).values)\n",
    "print(\"{:26s} {}\".format(\"Documents with Coordinates\", len(coord_pmids)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Shuffle the PMIDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "coord_pmids = np.random.choice(coord_pmids, size=len(coord_pmids), replace=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split randomly into train, dev, and test sets\n",
    "\n",
    "Train: 90%, Dev: 5%, Test: 5%"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train  16339\n",
      "Dev    908\n",
      "Test   908\n",
      "Total  18155\n"
     ]
    }
   ],
   "source": [
    "n_docs = len(coord_pmids)\n",
    "splits = {\"train\": coord_pmids[:int(n_docs*0.9)], \n",
    "          \"dev\": coord_pmids[int(n_docs*0.9):int(n_docs*0.95)],\n",
    "          \"test\": coord_pmids[int(n_docs*0.95):]}\n",
    "for split, split_ids in splits.items():\n",
    "    print(\"{:7s}{}\".format(split.title(), len(split_ids)))\n",
    "print(\"{:7s}{}\".format(\"Total\", len(splits[\"train\"])+len(splits[\"dev\"])+len(splits[\"test\"])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export the PMID lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "for split, split_ids in splits.items():\n",
    "    split_ids = list(split_ids.astype(str))\n",
    "    with open(\"data/splits/{}.txt\".format(split), \"w+\") as outfile:\n",
    "        outfile.write(\"\\n\".join(split_ids))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Aggregate the corpus\n",
    "\n",
    "## Load PMIDs from PubMed search for neuroimaging articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Documents from PubMed      30132\n"
     ]
    }
   ],
   "source": [
    "pub_pmids = [int(pmid.strip()) for pmid in open(\"data/pubmed_pmids.txt\", \"r\").readlines()]\n",
    "print(\"{:26s} {}\".format(\"Documents from PubMed\", len(pub_pmids)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Copy over available full texts to corpus directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "in_dir = \"/Users/ehbeam/Dropbox/Stanford/Research/Projects/Thesis/program/nlp/corpus\"\n",
    "for pmid in set(list(coord_pmids) + pub_pmids):\n",
    "    in_file = \"{}/{}.txt\".format(in_dir, pmid)\n",
    "    out_file = \"data/text/corpus/{}.txt\".format(pmid)\n",
    "    if os.path.isfile(in_file) and not os.path.isfile(out_file):\n",
    "        shutil.copy(in_file, out_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Documents in Corpus        29828\n"
     ]
    }
   ],
   "source": [
    "corpus_files = [file for file in os.listdir(\"data/text/corpus\") if not file.startswith(\".\")]\n",
    "print(\"{:26s} {}\".format(\"Documents in Corpus\", len(corpus_files)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Specify the lexicon\n",
    "\n",
    "Find the intersection of terms in the VSM and DTM models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vector space model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Embedding Dimension  100\n",
      "Terms in VSM         350543\n"
     ]
    }
   ],
   "source": [
    "vsm = pd.read_csv(\"data/text/glove_gen_n100_win15_min5_iter500_190428.txt\", \n",
    "                  sep = \" \", index_col=0, header=0)\n",
    "n_vocab = vsm.shape[0]\n",
    "n_emb = vsm.shape[1]\n",
    "print(\"{:21s}{}\".format(\"Embedding Dimension\", n_emb))\n",
    "print(\"{:21s}{}\".format(\"Terms in VSM\", n_vocab))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Document-term matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utilities import load_dtm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Documents            18155\n",
      "Terms in DTM         1683\n"
     ]
    }
   ],
   "source": [
    "dtm_bin = load_dtm()\n",
    "n_terms = dtm_bin.shape[1]\n",
    "print(\"{:21s}{}\".format(\"Documents\", dtm_bin.shape[0]))\n",
    "print(\"{:21s}{}\".format(\"Terms in DTM\", n_terms))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Lexicon of mental function terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Terms in Lexicon     1544\n"
     ]
    }
   ],
   "source": [
    "lexicon = list(dtm_bin.columns.intersection(vsm.index))\n",
    "lexicon += [\"<eos>\", \"<unk>\"]\n",
    "with open(\"data/text/lexicon.txt\", \"w+\") as file:\n",
    "    file.write(\"\\n\".join(lexicon))\n",
    "print(\"{:21s}{}\".format(\"Terms in Lexicon\", len(lexicon)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compile the LSTM corpora\n",
    "\n",
    "**Training set:**  (1) Training set of documents with coordinates, (2) Documents from PubMed without coordinates.\n",
    "\n",
    "**Dev and test sets:**  Splits from above consisting of documents with coordinates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Documents in LSTM Train Set  28012\n"
     ]
    }
   ],
   "source": [
    "pm_only = set(pub_pmids).difference(set(coord_pmids))\n",
    "texts_avail = [int(file.replace(\".txt\", \"\")) for file in corpus_files]\n",
    "lstm_set = set(splits[\"train\"]).union(pm_only).intersection(texts_avail)\n",
    "print(\"{:28s} {}\".format(\"Documents in LSTM Train Set\", len(lstm_set)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "lstm_splits = splits\n",
    "lstm_splits[\"train\"] = lstm_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/text/lstm_train_pmids.txt\", \"w+\") as fout:\n",
    "    for pmid in lstm_splits[\"train\"]:\n",
    "        fout.write(str(pmid) + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing train set\n",
      "   Processing 0th document\n",
      "   Processing 100th document\n",
      "   Processing 200th document\n",
      "   Processing 300th document\n",
      "   Processing 400th document\n",
      "   Processing 500th document\n",
      "   Processing 600th document\n",
      "   Processing 700th document\n",
      "   Processing 800th document\n",
      "   Processing 900th document\n",
      "   Processing 1000th document\n",
      "   Processing 1100th document\n",
      "   Processing 1200th document\n",
      "   Processing 1300th document\n",
      "   Processing 1400th document\n",
      "   Processing 1500th document\n",
      "   Processing 1600th document\n",
      "   Processing 1700th document\n",
      "   Processing 1800th document\n",
      "   Processing 1900th document\n",
      "   Processing 2000th document\n",
      "   Processing 2100th document\n",
      "   Processing 2200th document\n",
      "   Processing 2300th document\n",
      "   Processing 2400th document\n",
      "   Processing 2500th document\n",
      "   Processing 2600th document\n",
      "   Processing 2700th document\n",
      "   Processing 2800th document\n",
      "   Processing 2900th document\n",
      "   Processing 3000th document\n",
      "   Processing 3100th document\n",
      "   Processing 3200th document\n",
      "   Processing 3300th document\n",
      "   Processing 3400th document\n",
      "   Processing 3500th document\n",
      "   Processing 3600th document\n",
      "   Processing 3700th document\n",
      "   Processing 3800th document\n",
      "   Processing 3900th document\n",
      "   Processing 4000th document\n",
      "   Processing 4100th document\n",
      "   Processing 4200th document\n",
      "   Processing 4300th document\n",
      "   Processing 4400th document\n",
      "   Processing 4500th document\n",
      "   Processing 4600th document\n",
      "   Processing 4700th document\n",
      "   Processing 4800th document\n",
      "   Processing 4900th document\n",
      "   Processing 5000th document\n",
      "   Processing 5100th document\n",
      "   Processing 5200th document\n",
      "   Processing 5300th document\n",
      "   Processing 5400th document\n",
      "   Processing 5500th document\n",
      "   Processing 5600th document\n",
      "   Processing 5700th document\n",
      "   Processing 5800th document\n",
      "   Processing 5900th document\n",
      "   Processing 6000th document\n",
      "   Processing 6100th document\n",
      "   Processing 6200th document\n",
      "   Processing 6300th document\n",
      "   Processing 6400th document\n",
      "   Processing 6500th document\n",
      "   Processing 6600th document\n",
      "   Processing 6700th document\n",
      "   Processing 6800th document\n",
      "   Processing 6900th document\n",
      "   Processing 7000th document\n",
      "   Processing 7100th document\n",
      "   Processing 7200th document\n",
      "   Processing 7300th document\n",
      "   Processing 7400th document\n",
      "   Processing 7500th document\n",
      "   Processing 7600th document\n",
      "   Processing 7700th document\n",
      "   Processing 7800th document\n",
      "   Processing 7900th document\n",
      "   Processing 8000th document\n",
      "   Processing 8100th document\n",
      "   Processing 8200th document\n",
      "   Processing 8300th document\n",
      "   Processing 8400th document\n",
      "   Processing 8500th document\n",
      "   Processing 8600th document\n",
      "   Processing 8700th document\n",
      "   Processing 8800th document\n",
      "   Processing 8900th document\n",
      "   Processing 9000th document\n",
      "   Processing 9100th document\n",
      "   Processing 9200th document\n",
      "   Processing 9300th document\n",
      "   Processing 9400th document\n",
      "   Processing 9500th document\n",
      "   Processing 9600th document\n",
      "   Processing 9700th document\n",
      "   Processing 9800th document\n",
      "   Processing 9900th document\n",
      "   Processing 10000th document\n",
      "   Processing 10100th document\n",
      "   Processing 10200th document\n",
      "   Processing 10300th document\n",
      "   Processing 10400th document\n",
      "   Processing 10500th document\n",
      "   Processing 10600th document\n",
      "   Processing 10700th document\n",
      "   Processing 10800th document\n",
      "   Processing 10900th document\n",
      "   Processing 11000th document\n",
      "   Processing 11100th document\n",
      "   Processing 11200th document\n",
      "   Processing 11300th document\n",
      "   Processing 11400th document\n",
      "   Processing 11500th document\n",
      "   Processing 11600th document\n",
      "   Processing 11700th document\n",
      "   Processing 11800th document\n",
      "   Processing 11900th document\n",
      "   Processing 12000th document\n",
      "   Processing 12100th document\n",
      "   Processing 12200th document\n",
      "   Processing 12300th document\n",
      "   Processing 12400th document\n",
      "   Processing 12500th document\n",
      "   Processing 12600th document\n",
      "   Processing 12700th document\n",
      "   Processing 12800th document\n",
      "   Processing 12900th document\n",
      "   Processing 13000th document\n",
      "   Processing 13100th document\n",
      "   Processing 13200th document\n",
      "   Processing 13300th document\n",
      "   Processing 13400th document\n",
      "   Processing 13500th document\n",
      "   Processing 13600th document\n",
      "   Processing 13700th document\n",
      "   Processing 13800th document\n",
      "   Processing 13900th document\n",
      "   Processing 14000th document\n",
      "   Processing 14100th document\n",
      "   Processing 14200th document\n",
      "   Processing 14300th document\n",
      "   Processing 14400th document\n",
      "   Processing 14500th document\n",
      "   Processing 14600th document\n",
      "   Processing 14700th document\n",
      "   Processing 14800th document\n",
      "   Processing 14900th document\n",
      "   Processing 15000th document\n",
      "   Processing 15100th document\n",
      "   Processing 15200th document\n",
      "   Processing 15300th document\n",
      "   Processing 15400th document\n",
      "   Processing 15500th document\n",
      "   Processing 15600th document\n",
      "   Processing 15700th document\n",
      "   Processing 15800th document\n",
      "   Processing 15900th document\n",
      "   Processing 16000th document\n",
      "   Processing 16100th document\n",
      "   Processing 16200th document\n",
      "   Processing 16300th document\n",
      "   Processing 16400th document\n",
      "   Processing 16500th document\n",
      "   Processing 16600th document\n",
      "   Processing 16700th document\n",
      "   Processing 16800th document\n",
      "   Processing 16900th document\n",
      "   Processing 17000th document\n",
      "   Processing 17100th document\n",
      "   Processing 17200th document\n",
      "   Processing 17300th document\n",
      "   Processing 17400th document\n",
      "   Processing 17500th document\n",
      "   Processing 17600th document\n",
      "   Processing 17700th document\n",
      "   Processing 17800th document\n",
      "   Processing 17900th document\n",
      "   Processing 18000th document\n",
      "   Processing 18100th document\n",
      "   Processing 18200th document\n",
      "   Processing 18300th document\n",
      "   Processing 18400th document\n",
      "   Processing 18500th document\n",
      "   Processing 18600th document\n",
      "   Processing 18700th document\n",
      "   Processing 18800th document\n",
      "   Processing 18900th document\n",
      "   Processing 19000th document\n",
      "   Processing 19100th document\n",
      "   Processing 19200th document\n",
      "   Processing 19300th document\n",
      "   Processing 19400th document\n",
      "   Processing 19500th document\n",
      "   Processing 19600th document\n",
      "   Processing 19700th document\n",
      "   Processing 19800th document\n",
      "   Processing 19900th document\n",
      "   Processing 20000th document\n",
      "   Processing 20100th document\n",
      "   Processing 20200th document\n",
      "   Processing 20300th document\n",
      "   Processing 20400th document\n",
      "   Processing 20500th document\n",
      "   Processing 20600th document\n",
      "   Processing 20700th document\n",
      "   Processing 20800th document\n",
      "   Processing 20900th document\n",
      "   Processing 21000th document\n",
      "   Processing 21100th document\n",
      "   Processing 21200th document\n",
      "   Processing 21300th document\n",
      "   Processing 21400th document\n",
      "   Processing 21500th document\n",
      "   Processing 21600th document\n",
      "   Processing 21700th document\n",
      "   Processing 21800th document\n",
      "   Processing 21900th document\n",
      "   Processing 22000th document\n",
      "   Processing 22100th document\n",
      "   Processing 22200th document\n",
      "   Processing 22300th document\n",
      "   Processing 22400th document\n",
      "   Processing 22500th document\n",
      "   Processing 22600th document\n",
      "   Processing 22700th document\n",
      "   Processing 22800th document\n",
      "   Processing 22900th document\n",
      "   Processing 23000th document\n",
      "   Processing 23100th document\n",
      "   Processing 23200th document\n",
      "   Processing 23300th document\n",
      "   Processing 23400th document\n",
      "   Processing 23500th document\n",
      "   Processing 23600th document\n",
      "   Processing 23700th document\n",
      "   Processing 23800th document\n",
      "   Processing 23900th document\n",
      "   Processing 24000th document\n",
      "   Processing 24100th document\n",
      "   Processing 24200th document\n",
      "   Processing 24300th document\n",
      "   Processing 24400th document\n",
      "   Processing 24500th document\n",
      "   Processing 24600th document\n",
      "   Processing 24700th document\n",
      "   Processing 24800th document\n",
      "   Processing 24900th document\n",
      "   Processing 25000th document\n",
      "   Processing 25100th document\n",
      "   Processing 25200th document\n",
      "   Processing 25300th document\n",
      "   Processing 25400th document\n",
      "   Processing 25500th document\n",
      "   Processing 25600th document\n",
      "   Processing 25700th document\n",
      "   Processing 25800th document\n",
      "   Processing 25900th document\n",
      "   Processing 26000th document\n",
      "   Processing 26100th document\n",
      "   Processing 26200th document\n",
      "   Processing 26300th document\n",
      "   Processing 26400th document\n",
      "   Processing 26500th document\n",
      "   Processing 26600th document\n",
      "   Processing 26700th document\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Processing 26800th document\n",
      "   Processing 26900th document\n",
      "   Processing 27000th document\n",
      "   Processing 27100th document\n",
      "   Processing 27200th document\n",
      "   Processing 27300th document\n",
      "   Processing 27400th document\n",
      "   Processing 27500th document\n",
      "   Processing 27600th document\n",
      "   Processing 27700th document\n",
      "   Processing 27800th document\n",
      "   Processing 27900th document\n",
      "   Processing 28000th document\n",
      "Processing dev set\n",
      "   Processing 0th document\n",
      "   Processing 100th document\n",
      "   Processing 200th document\n",
      "   Processing 300th document\n",
      "   Processing 400th document\n",
      "   Processing 500th document\n",
      "   Processing 600th document\n",
      "   Processing 700th document\n",
      "   Processing 800th document\n",
      "   Processing 900th document\n",
      "Processing test set\n",
      "   Processing 0th document\n",
      "   Processing 100th document\n",
      "   Processing 200th document\n",
      "   Processing 300th document\n",
      "   Processing 400th document\n",
      "   Processing 500th document\n",
      "   Processing 600th document\n",
      "   Processing 700th document\n",
      "   Processing 800th document\n",
      "   Processing 900th document\n"
     ]
    }
   ],
   "source": [
    "for split, split_ids in lstm_splits.items():\n",
    "    corpus_file = open(\"data/text/corpus_{}.txt\".format(split), \"w+\")\n",
    "    docs = []\n",
    "    print(\"Processing {} set\".format(split))\n",
    "    for i, pmid in enumerate(splits[split]):\n",
    "        if i % 100 == 0:\n",
    "            print(\"   Processing {}th document\".format(i))\n",
    "        words = open(\"data/text/corpus/{}.txt\".format(pmid), \"r\").read().split()\n",
    "        words = [word for word in words if word in lexicon]\n",
    "        docs.append(\" \".join(words))\n",
    "    corpus_file.write(\"\\n\".join(docs))\n",
    "    corpus_file.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (Ontol)",
   "language": "python",
   "name": "ontol"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}