train_test_split.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import functions as f\n",
    "\n",
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "main_path = 'data/tales/'\n",
    "files = os.listdir(main_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "files_with_max_count = {title[:-5]: title[-5:-4] for title in files}\n",
    "files_unique = [key + str(np.random.randint(1, int(value)+1)) + '.txt' for key, value in files_with_max_count.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_size = 800\n",
    "\n",
    "files_ids = list(range(len(files_unique)))\n",
    "files_ids_sample = np.random.choice(files_ids, sample_size, replace=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_ids, test_ids = train_test_split(files_ids_sample, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = ''\n",
    "test = ''\n",
    "eot_token = '<| end of text |>'\n",
    "\n",
    "for i in train_ids:\n",
    "    tale = f.read_txt(main_path + files_unique[i])\n",
    "    tale = re.sub(' +', ' ', tale.replace('\\n',' '))\n",
    "    train += tale + ' ' + eot_token + ' '\n",
    "\n",
    "for i in test_ids:\n",
    "    tale = f.read_txt(main_path + files_unique[i])\n",
    "    tale = re.sub(' +', ' ', tale.replace('\\n',' '))\n",
    "    test += tale + ' ' + eot_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train\n",
      "Number of unique words: 50487\n",
      "Total number of words: 766970\n",
      "\n",
      "test\n",
      "Number of unique words: 21630\n",
      "Total number of words: 202860\n"
     ]
    }
   ],
   "source": [
    "exclude_words = [eot_token, '']\n",
    "print('train')\n",
    "f.text_summary(train, exclude_words)\n",
    "print('\\ntest')\n",
    "f.text_summary(test, exclude_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "f.save_txt(train, 'data/train.txt')\n",
    "f.save_txt(test, 'data/test.txt')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}