Final Recommender.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.metrics.pairwise import pairwise_distances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fastest source of contractions I could find: https://stackoverflow.com/questions/19790188/expanding-english-language-contractions-in-python\n",
    "contractions = { \n",
    "\"ain't\": \"am not / are not / is not / has not / have not\",\n",
    "\"aren't\": \"are not / am not\",\n",
    "\"can't\": \"cannot\",\n",
    "\"can't've\": \"cannot have\",\n",
    "\"'cause\": \"because\",\n",
    "\"could've\": \"could have\",\n",
    "\"couldn't\": \"could not\",\n",
    "\"couldn't've\": \"could not have\",\n",
    "\"didn't\": \"did not\",\n",
    "\"doesn't\": \"does not\",\n",
    "\"don't\": \"do not\",\n",
    "\"hadn't\": \"had not\",\n",
    "\"hadn't've\": \"had not have\",\n",
    "\"hasn't\": \"has not\",\n",
    "\"haven't\": \"have not\",\n",
    "\"he'd\": \"he had / he would\",\n",
    "\"he'd've\": \"he would have\",\n",
    "\"he'll\": \"he shall / he will\",\n",
    "\"he'll've\": \"he shall have / he will have\",\n",
    "\"he's\": \"he has / he is\",\n",
    "\"how'd\": \"how did\",\n",
    "\"how'd'y\": \"how do you\",\n",
    "\"how'll\": \"how will\",\n",
    "\"how's\": \"how has / how is / how does\",\n",
    "\"I'd\": \"I had / I would\",\n",
    "\"I'd've\": \"I would have\",\n",
    "\"I'll\": \"I shall / I will\",\n",
    "\"I'll've\": \"I shall have / I will have\",\n",
    "\"I'm\": \"I am\",\n",
    "\"I've\": \"I have\",\n",
    "\"isn't\": \"is not\",\n",
    "\"it'd\": \"it had / it would\",\n",
    "\"it'd've\": \"it would have\",\n",
    "\"it'll\": \"it shall / it will\",\n",
    "\"it'll've\": \"it shall have / it will have\",\n",
    "\"it's\": \"it has / it is\",\n",
    "\"let's\": \"let us\",\n",
    "\"ma'am\": \"madam\",\n",
    "\"mayn't\": \"may not\",\n",
    "\"might've\": \"might have\",\n",
    "\"mightn't\": \"might not\",\n",
    "\"mightn't've\": \"might not have\",\n",
    "\"must've\": \"must have\",\n",
    "\"mustn't\": \"must not\",\n",
    "\"mustn't've\": \"must not have\",\n",
    "\"needn't\": \"need not\",\n",
    "\"needn't've\": \"need not have\",\n",
    "\"o'clock\": \"of the clock\",\n",
    "\"oughtn't\": \"ought not\",\n",
    "\"oughtn't've\": \"ought not have\",\n",
    "\"shan't\": \"shall not\",\n",
    "\"sha'n't\": \"shall not\",\n",
    "\"shan't've\": \"shall not have\",\n",
    "\"she'd\": \"she had / she would\",\n",
    "\"she'd've\": \"she would have\",\n",
    "\"she'll\": \"she shall / she will\",\n",
    "\"she'll've\": \"she shall have / she will have\",\n",
    "\"she's\": \"she has / she is\",\n",
    "\"should've\": \"should have\",\n",
    "\"shouldn't\": \"should not\",\n",
    "\"shouldn't've\": \"should not have\",\n",
    "\"so've\": \"so have\",\n",
    "\"so's\": \"so as / so is\",\n",
    "\"that'd\": \"that would / that had\",\n",
    "\"that'd've\": \"that would have\",\n",
    "\"that's\": \"that has / that is\",\n",
    "\"there'd\": \"there had / there would\",\n",
    "\"there'd've\": \"there would have\",\n",
    "\"there's\": \"there has / there is\",\n",
    "\"they'd\": \"they had / they would\",\n",
    "\"they'd've\": \"they would have\",\n",
    "\"they'll\": \"they shall / they will\",\n",
    "\"they'll've\": \"they shall have / they will have\",\n",
    "\"they're\": \"they are\",\n",
    "\"they've\": \"they have\",\n",
    "\"to've\": \"to have\",\n",
    "\"wasn't\": \"was not\",\n",
    "\"we'd\": \"we had / we would\",\n",
    "\"we'd've\": \"we would have\",\n",
    "\"we'll\": \"we will\",\n",
    "\"we'll've\": \"we will have\",\n",
    "\"we're\": \"we are\",\n",
    "\"we've\": \"we have\",\n",
    "\"weren't\": \"were not\",\n",
    "\"what'll\": \"what shall / what will\",\n",
    "\"what'll've\": \"what shall have / what will have\",\n",
    "\"what're\": \"what are\",\n",
    "\"what's\": \"what has / what is\",\n",
    "\"what've\": \"what have\",\n",
    "\"when's\": \"when has / when is\",\n",
    "\"when've\": \"when have\",\n",
    "\"where'd\": \"where did\",\n",
    "\"where's\": \"where has / where is\",\n",
    "\"where've\": \"where have\",\n",
    "\"who'll\": \"who shall / who will\",\n",
    "\"who'll've\": \"who shall have / who will have\",\n",
    "\"who's\": \"who has / who is\",\n",
    "\"who've\": \"who have\",\n",
    "\"why's\": \"why has / why is\",\n",
    "\"why've\": \"why have\",\n",
    "\"will've\": \"will have\",\n",
    "\"won't\": \"will not\",\n",
    "\"won't've\": \"will not have\",\n",
    "\"would've\": \"would have\",\n",
    "\"wouldn't\": \"would not\",\n",
    "\"wouldn't've\": \"would not have\",\n",
    "\"y'all\": \"you all\",\n",
    "\"y'all'd\": \"you all would\",\n",
    "\"y'all'd've\": \"you all would have\",\n",
    "\"y'all're\": \"you all are\",\n",
    "\"y'all've\": \"you all have\",\n",
    "\"you'd\": \"you had / you would\",\n",
    "\"you'd've\": \"you would have\",\n",
    "\"you'll\": \"you shall / you will\",\n",
    "\"you'll've\": \"you shall have / you will have\",\n",
    "\"you're\": \"you are\",\n",
    "\"you've\": \"you have\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"raw transcripts/this_american_life.csv\")\n",
    "titles = pd.read_csv(\"raw data/episode_info_clean.csv\")\n",
    "\n",
    "data.sort_values(\"episode\")\n",
    "titles.sort_values(\"episode_number\", inplace=True)\n",
    "\n",
    "df = data.merge(titles, left_on=\"episode\", right_on=\"episode_number\").drop(columns=['episode', \"episode_number\", \"radio_date\"])\n",
    "\n",
    "df.set_index(\"title\", inplace = True)\n",
    "\n",
    "df.text = df.text.str.lower()\n",
    "for p in \".,?$%&^*!\":\n",
    "    df.text = df.text.str.replace(p, \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k, v in contractions.items():\n",
    "    df.text = df.text.str.replace(k.lower(), v.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"Cleaned data/TAL_clean.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "tvec = TfidfVectorizer(stop_words=\"english\", max_features=600)\n",
    "port = PorterStemmer()\n",
    "stops = stopwords.words(\"english\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, episode in enumerate(df.text):\n",
    "    df.text[i] = \" \".join([port.stem(str(s)) for s in str(df.text[i]).split(\" \") if not str(s) in stops])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>New Beginnings</td>\n",
       "      <td>joe franklin  readi ira glass oh emce show ir...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Small Scale Sin</td>\n",
       "      <td>ok three boy age 13 15 16 three chose appear ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Poultry Slam 1995</td>\n",
       "      <td>daniel hous ever sinc girl dinner come someti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Vacations</td>\n",
       "      <td>thing hawaii go anybodi say paradis liter wor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Anger and Forgiveness</td>\n",
       "      <td>hi ira glass record 2006 show listen fifth ep...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                    text\n",
       "title                                                                   \n",
       "New Beginnings          joe franklin  readi ira glass oh emce show ir...\n",
       "Small Scale Sin         ok three boy age 13 15 16 three chose appear ...\n",
       "Poultry Slam 1995       daniel hous ever sinc girl dinner come someti...\n",
       "Vacations               thing hawaii go anybodi say paradis liter wor...\n",
       "Anger and Forgiveness   hi ira glass record 2006 show listen fifth ep..."
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "recommender = pairwise_distances(tvec.fit_transform(df.text), metric=\"cosine\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "rec_df = pd.DataFrame(recommender, index=df.index, columns=df.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recommend(episode, top=11):\n",
    "    find = rec_df.loc[rec_df.index.str.contains(episode)].index[0]\n",
    "    return pd.DataFrame(rec_df[find].sort_values()[1:top])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>How Bad Is Bad?</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>title</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>Small Scale Sin</td>\n",
       "      <td>0.077747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Suckers</td>\n",
       "      <td>0.297652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Call For Help</td>\n",
       "      <td>0.302179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Hoaxing Yourself</td>\n",
       "      <td>0.306875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Confessions</td>\n",
       "      <td>0.307118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>I Thought I Knew You</td>\n",
       "      <td>0.311834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>This I Used to Believe</td>\n",
       "      <td>0.312196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>How to Take Money from Strangers</td>\n",
       "      <td>0.315627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Nice Work If You Can Get It</td>\n",
       "      <td>0.317445</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>Blackjack</td>\n",
       "      <td>0.318719</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  How Bad Is Bad?\n",
       "title                                            \n",
       "Small Scale Sin                          0.077747\n",
       "Suckers                                  0.297652\n",
       "Call For Help                            0.302179\n",
       "Hoaxing Yourself                         0.306875\n",
       "Confessions                              0.307118\n",
       "I Thought I Knew You                     0.311834\n",
       "This I Used to Believe                   0.312196\n",
       "How to Take Money from Strangers         0.315627\n",
       "Nice Work If You Can Get It              0.317445\n",
       "Blackjack                                0.318719"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recommend(\"Bad\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading in the corpus for the final recommender test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus = pd.read_csv(\"Recommender files/corpus.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "corpus.drop_duplicates(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus.set_index(\"title\", inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "genre_cols = [g for g in list(corpus.columns) if \"_genre\" in g]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "genres = corpus[genre_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy import sparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "genres_sparse = sparse.csc_matrix(np.array(genres.replace(0, np.NaN)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "tvec = TfidfVectorizer(stop_words=\"english\", max_features=2000, ngram_range=(1,2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "                dtype=<class 'numpy.float64'>, encoding='utf-8',\n",
       "                input='content', lowercase=True, max_df=1.0, max_features=2000,\n",
       "                min_df=1, ngram_range=(1, 2), norm='l2', preprocessor=None,\n",
       "                smooth_idf=True, stop_words='english', strip_accents=None,\n",
       "                sublinear_tf=False, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "                tokenizer=None, use_idf=True, vocabulary=None)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tvec.fit(corpus[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = tvec.transform(corpus[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "Unable to allocate array with shape (80099459327,) and data type int64",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-13-8b9b45ab2ee0>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mrec_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpairwise_distances\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mmetric\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"cosine\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\pairwise.py\u001b[0m in \u001b[0;36mpairwise_distances\u001b[1;34m(X, Y, metric, n_jobs, **kwds)\u001b[0m\n\u001b[0;32m   1586\u001b[0m         \u001b[0mfunc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpartial\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdistance\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcdist\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmetric\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmetric\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1587\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1588\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_parallel_pairwise\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mY\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1589\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1590\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\pairwise.py\u001b[0m in \u001b[0;36m_parallel_pairwise\u001b[1;34m(X, Y, func, n_jobs, **kwds)\u001b[0m\n\u001b[0;32m   1204\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1205\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0meffective_n_jobs\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mn_jobs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1206\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mY\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1207\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1208\u001b[0m     \u001b[1;31m# enforce a threading backend to prevent data communication overhead\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\pairwise.py\u001b[0m in \u001b[0;36mcosine_distances\u001b[1;34m(X, Y)\u001b[0m\n\u001b[0;32m    679\u001b[0m     \"\"\"\n\u001b[0;32m    680\u001b[0m     \u001b[1;31m# 1.0 - cosine_similarity(X, Y) without copy\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 681\u001b[1;33m     \u001b[0mS\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcosine_similarity\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mY\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    682\u001b[0m     \u001b[0mS\u001b[0m \u001b[1;33m*=\u001b[0m \u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    683\u001b[0m     \u001b[0mS\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\pairwise.py\u001b[0m in \u001b[0;36mcosine_similarity\u001b[1;34m(X, Y, dense_output)\u001b[0m\n\u001b[0;32m   1034\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1035\u001b[0m     K = safe_sparse_dot(X_normalized, Y_normalized.T,\n\u001b[1;32m-> 1036\u001b[1;33m                         dense_output=dense_output)\n\u001b[0m\u001b[0;32m   1037\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1038\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mK\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\extmath.py\u001b[0m in \u001b[0;36msafe_sparse_dot\u001b[1;34m(a, b, dense_output)\u001b[0m\n\u001b[0;32m    135\u001b[0m     \"\"\"\n\u001b[0;32m    136\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0msparse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0missparse\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0msparse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0missparse\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 137\u001b[1;33m         \u001b[0mret\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0ma\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mb\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    138\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mdense_output\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mret\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"toarray\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    139\u001b[0m             \u001b[0mret\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mret\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\scipy\\sparse\\base.py\u001b[0m in \u001b[0;36m__mul__\u001b[1;34m(self, other)\u001b[0m\n\u001b[0;32m    479\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mother\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    480\u001b[0m                 \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'dimension mismatch'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 481\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_mul_sparse_matrix\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mother\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    482\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    483\u001b[0m         \u001b[1;31m# If it's a list or whatever, treat it like a matrix\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\scipy\\sparse\\compressed.py\u001b[0m in \u001b[0;36m_mul_sparse_matrix\u001b[1;34m(self, other)\u001b[0m\n\u001b[0;32m    514\u001b[0m                                     maxval=nnz)\n\u001b[0;32m    515\u001b[0m         \u001b[0mindptr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindptr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0midx_dtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 516\u001b[1;33m         \u001b[0mindices\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnnz\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0midx_dtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    517\u001b[0m         \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnnz\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mupcast\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mother\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    518\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mMemoryError\u001b[0m: Unable to allocate array with shape (80099459327,) and data type int64"
     ]
    }
   ],
   "source": [
    "rec_data = pairwise_distances(text,metric=\"cosine\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data is is too large for personal computer, final recommender is to be worked on a cloud computing system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rec_df = pd.DataFrame(rec_data, index=corpus[\"title\"], columns=[\"title\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recommend(episode, top=11):\n",
    "    find = rec_df.loc[rec_df.index.str.contains(episode)].index[0]\n",
    "    return pd.DataFrame(rec_df[find].sort_values()[1:top])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recommend(\"Small Scale Sin\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}