06-similarity-match-with-ann.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Similarity Match with Approximate Nearest Neighbors\n",
    "\n",
    "In this notebook we will be experimenting with the embedding features. Using the `faiss` package we can efficiently find similar movies based on the cosine similarity."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from src.similarity import create_search_index, find_top_similar, compute_cosine_similarity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finding Nearest Neighbors in the Embedding Space\n",
    "\n",
    "Here we will get the 30 nearest neighbors for each movie considering content based and collaborative filtering embedding features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>factor_0</th>\n",
       "      <th>factor_1</th>\n",
       "      <th>factor_2</th>\n",
       "      <th>factor_3</th>\n",
       "      <th>factor_4</th>\n",
       "      <th>factor_5</th>\n",
       "      <th>factor_6</th>\n",
       "      <th>factor_7</th>\n",
       "      <th>factor_8</th>\n",
       "      <th>factor_9</th>\n",
       "      <th>...</th>\n",
       "      <th>factor_290</th>\n",
       "      <th>factor_291</th>\n",
       "      <th>factor_292</th>\n",
       "      <th>factor_293</th>\n",
       "      <th>factor_294</th>\n",
       "      <th>factor_295</th>\n",
       "      <th>factor_296</th>\n",
       "      <th>factor_297</th>\n",
       "      <th>factor_298</th>\n",
       "      <th>factor_299</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.13718</td>\n",
       "      <td>-0.03387</td>\n",
       "      <td>-0.03017</td>\n",
       "      <td>0.05062</td>\n",
       "      <td>0.03955</td>\n",
       "      <td>-0.00645</td>\n",
       "      <td>-0.16905</td>\n",
       "      <td>0.03667</td>\n",
       "      <td>0.06681</td>\n",
       "      <td>-0.00628</td>\n",
       "      <td>...</td>\n",
       "      <td>0.04458</td>\n",
       "      <td>-0.00807</td>\n",
       "      <td>-0.02188</td>\n",
       "      <td>0.01180</td>\n",
       "      <td>0.06933</td>\n",
       "      <td>0.11840</td>\n",
       "      <td>-0.09077</td>\n",
       "      <td>0.07540</td>\n",
       "      <td>0.00299</td>\n",
       "      <td>-0.07051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.03843</td>\n",
       "      <td>0.02700</td>\n",
       "      <td>-0.01565</td>\n",
       "      <td>0.02153</td>\n",
       "      <td>-0.09215</td>\n",
       "      <td>0.00628</td>\n",
       "      <td>0.00412</td>\n",
       "      <td>-0.04694</td>\n",
       "      <td>0.07562</td>\n",
       "      <td>-0.04912</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.01289</td>\n",
       "      <td>0.06133</td>\n",
       "      <td>-0.12022</td>\n",
       "      <td>-0.02296</td>\n",
       "      <td>-0.00038</td>\n",
       "      <td>0.10698</td>\n",
       "      <td>-0.09957</td>\n",
       "      <td>0.01236</td>\n",
       "      <td>0.07305</td>\n",
       "      <td>-0.00638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.07394</td>\n",
       "      <td>-0.06563</td>\n",
       "      <td>-0.05454</td>\n",
       "      <td>0.03857</td>\n",
       "      <td>0.06494</td>\n",
       "      <td>-0.03999</td>\n",
       "      <td>-0.12235</td>\n",
       "      <td>-0.07590</td>\n",
       "      <td>0.08604</td>\n",
       "      <td>-0.01678</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.05933</td>\n",
       "      <td>0.02223</td>\n",
       "      <td>-0.02603</td>\n",
       "      <td>0.00313</td>\n",
       "      <td>0.08757</td>\n",
       "      <td>0.00446</td>\n",
       "      <td>-0.05378</td>\n",
       "      <td>-0.09055</td>\n",
       "      <td>0.08543</td>\n",
       "      <td>-0.05654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.05019</td>\n",
       "      <td>-0.07952</td>\n",
       "      <td>-0.03727</td>\n",
       "      <td>0.05436</td>\n",
       "      <td>-0.09217</td>\n",
       "      <td>-0.02115</td>\n",
       "      <td>-0.06872</td>\n",
       "      <td>-0.03769</td>\n",
       "      <td>0.04223</td>\n",
       "      <td>-0.06436</td>\n",
       "      <td>...</td>\n",
       "      <td>0.00114</td>\n",
       "      <td>0.04767</td>\n",
       "      <td>-0.15716</td>\n",
       "      <td>-0.04998</td>\n",
       "      <td>0.01877</td>\n",
       "      <td>0.08810</td>\n",
       "      <td>-0.11753</td>\n",
       "      <td>-0.02773</td>\n",
       "      <td>0.05437</td>\n",
       "      <td>-0.03121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.00977</td>\n",
       "      <td>0.00258</td>\n",
       "      <td>-0.04626</td>\n",
       "      <td>-0.03330</td>\n",
       "      <td>-0.04444</td>\n",
       "      <td>0.14469</td>\n",
       "      <td>0.09473</td>\n",
       "      <td>0.02177</td>\n",
       "      <td>-0.00838</td>\n",
       "      <td>0.01385</td>\n",
       "      <td>...</td>\n",
       "      <td>0.03475</td>\n",
       "      <td>0.06998</td>\n",
       "      <td>-0.03812</td>\n",
       "      <td>0.02543</td>\n",
       "      <td>-0.00995</td>\n",
       "      <td>0.09979</td>\n",
       "      <td>-0.02725</td>\n",
       "      <td>-0.03844</td>\n",
       "      <td>0.00601</td>\n",
       "      <td>0.03093</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 300 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   factor_0  factor_1  factor_2  factor_3  factor_4  factor_5  factor_6  \\\n",
       "0   0.13718  -0.03387  -0.03017   0.05062   0.03955  -0.00645  -0.16905   \n",
       "1   0.03843   0.02700  -0.01565   0.02153  -0.09215   0.00628   0.00412   \n",
       "2   0.07394  -0.06563  -0.05454   0.03857   0.06494  -0.03999  -0.12235   \n",
       "3   0.05019  -0.07952  -0.03727   0.05436  -0.09217  -0.02115  -0.06872   \n",
       "4   0.00977   0.00258  -0.04626  -0.03330  -0.04444   0.14469   0.09473   \n",
       "\n",
       "   factor_7  factor_8  factor_9  ...  factor_290  factor_291  factor_292  \\\n",
       "0   0.03667   0.06681  -0.00628  ...     0.04458    -0.00807    -0.02188   \n",
       "1  -0.04694   0.07562  -0.04912  ...    -0.01289     0.06133    -0.12022   \n",
       "2  -0.07590   0.08604  -0.01678  ...    -0.05933     0.02223    -0.02603   \n",
       "3  -0.03769   0.04223  -0.06436  ...     0.00114     0.04767    -0.15716   \n",
       "4   0.02177  -0.00838   0.01385  ...     0.03475     0.06998    -0.03812   \n",
       "\n",
       "   factor_293  factor_294  factor_295  factor_296  factor_297  factor_298  \\\n",
       "0     0.01180     0.06933     0.11840    -0.09077     0.07540     0.00299   \n",
       "1    -0.02296    -0.00038     0.10698    -0.09957     0.01236     0.07305   \n",
       "2     0.00313     0.08757     0.00446    -0.05378    -0.09055     0.08543   \n",
       "3    -0.04998     0.01877     0.08810    -0.11753    -0.02773     0.05437   \n",
       "4     0.02543    -0.00995     0.09979    -0.02725    -0.03844     0.00601   \n",
       "\n",
       "   factor_299  \n",
       "0    -0.07051  \n",
       "1    -0.00638  \n",
       "2    -0.05654  \n",
       "3    -0.03121  \n",
       "4     0.03093  \n",
       "\n",
       "[5 rows x 300 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "content_meta = pd.read_csv('output/content_embedding_meta.tsv', sep='\\t')\n",
    "cb_meta = pd.read_csv('output/cb_embedding_meta.tsv', sep='\\t')\n",
    "\n",
    "col_names = ['factor_{}'.format(i) for i in range(300)]\n",
    "cb_embedding = pd.read_csv('output/collab_filt_vectors.tsv', sep='\\t', names=col_names)\n",
    "tfidf_embedding = pd.read_csv('output/tfidf_embedding_vectors.tsv', sep='\\t', names=col_names)\n",
    "tfidf_crew_embedding = pd.read_csv('output/tfidf_crew_embedding_vectors.tsv', sep='\\t', names=col_names)\n",
    "tfidf_crew_cast_embedding = pd.read_csv('output/tfidf_crew_cast_embedding_vectors.tsv', sep='\\t', names=col_names)\n",
    "\n",
    "cb_embedding.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Nearest Neighbors for Collaborative Filtering Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 20 ms, sys: 20 ms, total: 40 ms\n",
      "Wall time: 37.1 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "cb_index = create_search_index(cb_embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 368 ms, sys: 32 ms, total: 400 ms\n",
      "Wall time: 120 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_right</th>\n",
       "      <th>similarity</th>\n",
       "      <th>id_left</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>836</th>\n",
       "      <td>1196</td>\n",
       "      <td>0.898831</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>847</th>\n",
       "      <td>1210</td>\n",
       "      <td>0.805315</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9828</th>\n",
       "      <td>122886</td>\n",
       "      <td>0.696018</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10564</th>\n",
       "      <td>166528</td>\n",
       "      <td>0.646935</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>431</th>\n",
       "      <td>561</td>\n",
       "      <td>0.544250</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id_right  similarity  id_left\n",
       "836        1196    0.898831      260\n",
       "847        1210    0.805315      260\n",
       "9828     122886    0.696018      260\n",
       "10564    166528    0.646935      260\n",
       "431         561    0.544250      260"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "similars = find_top_similar([260], cb_index, 15, cb_embedding, cb_meta)\n",
    "similars.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>204</th>\n",
       "      <td>260</td>\n",
       "      <td>Star Wars</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id      title\n",
       "204  260  Star Wars"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_right</th>\n",
       "      <th>similarity</th>\n",
       "      <th>id_left</th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1196</td>\n",
       "      <td>0.898831</td>\n",
       "      <td>260</td>\n",
       "      <td>1196</td>\n",
       "      <td>The Empire Strikes Back</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1210</td>\n",
       "      <td>0.805315</td>\n",
       "      <td>260</td>\n",
       "      <td>1210</td>\n",
       "      <td>Return of the Jedi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>122886</td>\n",
       "      <td>0.696018</td>\n",
       "      <td>260</td>\n",
       "      <td>122886</td>\n",
       "      <td>Star Wars: The Force Awakens</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>166528</td>\n",
       "      <td>0.646935</td>\n",
       "      <td>260</td>\n",
       "      <td>166528</td>\n",
       "      <td>Rogue One: A Star Wars Story</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>561</td>\n",
       "      <td>0.544250</td>\n",
       "      <td>260</td>\n",
       "      <td>561</td>\n",
       "      <td>Killer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>31247</td>\n",
       "      <td>0.525195</td>\n",
       "      <td>260</td>\n",
       "      <td>31247</td>\n",
       "      <td>The Fighting Sullivans</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>33493</td>\n",
       "      <td>0.487151</td>\n",
       "      <td>260</td>\n",
       "      <td>33493</td>\n",
       "      <td>Star Wars: Episode III - Revenge of the Sith</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>5199</td>\n",
       "      <td>0.470113</td>\n",
       "      <td>260</td>\n",
       "      <td>5199</td>\n",
       "      <td>The Long Riders</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>136485</td>\n",
       "      <td>0.467294</td>\n",
       "      <td>260</td>\n",
       "      <td>136485</td>\n",
       "      <td>Robot Chicken: Star Wars</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>168026</td>\n",
       "      <td>0.452794</td>\n",
       "      <td>260</td>\n",
       "      <td>168026</td>\n",
       "      <td>Marvel One-Shot: Agent Carter</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>26502</td>\n",
       "      <td>0.432812</td>\n",
       "      <td>260</td>\n",
       "      <td>26502</td>\n",
       "      <td>A Christmas Carol</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>6528</td>\n",
       "      <td>0.431993</td>\n",
       "      <td>260</td>\n",
       "      <td>6528</td>\n",
       "      <td>Start the Revolution Without Me</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>5378</td>\n",
       "      <td>0.426094</td>\n",
       "      <td>260</td>\n",
       "      <td>5378</td>\n",
       "      <td>Star Wars: Episode II - Attack of the Clones</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>44041</td>\n",
       "      <td>0.424542</td>\n",
       "      <td>260</td>\n",
       "      <td>44041</td>\n",
       "      <td>All Quiet on the Western Front</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2628</td>\n",
       "      <td>0.418522</td>\n",
       "      <td>260</td>\n",
       "      <td>2628</td>\n",
       "      <td>Star Wars: Episode I - The Phantom Menace</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id_right  similarity  id_left      id  \\\n",
       "0       1196    0.898831      260    1196   \n",
       "1       1210    0.805315      260    1210   \n",
       "2     122886    0.696018      260  122886   \n",
       "3     166528    0.646935      260  166528   \n",
       "4        561    0.544250      260     561   \n",
       "5      31247    0.525195      260   31247   \n",
       "6      33493    0.487151      260   33493   \n",
       "7       5199    0.470113      260    5199   \n",
       "8     136485    0.467294      260  136485   \n",
       "9     168026    0.452794      260  168026   \n",
       "10     26502    0.432812      260   26502   \n",
       "11      6528    0.431993      260    6528   \n",
       "12      5378    0.426094      260    5378   \n",
       "13     44041    0.424542      260   44041   \n",
       "14      2628    0.418522      260    2628   \n",
       "\n",
       "                                           title  \n",
       "0                        The Empire Strikes Back  \n",
       "1                             Return of the Jedi  \n",
       "2                   Star Wars: The Force Awakens  \n",
       "3                   Rogue One: A Star Wars Story  \n",
       "4                                         Killer  \n",
       "5                         The Fighting Sullivans  \n",
       "6   Star Wars: Episode III - Revenge of the Sith  \n",
       "7                                The Long Riders  \n",
       "8                       Robot Chicken: Star Wars  \n",
       "9                  Marvel One-Shot: Agent Carter  \n",
       "10                             A Christmas Carol  \n",
       "11               Start the Revolution Without Me  \n",
       "12  Star Wars: Episode II - Attack of the Clones  \n",
       "13                All Quiet on the Western Front  \n",
       "14     Star Wars: Episode I - The Phantom Menace  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(cb_meta.loc[cb_meta['id']==260])\n",
    "display(pd.merge(similars, cb_meta, left_on='id_right', right_on='id', how='inner'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Nearest Neighbors for Text Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 440 ms, sys: 48 ms, total: 488 ms\n",
      "Wall time: 72.1 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "tfidf_index = create_search_index(tfidf_embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 396 ms, sys: 12 ms, total: 408 ms\n",
      "Wall time: 42.5 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_right</th>\n",
       "      <th>similarity</th>\n",
       "      <th>id_left</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>845</th>\n",
       "      <td>1196</td>\n",
       "      <td>0.900626</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>856</th>\n",
       "      <td>1210</td>\n",
       "      <td>0.876032</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3921</th>\n",
       "      <td>5378</td>\n",
       "      <td>0.808551</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1923</th>\n",
       "      <td>2628</td>\n",
       "      <td>0.795053</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6935</th>\n",
       "      <td>33493</td>\n",
       "      <td>0.787766</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id_right  similarity  id_left\n",
       "845       1196    0.900626      260\n",
       "856       1210    0.876032      260\n",
       "3921      5378    0.808551      260\n",
       "1923      2628    0.795053      260\n",
       "6935     33493    0.787766      260"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "similars = find_top_similar([260], tfidf_index, 15, tfidf_embedding, content_meta)\n",
    "similars.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>204</th>\n",
       "      <td>260</td>\n",
       "      <td>Star Wars</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id      title\n",
       "204  260  Star Wars"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_right</th>\n",
       "      <th>similarity</th>\n",
       "      <th>id_left</th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1196</td>\n",
       "      <td>0.900626</td>\n",
       "      <td>260</td>\n",
       "      <td>1196</td>\n",
       "      <td>The Empire Strikes Back</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1210</td>\n",
       "      <td>0.876032</td>\n",
       "      <td>260</td>\n",
       "      <td>1210</td>\n",
       "      <td>Return of the Jedi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5378</td>\n",
       "      <td>0.808551</td>\n",
       "      <td>260</td>\n",
       "      <td>5378</td>\n",
       "      <td>Star Wars: Episode II - Attack of the Clones</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2628</td>\n",
       "      <td>0.795053</td>\n",
       "      <td>260</td>\n",
       "      <td>2628</td>\n",
       "      <td>Star Wars: Episode I - The Phantom Menace</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>33493</td>\n",
       "      <td>0.787766</td>\n",
       "      <td>260</td>\n",
       "      <td>33493</td>\n",
       "      <td>Star Wars: Episode III - Revenge of the Sith</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1200</td>\n",
       "      <td>0.697495</td>\n",
       "      <td>260</td>\n",
       "      <td>1200</td>\n",
       "      <td>Aliens</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>113345</td>\n",
       "      <td>0.697349</td>\n",
       "      <td>260</td>\n",
       "      <td>113345</td>\n",
       "      <td>Jupiter Ascending</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>68358</td>\n",
       "      <td>0.674341</td>\n",
       "      <td>260</td>\n",
       "      <td>68358</td>\n",
       "      <td>Star Trek</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>122886</td>\n",
       "      <td>0.665957</td>\n",
       "      <td>260</td>\n",
       "      <td>122886</td>\n",
       "      <td>Star Wars: The Force Awakens</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>112852</td>\n",
       "      <td>0.659662</td>\n",
       "      <td>260</td>\n",
       "      <td>112852</td>\n",
       "      <td>Guardians of the Galaxy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>102445</td>\n",
       "      <td>0.641535</td>\n",
       "      <td>260</td>\n",
       "      <td>102445</td>\n",
       "      <td>Star Trek Into Darkness</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1097</td>\n",
       "      <td>0.640180</td>\n",
       "      <td>260</td>\n",
       "      <td>1097</td>\n",
       "      <td>E.T. the Extra-Terrestrial</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>166528</td>\n",
       "      <td>0.635602</td>\n",
       "      <td>260</td>\n",
       "      <td>166528</td>\n",
       "      <td>Rogue One: A Star Wars Story</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>316</td>\n",
       "      <td>0.629126</td>\n",
       "      <td>260</td>\n",
       "      <td>316</td>\n",
       "      <td>Stargate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1214</td>\n",
       "      <td>0.626877</td>\n",
       "      <td>260</td>\n",
       "      <td>1214</td>\n",
       "      <td>Alien</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id_right  similarity  id_left      id  \\\n",
       "0       1196    0.900626      260    1196   \n",
       "1       1210    0.876032      260    1210   \n",
       "2       5378    0.808551      260    5378   \n",
       "3       2628    0.795053      260    2628   \n",
       "4      33493    0.787766      260   33493   \n",
       "5       1200    0.697495      260    1200   \n",
       "6     113345    0.697349      260  113345   \n",
       "7      68358    0.674341      260   68358   \n",
       "8     122886    0.665957      260  122886   \n",
       "9     112852    0.659662      260  112852   \n",
       "10    102445    0.641535      260  102445   \n",
       "11      1097    0.640180      260    1097   \n",
       "12    166528    0.635602      260  166528   \n",
       "13       316    0.629126      260     316   \n",
       "14      1214    0.626877      260    1214   \n",
       "\n",
       "                                           title  \n",
       "0                        The Empire Strikes Back  \n",
       "1                             Return of the Jedi  \n",
       "2   Star Wars: Episode II - Attack of the Clones  \n",
       "3      Star Wars: Episode I - The Phantom Menace  \n",
       "4   Star Wars: Episode III - Revenge of the Sith  \n",
       "5                                         Aliens  \n",
       "6                              Jupiter Ascending  \n",
       "7                                      Star Trek  \n",
       "8                   Star Wars: The Force Awakens  \n",
       "9                        Guardians of the Galaxy  \n",
       "10                       Star Trek Into Darkness  \n",
       "11                    E.T. the Extra-Terrestrial  \n",
       "12                  Rogue One: A Star Wars Story  \n",
       "13                                      Stargate  \n",
       "14                                         Alien  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(content_meta.loc[content_meta['id']==260])\n",
    "display(pd.merge(similars, content_meta, left_on='id_right', right_on='id', how='inner'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining Target Movies\n",
    "\n",
    "To reduce the seacrh space, let's consider just those movies we have selected when computing the similarity by user correlation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2, 122886, 21, 57368, 2080, 34, 122918, 77866, 6187, 50, 2100, 65588, 116797, 62, 69, 70, 72, 6218, 2132, 2133, 2137, 30812, 2144, 6242, 26729, 59501, 110, 111, 112, 2160, 4223, 161922, 140, 145, 2194, 150, 161, 162, 165, 8360, 86190, 6323, 6333, 6339, 2247, 2253, 6349, 208, 4306, 4318, 223, 6377, 235, 2288, 247, 253, 2302, 55553, 6403, 260, 292, 296, 2352, 2353, 316, 318, 57669, 329, 333, 337, 8533, 2394, 2395, 4444, 350, 4446, 4447, 4452, 364, 367, 4465, 2420, 380, 4478, 6541, 2467, 6565, 441, 2490, 8633, 45499, 8641, 8643, 457, 104913, 94677, 8665, 4571, 6620, 76251, 480, 481, 6636, 493, 2542, 47610, 115210, 2571, 2572, 527, 2580, 4641, 551, 2599, 555, 6713, 94777, 588, 589, 593, 594, 4700, 608, 4718, 6769, 43635, 635, 2683, 166526, 8831, 31364, 2694, 647, 68237, 49815, 2716, 60069, 60074, 4782, 4801, 53953, 2763, 33493, 4823, 733, 736, 103141, 2791, 6893, 750, 60141, 8949, 778, 2826, 4874, 4878, 784, 786, 4886, 6934, 2840, 4901, 2858, 832, 838, 6987, 103249, 37729, 2924, 4973, 7022, 4975, 2941, 4993, 904, 908, 2959, 913, 5010, 33683, 50068, 920, 2968, 924, 2973, 62376, 2985, 2987, 2990, 109487, 95167, 72641, 3033, 996, 7147, 7153, 7154, 5107, 134130, 7162, 3074, 3082, 1042, 97304, 3101, 74789, 3114, 1073, 1089, 1090, 1091, 1092, 1093, 1097, 3147, 1101, 27728, 3160, 5218, 3175, 134248, 64620, 1136, 3210, 25750, 1176, 31900, 44191, 44195, 1193, 5291, 1196, 1197, 1201, 3253, 1206, 1208, 1209, 1210, 1214, 1215, 1216, 3264, 58559, 83134, 1222, 1225, 3273, 3275, 1231, 1235, 42197, 1240, 1244, 1245, 1246, 1252, 5349, 54503, 60647, 1258, 1259, 1266, 79091, 1268, 1270, 1272, 1278, 34048, 5377, 1285, 1287, 1288, 48394, 1291, 87306, 85261, 7438, 1307, 3362, 72998, 5418, 1333, 3384, 7482, 1343, 1347, 5445, 93510, 64839, 132424, 3408, 3409, 5463, 1370, 3421, 1374, 91500, 1390, 66934, 89470, 48516, 56715, 3471, 79251, 3499, 5580, 3536, 3555, 46578, 52722, 1527, 5629, 44555, 106002, 5669, 1594, 1608, 1610, 1619, 3671, 1625, 3676, 3677, 48738, 34405, 1639, 63082, 3699, 3708, 3713, 1673, 1676, 59022, 1682, 52885, 1693, 5791, 3751, 1704, 3753, 50872, 1732, 134853, 1735, 52973, 5881, 53000, 69406, 3880, 1833, 104241, 1848, 1857, 32587, 48982, 1887, 71520, 1892, 48997, 3948, 1907, 3988, 3996, 1953, 1956, 1959, 112552, 1961, 1963, 6059, 1968, 81845, 81847, 4025, 4034, 2009, 2011, 2012, 2028, 30707, 4085]\n"
     ]
    }
   ],
   "source": [
    "target = pd.read_csv('output/movie_similarity.csv', usecols=['id_left','id_right','similarity'])\n",
    "\n",
    "target_ids = set(target['id_left'])\n",
    "target_ids = target_ids.intersection(set(content_meta['id'].values))\n",
    "target_ids = list(target_ids.intersection(set(cb_meta['id'].values)))\n",
    "\n",
    "print(target_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.74 s, sys: 188 ms, total: 4.92 s\n",
      "Wall time: 555 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "cb_similars = find_top_similar(target_ids, cb_index, 30, cb_embedding, cb_meta)\n",
    "cb_similars.to_csv('output/movie_similarity_cb.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.91 s, sys: 44 ms, total: 2.95 s\n",
      "Wall time: 155 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "tfidf_similars = find_top_similar(target_ids, tfidf_index, 30, tfidf_embedding, content_meta)\n",
    "tfidf_similars.to_csv('output/movie_similarity_tfidf.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Nearest Neighbors for Content Embedding with Crew"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.68 s, sys: 200 ms, total: 6.88 s\n",
      "Wall time: 398 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "tfidf_crew_index = create_search_index(tfidf_crew_embedding)\n",
    "\n",
    "tfidf_crew_similars = find_top_similar(target_ids, tfidf_crew_index, 30, tfidf_crew_embedding, content_meta)\n",
    "tfidf_crew_similars.to_csv('output/movie_similarity_tfidf_crew.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Nearest Neighbors for Content Embedding with Crew & Cast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.8 s, sys: 168 ms, total: 5.97 s\n",
      "Wall time: 381 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "tfidf_crew_cast_index = create_search_index(tfidf_crew_cast_embedding)\n",
    "\n",
    "tfidf_crew_cast_similars = find_top_similar(target_ids, tfidf_crew_cast_index, 30, tfidf_crew_cast_embedding, content_meta)\n",
    "tfidf_crew_cast_similars.to_csv('output/movie_similarity_tfidf_crew_cast.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Computing Cosine Similarity in the Embedding Space\n",
    "\n",
    "Now, let's apply the embedding features to compute the similarity between a pair of movies. We will consider the same pairs we found by user correlation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_left</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id_right</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>2</td>\n",
       "      <td>0.335812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480</th>\n",
       "      <td>2</td>\n",
       "      <td>0.317939</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367</th>\n",
       "      <td>2</td>\n",
       "      <td>0.314605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>588</th>\n",
       "      <td>2</td>\n",
       "      <td>0.298155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>589</th>\n",
       "      <td>2</td>\n",
       "      <td>0.263034</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          id_left    target\n",
       "id_right                   \n",
       "364             2  0.335812\n",
       "480             2  0.317939\n",
       "367             2  0.314605\n",
       "588             2  0.298155\n",
       "589             2  0.263034"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "right_ids = set(target['id_right'])\n",
    "right_ids = right_ids.intersection(set(content_meta['id'].values))\n",
    "right_ids = list(right_ids.intersection(set(cb_meta['id'].values)))\n",
    "\n",
    "target_new = target.loc[target['id_left'].isin(target_ids) & target['id_right'].isin(right_ids)].copy()\n",
    "target_new.rename(columns={'similarity':'target'}, inplace=True)\n",
    "target_new.set_index('id_right', inplace=True)\n",
    "\n",
    "target_new.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cosine Similarity with Collaborative Filtering Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.26 s, sys: 0 ns, total: 3.26 s\n",
      "Wall time: 3.26 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_right</th>\n",
       "      <th>similarity</th>\n",
       "      <th>id_left</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>253</td>\n",
       "      <td>0.343631</td>\n",
       "      <td>2</td>\n",
       "      <td>0.237054</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>110</td>\n",
       "      <td>0.289399</td>\n",
       "      <td>2</td>\n",
       "      <td>0.242349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2011</td>\n",
       "      <td>0.279259</td>\n",
       "      <td>2</td>\n",
       "      <td>0.248665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>165</td>\n",
       "      <td>0.244945</td>\n",
       "      <td>2</td>\n",
       "      <td>0.234542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1097</td>\n",
       "      <td>0.237925</td>\n",
       "      <td>2</td>\n",
       "      <td>0.233132</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id_right  similarity  id_left    target\n",
       "0       253    0.343631        2  0.237054\n",
       "1       110    0.289399        2  0.242349\n",
       "2      2011    0.279259        2  0.248665\n",
       "3       165    0.244945        2  0.234542\n",
       "4      1097    0.237925        2  0.233132"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "parts = []\n",
    "for left_id in target_new['id_left'].unique():\n",
    "    selected = target_new.loc[target_new['id_left']==left_id, ['target']]\n",
    "    similars = compute_cosine_similarity([left_id], selected.index, cb_embedding, cb_meta)\n",
    "    similars.set_index('id_right', drop=False, inplace=True)\n",
    "    parts.append(similars.join(selected, how='inner'))\n",
    "\n",
    "cb_ranking = pd.concat(parts, axis=0, ignore_index=True)\n",
    "cb_ranking.to_csv('output/movie_ranking_cb.csv', index=False)\n",
    "\n",
    "cb_ranking.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cosine Similarity with Text Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.9 s, sys: 0 ns, total: 2.9 s\n",
      "Wall time: 2.9 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_right</th>\n",
       "      <th>similarity</th>\n",
       "      <th>id_left</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4886</td>\n",
       "      <td>0.433568</td>\n",
       "      <td>2</td>\n",
       "      <td>0.235485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>110</td>\n",
       "      <td>0.401254</td>\n",
       "      <td>2</td>\n",
       "      <td>0.242349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4306</td>\n",
       "      <td>0.387088</td>\n",
       "      <td>2</td>\n",
       "      <td>0.240394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1097</td>\n",
       "      <td>0.383907</td>\n",
       "      <td>2</td>\n",
       "      <td>0.233132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1291</td>\n",
       "      <td>0.352362</td>\n",
       "      <td>2</td>\n",
       "      <td>0.232228</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id_right  similarity  id_left    target\n",
       "0      4886    0.433568        2  0.235485\n",
       "1       110    0.401254        2  0.242349\n",
       "2      4306    0.387088        2  0.240394\n",
       "3      1097    0.383907        2  0.233132\n",
       "4      1291    0.352362        2  0.232228"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "parts = []\n",
    "for left_id in target_new['id_left'].unique():\n",
    "    selected = target_new.loc[target_new['id_left']==left_id, ['target']]\n",
    "    similars = compute_cosine_similarity([left_id], selected.index, tfidf_embedding, content_meta)\n",
    "    similars.set_index('id_right', drop=False, inplace=True)\n",
    "    parts.append(similars.join(selected, how='inner'))\n",
    "\n",
    "tfidf_ranking = pd.concat(parts, axis=0, ignore_index=True)\n",
    "tfidf_ranking.to_csv('output/movie_ranking_tfidf.csv', index=False)\n",
    "\n",
    "tfidf_ranking.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cosine Similarity with Content Embedding (Text + Crew)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.75 s, sys: 0 ns, total: 2.75 s\n",
      "Wall time: 2.74 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_right</th>\n",
       "      <th>similarity</th>\n",
       "      <th>id_left</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6377</td>\n",
       "      <td>0.382239</td>\n",
       "      <td>2</td>\n",
       "      <td>0.229354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4306</td>\n",
       "      <td>0.336810</td>\n",
       "      <td>2</td>\n",
       "      <td>0.240394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>364</td>\n",
       "      <td>0.310683</td>\n",
       "      <td>2</td>\n",
       "      <td>0.335812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1097</td>\n",
       "      <td>0.279052</td>\n",
       "      <td>2</td>\n",
       "      <td>0.233132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4886</td>\n",
       "      <td>0.272782</td>\n",
       "      <td>2</td>\n",
       "      <td>0.235485</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id_right  similarity  id_left    target\n",
       "0      6377    0.382239        2  0.229354\n",
       "1      4306    0.336810        2  0.240394\n",
       "2       364    0.310683        2  0.335812\n",
       "3      1097    0.279052        2  0.233132\n",
       "4      4886    0.272782        2  0.235485"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "parts = []\n",
    "for left_id in target_new['id_left'].unique():\n",
    "    selected = target_new.loc[target_new['id_left']==left_id, ['target']]\n",
    "    similars = compute_cosine_similarity([left_id], selected.index, tfidf_crew_embedding, content_meta)\n",
    "    similars.set_index('id_right', drop=False, inplace=True)\n",
    "    parts.append(similars.join(selected, how='inner'))\n",
    "\n",
    "tfidf_crew_ranking = pd.concat(parts, axis=0, ignore_index=True)\n",
    "tfidf_crew_ranking.to_csv('output/movie_ranking_tfidf_crew.csv', index=False)\n",
    "\n",
    "tfidf_crew_ranking.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cosine Similarity with Content Embedding (Text + Crew + Cast)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.93 s, sys: 0 ns, total: 2.93 s\n",
      "Wall time: 2.93 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_right</th>\n",
       "      <th>similarity</th>\n",
       "      <th>id_left</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4306</td>\n",
       "      <td>0.384327</td>\n",
       "      <td>2</td>\n",
       "      <td>0.240394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2716</td>\n",
       "      <td>0.223461</td>\n",
       "      <td>2</td>\n",
       "      <td>0.222825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>364</td>\n",
       "      <td>0.164086</td>\n",
       "      <td>2</td>\n",
       "      <td>0.335812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>736</td>\n",
       "      <td>0.158615</td>\n",
       "      <td>2</td>\n",
       "      <td>0.256185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2011</td>\n",
       "      <td>0.147242</td>\n",
       "      <td>2</td>\n",
       "      <td>0.248665</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id_right  similarity  id_left    target\n",
       "0      4306    0.384327        2  0.240394\n",
       "1      2716    0.223461        2  0.222825\n",
       "2       364    0.164086        2  0.335812\n",
       "3       736    0.158615        2  0.256185\n",
       "4      2011    0.147242        2  0.248665"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "parts = []\n",
    "for left_id in target_new['id_left'].unique():\n",
    "    selected = target_new.loc[target_new['id_left']==left_id, ['target']]\n",
    "    similars = compute_cosine_similarity([left_id], selected.index, tfidf_crew_cast_embedding, content_meta)\n",
    "    similars.set_index('id_right', drop=False, inplace=True)\n",
    "    parts.append(similars.join(selected, how='inner'))\n",
    "\n",
    "tfidf_crew_cast_ranking = pd.concat(parts, axis=0, ignore_index=True)\n",
    "tfidf_crew_cast_ranking.to_csv('output/movie_ranking_tfidf_crew_cast.csv', index=False)\n",
    "\n",
    "tfidf_crew_cast_ranking.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}