personalization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "d3b54394",
    "outputId": "f7f06c45-ab19-4977-aaf3-a58d07ff91e7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting lightfm\n",
      "  Using cached lightfm-1.16-cp36-cp36m-linux_x86_64.whl\n",
      "Requirement already satisfied: numpy in /home/ec2-user/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages (from lightfm) (1.18.5)\n",
      "Requirement already satisfied: scipy>=0.17.0 in /home/ec2-user/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages (from lightfm) (1.5.3)\n",
      "Requirement already satisfied: scikit-learn in /home/ec2-user/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages (from lightfm) (0.24.1)\n",
      "Requirement already satisfied: requests in /home/ec2-user/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages (from lightfm) (2.25.1)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /home/ec2-user/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages (from requests->lightfm) (2.10)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/ec2-user/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages (from requests->lightfm) (1.26.5)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages (from requests->lightfm) (2021.5.30)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /home/ec2-user/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages (from requests->lightfm) (3.0.4)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/ec2-user/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages (from scikit-learn->lightfm) (2.1.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /home/ec2-user/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages (from scikit-learn->lightfm) (1.0.1)\n",
      "Installing collected packages: lightfm\n",
      "Successfully installed lightfm-1.16\n",
      "\u001b[33mWARNING: You are using pip version 21.1.2; however, version 21.1.3 is available.\n",
      "You should consider upgrading via the '/home/ec2-user/anaconda3/envs/tensorflow2_p36/bin/python -m pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install lightfm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "93edcb91"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pickle\n",
    "from datetime import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "from pandas.io.json import json_normalize\n",
    "import numpy as np\n",
    "from scipy import sparse\n",
    "from scipy.sparse import csr_matrix, coo_matrix\n",
    "from lightfm import LightFM\n",
    "from lightfm.data import Dataset\n",
    "from lightfm.cross_validation import random_train_test_split\n",
    "from lightfm.evaluation import auc_score, precision_at_k, recall_at_k"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "685a21db"
   },
   "source": [
    "## Processing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "3e09a595",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "data = [json.loads(line) for line in open('raw_data.json', 'r')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "de7421c2"
   },
   "outputs": [],
   "source": [
    "df = pd.json_normalize(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "id": "af46bd00",
    "outputId": "7890bf56-bdcc-438f-8bde-4acfc52aaeb7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44876"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.customerVisitorId.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ce5ff415",
    "outputId": "75e278a7-4e68-452c-91f0-d506bef36369"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['customerId', 'customerArea', 'customerVisitorId', 'customerLoginId',\n",
       "       'customerSessionId', 'sessionStartTime', 'sessionEndTime',\n",
       "       'customerSessionNumber', 'hits', 'totals.timeOnSite', 'totals.events',\n",
       "       'totals.newSession', 'totals.bounce', 'totals.totalSearches',\n",
       "       'totals.totalNavigations', 'totals.totalRefinements',\n",
       "       'totals.totalViewProducts', 'totals.totalViewProductRevenue',\n",
       "       'totals.totalNullSearches', 'totals.uniqueViewProducts',\n",
       "       'trafficSource.gbiFlag', 'trafficSource.deviceCategory',\n",
       "       'trafficSource.device', 'trafficSource.browser', 'trafficSource.os',\n",
       "       'trafficSource.userAgent', 'trafficSource.language',\n",
       "       'trafficSource.page', 'trafficSource.city', 'trafficSource.region',\n",
       "       'trafficSource.country', 'trafficSource.campaign',\n",
       "       'trafficSource.source', 'trafficSource.medium', 'trafficSource.content',\n",
       "       'totals.totalAddToCarts', 'totals.totalAddToCartQty',\n",
       "       'totals.totalAddToCartRevenue', 'totals.uniqueAddToCarts',\n",
       "       'trafficSource.term', 'totals.uniqueSearches', 'totals.queriesSearched',\n",
       "       'totals.uniqueRefinements', 'totals.uniqueNavigations',\n",
       "       'attributedTotals.totalViewProducts',\n",
       "       'attributedTotals.totalViewProductRevenue',\n",
       "       'attributedTotals.uniqueSearches', 'attributedTotals.queriesAttributed',\n",
       "       'attributedTotals.uniqueViewProducts',\n",
       "       'attributedTotals.totalAddToCarts',\n",
       "       'attributedTotals.totalAddToCartQty',\n",
       "       'attributedTotals.totalAddToCartRevenue',\n",
       "       'attributedTotals.uniqueAddToCarts', 'totals.totalOrders',\n",
       "       'totals.totalOrderQty', 'totals.totalOrderRevenue',\n",
       "       'totals.uniqueOrders', 'attributedTotals.totalOrders',\n",
       "       'attributedTotals.totalOrderQty', 'attributedTotals.totalOrderRevenue',\n",
       "       'attributedTotals.uniqueOrders'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 304,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = ['customerVisitorId', 'customerSessionId','sessionStartTime', 'sessionEndTime', 'customerSessionNumber']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "aac0f36d"
   },
   "outputs": [],
   "source": [
    "hits = pd.json_normalize(data, record_path=['hits', 'product'], meta=cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "4e2b7356",
    "outputId": "e572dd8e-3a98-4054-993a-de3cb8185cad"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sku</th>\n",
       "      <th>ID</th>\n",
       "      <th>name</th>\n",
       "      <th>price</th>\n",
       "      <th>collection</th>\n",
       "      <th>attributedSearches</th>\n",
       "      <th>quantity</th>\n",
       "      <th>listLocation</th>\n",
       "      <th>listPosition</th>\n",
       "      <th>customerVisitorId</th>\n",
       "      <th>customerSessionId</th>\n",
       "      <th>sessionStartTime</th>\n",
       "      <th>sessionEndTime</th>\n",
       "      <th>customerSessionNumber</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1474200</th>\n",
       "      <td>SWA051</td>\n",
       "      <td>SWA051</td>\n",
       "      <td>dr. stephen langer's ultimate 16 strain probio...</td>\n",
       "      <td>14.79</td>\n",
       "      <td>Swanson Probiotics</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cki8pfl1y000139ebdeezn6pn</td>\n",
       "      <td>cki8pfl1y000139ebdeezn6pn1606894616</td>\n",
       "      <td>2020-12-02 07:36:56.522423 UTC</td>\n",
       "      <td>2020-12-02 07:40:45.201722 UTC</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1474201</th>\n",
       "      <td>SWA051</td>\n",
       "      <td>SWA051</td>\n",
       "      <td>dr. stephen langer's ultimate 16 strain probio...</td>\n",
       "      <td>14.79</td>\n",
       "      <td>default</td>\n",
       "      <td>[]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cki8pfl1y000139ebdeezn6pn</td>\n",
       "      <td>cki8pfl1y000139ebdeezn6pn1606894616</td>\n",
       "      <td>2020-12-02 07:36:56.522423 UTC</td>\n",
       "      <td>2020-12-02 07:40:45.201722 UTC</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1474202</th>\n",
       "      <td>SWA051</td>\n",
       "      <td>SWA051</td>\n",
       "      <td>dr. stephen langer's ultimate 16 strain probio...</td>\n",
       "      <td>14.79</td>\n",
       "      <td>default</td>\n",
       "      <td>[]</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cki8pfl1y000139ebdeezn6pn</td>\n",
       "      <td>cki8pfl1y000139ebdeezn6pn1606894616</td>\n",
       "      <td>2020-12-02 07:36:56.522423 UTC</td>\n",
       "      <td>2020-12-02 07:40:45.201722 UTC</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1474203</th>\n",
       "      <td>SWA051</td>\n",
       "      <td>SWA051</td>\n",
       "      <td>3.2 billion cfu 60 veg drcaps</td>\n",
       "      <td>14.79</td>\n",
       "      <td>Swanson Probiotics</td>\n",
       "      <td>[]</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cki8pfl1y000139ebdeezn6pn</td>\n",
       "      <td>cki8pfl1y000139ebdeezn6pn1606894616</td>\n",
       "      <td>2020-12-02 07:36:56.522423 UTC</td>\n",
       "      <td>2020-12-02 07:40:45.201722 UTC</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            sku      ID                                               name  \\\n",
       "1474200  SWA051  SWA051  dr. stephen langer's ultimate 16 strain probio...   \n",
       "1474201  SWA051  SWA051  dr. stephen langer's ultimate 16 strain probio...   \n",
       "1474202  SWA051  SWA051  dr. stephen langer's ultimate 16 strain probio...   \n",
       "1474203  SWA051  SWA051                      3.2 billion cfu 60 veg drcaps   \n",
       "\n",
       "         price          collection attributedSearches quantity listLocation  \\\n",
       "1474200  14.79  Swanson Probiotics                 []      NaN          NaN   \n",
       "1474201  14.79             default                 []      NaN          NaN   \n",
       "1474202  14.79             default                 []        3          NaN   \n",
       "1474203  14.79  Swanson Probiotics                 []        3          NaN   \n",
       "\n",
       "        listPosition          customerVisitorId  \\\n",
       "1474200          NaN  cki8pfl1y000139ebdeezn6pn   \n",
       "1474201          NaN  cki8pfl1y000139ebdeezn6pn   \n",
       "1474202          NaN  cki8pfl1y000139ebdeezn6pn   \n",
       "1474203          NaN  cki8pfl1y000139ebdeezn6pn   \n",
       "\n",
       "                           customerSessionId                sessionStartTime  \\\n",
       "1474200  cki8pfl1y000139ebdeezn6pn1606894616  2020-12-02 07:36:56.522423 UTC   \n",
       "1474201  cki8pfl1y000139ebdeezn6pn1606894616  2020-12-02 07:36:56.522423 UTC   \n",
       "1474202  cki8pfl1y000139ebdeezn6pn1606894616  2020-12-02 07:36:56.522423 UTC   \n",
       "1474203  cki8pfl1y000139ebdeezn6pn1606894616  2020-12-02 07:36:56.522423 UTC   \n",
       "\n",
       "                         sessionEndTime customerSessionNumber  \n",
       "1474200  2020-12-02 07:40:45.201722 UTC                     1  \n",
       "1474201  2020-12-02 07:40:45.201722 UTC                     1  \n",
       "1474202  2020-12-02 07:40:45.201722 UTC                     1  \n",
       "1474203  2020-12-02 07:40:45.201722 UTC                     1  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hits[hits.customerVisitorId=='cki8pfl1y000139ebdeezn6pn']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "swansonhealthproduction    1573529\n",
       "default                      36134\n",
       "Swanson Premium              17705\n",
       "Swanson Ultra                11953\n",
       "NOW Foods                     6493\n",
       "                            ...   \n",
       "Atkins                           1\n",
       "Dr. Linus Pauling                1\n",
       "EFX Sports                       1\n",
       "Aaraam                           1\n",
       "Red Ace                          1\n",
       "Name: collection, Length: 905, dtype: int64"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hits.collection.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "9896b553"
   },
   "outputs": [],
   "source": [
    "all_cols = df.columns[df.columns.str.contains('totals')].tolist() + cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "10e41c7a"
   },
   "outputs": [],
   "source": [
    "df_merged = df[all_cols].merge(hits, how='left', on=cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 576
    },
    "id": "1403d256",
    "outputId": "975e1a51-087b-4102-f6b9-e0686d8eefb9"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>name</th>\n",
       "      <th>price</th>\n",
       "      <th>totals.totalOrders</th>\n",
       "      <th>totals.totalOrderQty</th>\n",
       "      <th>customerSessionId</th>\n",
       "      <th>customerSessionNumber</th>\n",
       "      <th>totals.uniqueOrders</th>\n",
       "      <th>totals.totalViewProducts</th>\n",
       "      <th>totals.totalAddToCarts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1474537</th>\n",
       "      <td>SWA051</td>\n",
       "      <td>dr. stephen langer's ultimate 16 strain probio...</td>\n",
       "      <td>14.79</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>cki8pfl1y000139ebdeezn6pn1606894616</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1474538</th>\n",
       "      <td>SWA051</td>\n",
       "      <td>dr. stephen langer's ultimate 16 strain probio...</td>\n",
       "      <td>14.79</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>cki8pfl1y000139ebdeezn6pn1606894616</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1474539</th>\n",
       "      <td>SWA051</td>\n",
       "      <td>dr. stephen langer's ultimate 16 strain probio...</td>\n",
       "      <td>14.79</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>cki8pfl1y000139ebdeezn6pn1606894616</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1474540</th>\n",
       "      <td>SWA051</td>\n",
       "      <td>3.2 billion cfu 60 veg drcaps</td>\n",
       "      <td>14.79</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>cki8pfl1y000139ebdeezn6pn1606894616</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             ID                                               name  price  \\\n",
       "1474537  SWA051  dr. stephen langer's ultimate 16 strain probio...  14.79   \n",
       "1474538  SWA051  dr. stephen langer's ultimate 16 strain probio...  14.79   \n",
       "1474539  SWA051  dr. stephen langer's ultimate 16 strain probio...  14.79   \n",
       "1474540  SWA051                      3.2 billion cfu 60 veg drcaps  14.79   \n",
       "\n",
       "        totals.totalOrders totals.totalOrderQty  \\\n",
       "1474537                  1                    3   \n",
       "1474538                  1                    3   \n",
       "1474539                  1                    3   \n",
       "1474540                  1                    3   \n",
       "\n",
       "                           customerSessionId customerSessionNumber  \\\n",
       "1474537  cki8pfl1y000139ebdeezn6pn1606894616                     1   \n",
       "1474538  cki8pfl1y000139ebdeezn6pn1606894616                     1   \n",
       "1474539  cki8pfl1y000139ebdeezn6pn1606894616                     1   \n",
       "1474540  cki8pfl1y000139ebdeezn6pn1606894616                     1   \n",
       "\n",
       "        totals.uniqueOrders totals.totalViewProducts totals.totalAddToCarts  \n",
       "1474537                   1                        2                      1  \n",
       "1474538                   1                        2                      1  \n",
       "1474539                   1                        2                      1  \n",
       "1474540                   1                        2                      1  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merged[df_merged.customerVisitorId=='cki8pfl1y000139ebdeezn6pn'][['ID', 'name', 'price', 'totals.totalOrders', 'totals.totalOrderQty', 'customerSessionId', 'customerSessionNumber', 'totals.uniqueOrders', 'totals.totalViewProducts', 'totals.totalAddToCarts']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 352
    },
    "id": "OfHlBXMWt9UW",
    "outputId": "32b3e7ea-126f-41a7-9c1c-76cfb792fc55"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1.23361e+05, 1.46570e+04, 2.08600e+03, 4.64000e+02, 9.60000e+01,\n",
       "        4.90000e+01, 5.00000e+00, 1.00000e+00, 8.00000e+00, 9.00000e+00]),\n",
       " array([  0.66 ,  23.914,  47.168,  70.422,  93.676, 116.93 , 140.184,\n",
       "        163.438, 186.692, 209.946, 233.2  ]),\n",
       " <a list of 10 Patch objects>)"
      ]
     },
     "execution_count": 52,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAD4CAYAAAAZ1BptAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAS4klEQVR4nO3df6zd9X3f8edrdmFpusQQrqzMtmarsTY5qE2IRTxlqrJ4AkOqmklJZDQVL7PiTTFdOlVqTfeHpyRIRNvKipQgebWHibI4iCbCWpy6FqGK9ocJl4AAQylX/Ci2AN/GDnRDCTN974/zcXN6uR8b32Ofa3yfD+nofL/vz+f7/X6+X53rF98f55CqQpKk2fy9+R6AJOnCZUhIkroMCUlSlyEhSeoyJCRJXYvnewDn2hVXXFErV66c72FI0jvKww8//FdVNTGzftGFxMqVK5mcnJzvYUjSO0qSF2are7lJktRlSEiSugwJSVKXISFJ6jIkJEldhoQkqcuQkCR1GRKSpC5DQpLUddF943oUK7d/d962/fxtn5y3bUtSj2cSkqSuM4ZEkt1JjiV5Yqj2n5P8eZLHknwnyZKhtluSTCV5Osm1Q/UNrTaVZPtQfVWSB1v9W0kuafVL2/xUa195rnZakvT2vJ0zibuADTNqB4Erq+pXgL8AbgFIsgbYBHywLfO1JIuSLAK+ClwHrAFubH0BvgLcXlUfAE4AW1p9C3Ci1W9v/SRJY3TGkKiqHwDHZ9T+tKpOttlDwPI2vRHYW1U/q6rngCng6vaaqqpnq+oNYC+wMUmATwD3tuX3ADcMrWtPm74XWN/6S5LG5Fzck/g3wPfa9DLgxaG2I63Wq78P+MlQ4Jyq/511tfZXW/+3SLI1yWSSyenp6ZF3SJI0MFJIJPmPwEngG+dmOHNTVTuram1VrZ2YeMv/M0OSNEdzfgQ2yb8Gfh1YX1XVykeBFUPdlrcanfqPgSVJFrezheH+p9Z1JMli4L2tvyRpTOZ0JpFkA/C7wG9U1etDTfuATe3JpFXAauCHwEPA6vYk0yUMbm7va+HyAPCptvxm4L6hdW1u058Cvj8URpKkMTjjmUSSbwIfB65IcgTYweBppkuBg+1e8qGq+ndVdTjJPcCTDC5DbauqN9t6bgYOAIuA3VV1uG3i94C9Sb4MPALsavVdwNeTTDG4cb7pHOyvJOksnDEkqurGWcq7Zqmd6n8rcOss9f3A/lnqzzJ4+mlm/afAp880PknS+eM3riVJXYaEJKnLkJAkdRkSkqQuQ0KS1GVISJK6DAlJUpchIUnqMiQkSV2GhCSpy5CQJHUZEpKkLkNCktRlSEiSugwJSVKXISFJ6jIkJEldhoQkqcuQkCR1GRKSpC5DQpLUZUhIkroMCUlSlyEhSeoyJCRJXWcMiSS7kxxL8sRQ7fIkB5M8094va/UkuSPJVJLHklw1tMzm1v+ZJJuH6h9J8nhb5o4kOd02JEnj83bOJO4CNsyobQfur6rVwP1tHuA6YHV7bQXuhME/+MAO4KPA1cCOoX/07wQ+N7TchjNsQ5I0JmcMiar6AXB8RnkjsKdN7wFuGKrfXQOHgCVJ3g9cCxysquNVdQI4CGxobe+pqkNVVcDdM9Y12zYkSWMy13sSS6vqpTb9MrC0TS8DXhzqd6TVTlc/Mkv9dNt4iyRbk0wmmZyenp7D7kiSZjPyjet2BlDnYCxz3kZV7ayqtVW1dmJi4nwORZIWlLmGxCvtUhHt/VirHwVWDPVb3mqnqy+fpX66bUiSxmSuIbEPOPWE0mbgvqH6Te0pp3XAq+2S0QHgmiSXtRvW1wAHWttrSda1p5pumrGu2bYhSRqTxWfqkOSbwMeBK5IcYfCU0m3APUm2AC8An2nd9wPXA1PA68BnAarqeJIvAQ+1fl+sqlM3wz/P4AmqdwHfay9Osw1J0picMSSq6sZO0/pZ+hawrbOe3cDuWeqTwJWz1H882zYkSePjN64lSV2GhCSpy5CQJHUZEpKkLkNCktRlSEiSugwJSVKXISFJ6jIkJEldhoQkqcuQkCR1GRKSpC5DQpLUZUhIkroMCUlSlyEhSeoyJCRJXYaEJKnLkJAkdRkSkqQuQ0KS1GVISJK6DAlJUpchIUnqGikkkvyHJIeTPJHkm0n+fpJVSR5MMpXkW0kuaX0vbfNTrX3l0HpuafWnk1w7VN/QalNJto8yVknS2ZtzSCRZBvx7YG1VXQksAjYBXwFur6oPACeALW2RLcCJVr+99SPJmrbcB4ENwNeSLEqyCPgqcB2wBrix9ZUkjcmol5sWA+9Kshj4ReAl4BPAva19D3BDm97Y5mnt65Ok1fdW1c+q6jlgCri6vaaq6tmqegPY2/pKksZkziFRVUeB/wL8JYNweBV4GPhJVZ1s3Y4Ay9r0MuDFtuzJ1v99w/UZy/TqkqQxGeVy02UM/st+FfAPgXczuFw0dkm2JplMMjk9PT0fQ5Cki9Iol5v+BfBcVU1X1f8Dvg18DFjSLj8BLAeOtumjwAqA1v5e4MfD9RnL9OpvUVU7q2ptVa2dmJgYYZckScNGCYm/BNYl+cV2b2E98CTwAPCp1mczcF+b3tfmae3fr6pq9U3t6adVwGrgh8BDwOr2tNQlDG5u7xthvJKks7T4zF1mV1UPJrkX+BFwEngE2Al8F9ib5Muttqstsgv4epIp4DiDf/SpqsNJ7mEQMCeBbVX1JkCSm4EDDJ6c2l1Vh+c6XknS2ZtzSABU1Q5gx4zyswyeTJrZ96fApzvruRW4dZb6fmD/KGOUJM2d37iWJHUZEpKkLkNCktRlSEiSugwJSVKXISFJ6jIkJEldhoQkqcuQkCR1GRKSpC5DQpLUZUhIkroMCUlSlyEhSeoyJCRJXYaEJKnLkJAkdRkSkqQuQ0KS1GVISJK6DAlJUpchIUnqMiQkSV2GhCSpy5CQJHWNFBJJliS5N8mfJ3kqyT9NcnmSg0meae+Xtb5JckeSqSSPJblqaD2bW/9nkmweqn8kyeNtmTuSZJTxSpLOzqhnEn8I/ElV/RPgV4GngO3A/VW1Gri/zQNcB6xur63AnQBJLgd2AB8FrgZ2nAqW1udzQ8ttGHG8kqSzMOeQSPJe4NeAXQBV9UZV/QTYCOxp3fYAN7TpjcDdNXAIWJLk/cC1wMGqOl5VJ4CDwIbW9p6qOlRVBdw9tC5J0hiMciaxCpgG/keSR5L8UZJ3A0ur6qXW52VgaZteBrw4tPyRVjtd/cgs9bdIsjXJZJLJ6enpEXZJkjRslJBYDFwF3FlVHwb+Lz+/tARAOwOoEbbxtlTVzqpaW1VrJyYmzvfmJGnBGCUkjgBHqurBNn8vg9B4pV0qor0fa+1HgRVDyy9vtdPVl89SlySNyZxDoqpeBl5M8o9baT3wJLAPOPWE0mbgvja9D7ipPeW0Dni1XZY6AFyT5LJ2w/oa4EBrey3JuvZU001D65IkjcHiEZf/LeAbSS4BngU+yyB47kmyBXgB+Ezrux+4HpgCXm99qarjSb4EPNT6fbGqjrfpzwN3Ae8CvtdekqQxGSkkqupRYO0sTetn6VvAts56dgO7Z6lPAleOMkZJ0tz5jWtJUpchIUnqMiQkSV2GhCSpy5CQJHUZEpKkLkNCktRlSEiSugwJSVKXISFJ6jIkJEldhoQkqcuQkCR1GRKSpC5DQpLUZUhIkroMCUlSlyEhSeoyJCRJXYaEJKnLkJAkdRkSkqQuQ0KS1GVISJK6Rg6JJIuSPJLkf7X5VUkeTDKV5FtJLmn1S9v8VGtfObSOW1r96STXDtU3tNpUku2jjlWSdHbOxZnEF4Cnhua/AtxeVR8ATgBbWn0LcKLVb2/9SLIG2AR8ENgAfK0FzyLgq8B1wBrgxtZXkjQmI4VEkuXAJ4E/avMBPgHc27rsAW5o0xvbPK19feu/EdhbVT+rqueAKeDq9pqqqmer6g1gb+srSRqTUc8k/hvwu8DftPn3AT+pqpNt/giwrE0vA14EaO2vtv5/W5+xTK8uSRqTOYdEkl8HjlXVw+dwPHMdy9Ykk0kmp6en53s4knTRGOVM4mPAbyR5nsGloE8AfwgsSbK49VkOHG3TR4EVAK39vcCPh+szlunV36KqdlbV2qpaOzExMcIuSZKGzTkkquqWqlpeVSsZ3Hj+flX9K+AB4FOt22bgvja9r83T2r9fVdXqm9rTT6uA1cAPgYeA1e1pqUvaNvbNdbySpLO3+MxdztrvAXuTfBl4BNjV6ruAryeZAo4z+Eefqjqc5B7gSeAksK2q3gRIcjNwAFgE7K6qw+dhvJKkjnMSElX1Z8CftelnGTyZNLPPT4FPd5a/Fbh1lvp+YP+5GKMk6ez5jWtJUpchIUnqMiQkSV2GhCSpy5CQJHUZEpKkLkNCktRlSEiSugwJSVKXISFJ6jIkJEldhoQkqcuQkCR1GRKSpC5DQpLUZUhIkroMCUlSlyEhSeoyJCRJXYaEJKnLkJAkdRkSkqQuQ0KS1GVISJK6DAlJUtecQyLJiiQPJHkyyeEkX2j1y5McTPJMe7+s1ZPkjiRTSR5LctXQuja3/s8k2TxU/0iSx9sydyTJKDsrSTo7o5xJnAR+p6rWAOuAbUnWANuB+6tqNXB/mwe4DljdXluBO2EQKsAO4KPA1cCOU8HS+nxuaLkNI4xXknSW5hwSVfVSVf2oTf818BSwDNgI7Gnd9gA3tOmNwN01cAhYkuT9wLXAwao6XlUngIPAhtb2nqo6VFUF3D20LknSGJyTexJJVgIfBh4EllbVS63pZWBpm14GvDi02JFWO139yCz12ba/Nclkksnp6emR9kWS9HMjh0SSXwL+GPjtqnptuK2dAdSo2ziTqtpZVWurau3ExMT53pwkLRgjhUSSX2AQEN+oqm+38ivtUhHt/VirHwVWDC2+vNVOV18+S12SNCajPN0UYBfwVFX9wVDTPuDUE0qbgfuG6je1p5zWAa+2y1IHgGuSXNZuWF8DHGhtryVZ17Z109C6JEljsHiEZT8G/CbweJJHW+33gduAe5JsAV4APtPa9gPXA1PA68BnAarqeJIvAQ+1fl+squNt+vPAXcC7gO+1lyRpTOYcElX1v4He9xbWz9K/gG2dde0Gds9SnwSunOsYJUmj8RvXkqQuQ0KS1GVISJK6RrlxrXNo5fbvzst2n7/tk/OyXUnvDJ5JSJK6DAlJUpchIUnqMiQkSV2GhCSpy5CQJHUZEpKkLkNCktRlSEiSugwJSVKXISFJ6jIkJEldhoQkqcuQkCR1GRKSpC5DQpLUZUhIkroMCUlSlyEhSeoyJCRJXYvnewCaXyu3f3detvv8bZ+cl+1KOjsX/JlEkg1Jnk4ylWT7fI9HkhaSCzokkiwCvgpcB6wBbkyyZn5HJUkLx4V+uelqYKqqngVIshfYCDw5r6PSyObrMhd4qUs6Gxd6SCwDXhyaPwJ8dGanJFuBrW32/yR5eg7bugL4qzksd7G56I9DvnLGLhf9MXibPA4L6xj8o9mKF3pIvC1VtRPYOco6kkxW1dpzNKR3LI+Dx+AUj4PHAC7wexLAUWDF0PzyVpMkjcGFHhIPAauTrEpyCbAJ2DfPY5KkBeOCvtxUVSeT3AwcABYBu6vq8Hna3EiXqy4iHgePwSkeB48Bqar5HoMk6QJ1oV9ukiTNI0NCktRlSLBwf/ojyfNJHk/yaJLJVrs8ycEkz7T3y+Z7nOdakt1JjiV5Yqg2635n4I722XgsyVXzN/Jzp3MM/lOSo+3z8GiS64fabmnH4Okk187PqM+tJCuSPJDkySSHk3yh1RfUZ+FMFnxI+NMf/POq+tDQs+DbgfurajVwf5u/2NwFbJhR6+33dcDq9toK3DmmMZ5vd/HWYwBwe/s8fKiq9gO0v4dNwAfbMl9rfzfvdCeB36mqNcA6YFvb14X2WTitBR8SDP30R1W9AZz66Y+FaiOwp03vAW6Yx7GcF1X1A+D4jHJvvzcCd9fAIWBJkvePZ6TnT+cY9GwE9lbVz6rqOWCKwd/NO1pVvVRVP2rTfw08xeBXHhbUZ+FMDInZf/pj2TyNZdwK+NMkD7efNgFYWlUvtemXgaXzM7Sx6+33Qvt83NwupeweutR40R+DJCuBDwMP4mfh7zAkFrZ/VlVXMTiN3pbk14Yba/B89IJ7Rnqh7jeDyye/DHwIeAn4r/M7nPFI8kvAHwO/XVWvDbct4M/C3zIkFvBPf1TV0fZ+DPgOg0sIr5w6hW7vx+ZvhGPV2+8F8/moqleq6s2q+hvgv/PzS0oX7TFI8gsMAuIbVfXtVl7wn4VhhsQC/emPJO9O8g9OTQPXAE8w2PfNrdtm4L75GeHY9fZ7H3BTe7JlHfDq0KWIi8qM6+v/ksHnAQbHYFOSS5OsYnDj9ofjHt+5liTALuCpqvqDoaYF/1kYdkH/LMc4jPmnPy4kS4HvDP5OWAz8z6r6kyQPAfck2QK8AHxmHsd4XiT5JvBx4IokR4AdwG3Mvt/7gesZ3Kx9Hfjs2Ad8HnSOwceTfIjB5ZXngX8LUFWHk9zD4P/jchLYVlVvzse4z7GPAb8JPJ7k0Vb7fRbYZ+FM/FkOSVKXl5skSV2GhCSpy5CQJHUZEpKkLkNCktRlSEiSugwJSVLX/wf2n994PegxHwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(hits.price)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merged2 = df_merged[(df_merged.price>=11) & (df_merged.price<=24)] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "nuXyRvPCuYN_"
   },
   "outputs": [],
   "source": [
    "md = np.median(hits[~hits.price.isnull()].price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4z5CRnnltqH_",
    "outputId": "4f1cadbf-c900-4472-9100-42db334b2d1b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1714266"
      ]
     },
     "execution_count": 48,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(hits['price'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "77746187"
   },
   "outputs": [],
   "source": [
    "df_rm_dup = df_merged.drop_duplicates(['customerVisitorId', 'ID', 'price', 'customerSessionId'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "swansonhealthproduction    9.164339e-01\n",
       "Swanson Premium            1.230345e-02\n",
       "Swanson Ultra              8.300530e-03\n",
       "default                    6.516277e-03\n",
       "NOW Foods                  4.755789e-03\n",
       "                               ...     \n",
       "Ostrim                     9.505875e-07\n",
       "Inholtra                   9.505875e-07\n",
       "Maximum International      9.505875e-07\n",
       "Baie Run                   9.505875e-07\n",
       "Superfood For Skin         9.505875e-07\n",
       "Name: collection, Length: 905, dtype: float64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_rm_dup.collection.value_counts(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ZChVgmS3tXV_",
    "outputId": "7c33af4e-2c09-4a2e-c41f-d98f7f2a8a3c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 1052318 entries, 0 to 1714602\n",
      "Data columns (total 4 columns):\n",
      " #   Column      Non-Null Count    Dtype  \n",
      "---  ------      --------------    -----  \n",
      " 0   ID          1051981 non-null  object \n",
      " 1   price       87909 non-null    float64\n",
      " 2   collection  1051981 non-null  object \n",
      " 3   name        87910 non-null    object \n",
      "dtypes: float64(1), object(3)\n",
      "memory usage: 40.1+ MB\n"
     ]
    }
   ],
   "source": [
    "df_rm_dup[['ID', 'price', 'collection', 'name']].info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2441          NaN\n",
       "2442       CSN193\n",
       "2443          NaN\n",
       "2444       SWF116\n",
       "2445       SWF068\n",
       "            ...  \n",
       "1714334    CRL396\n",
       "1714335    SW1453\n",
       "1714336    SWU248\n",
       "1714337    NWF270\n",
       "1714338    NWF139\n",
       "Name: ID, Length: 964408, dtype: object"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_rm_dup[df_rm_dup.name.isnull()].ID"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create dictionary of item ID, name, price, collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "mult_collection = df_rm_dup.groupby('ID')['collection'].nunique().sort_values(ascending=False).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>collection</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SW923</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SW1371</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>SW1113</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>SWA051</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>SW1782</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17475</th>\n",
       "      <td>KND004</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17476</th>\n",
       "      <td>SN135</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17477</th>\n",
       "      <td>NWF1044</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17478</th>\n",
       "      <td>SN132</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17479</th>\n",
       "      <td>NCL095</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17480 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            ID  collection\n",
       "0        SW923           7\n",
       "1       SW1371           6\n",
       "2       SW1113           6\n",
       "3       SWA051           6\n",
       "4       SW1782           6\n",
       "...        ...         ...\n",
       "17475   KND004           1\n",
       "17476    SN135           1\n",
       "17477  NWF1044           1\n",
       "17478    SN132           1\n",
       "17479   NCL095           1\n",
       "\n",
       "[17480 rows x 2 columns]"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mult_collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['swansonhealthproduction', 'Swanson Premium', 'default', 'Kal',\n",
       "       'Life Extension', 'Swanson Ultra', 'Swanson Superior Herbs'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hits[hits.ID=='SW923'].collection.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "dic = {}\n",
    "for i in range(len(df_rm_dup)):\n",
    "    prod = df_rm_dup.iloc[i]\n",
    "    if prod.ID not in dic.keys() and str(prod.sku) != 'nan':\n",
    "        dic[prod.ID] = {}\n",
    "        dic[prod.ID]['name'] = prod['name']\n",
    "        dic[prod.ID]['collection'] = prod.collection\n",
    "        dic[prod.ID]['price'] = prod.price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "eb8da79b",
    "outputId": "df749e44-53cc-4830-d05c-db43ee77dc21"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ipykernel/__main__.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  if __name__ == '__main__':\n",
      "/home/ec2-user/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/ipykernel/__main__.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    }
   ],
   "source": [
    "df_rm_dup['totals.totalOrderQty'] = df_rm_dup['totals.totalOrderQty'].fillna(0)\n",
    "df_rm_dup['totals.totalOrderQty'] = df_rm_dup['totals.totalOrderQty'].astype(int)\n",
    "num_purchase = df_rm_dup.groupby(['customerVisitorId', 'ID']).agg({'totals.totalOrderQty':'sum', 'price':'mean', 'customerSessionId':'nunique', 'collection':'first'})\n",
    "num_purchase = num_purchase.reset_index().rename(columns={\"totals.totalOrderQty\":\"numPurchase\"})\n",
    "num_purchase['Purchase'] = num_purchase.numPurchase.apply(lambda x: 1 if x!=0 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "cust_collection = num_purchase.groupby(\"customerVisitorId\")['collection'].apply(list).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "cust_collection['contains_swansonhealth'] = cust_collection['collection'].apply(lambda x: 'swansonhealthproduction' not in x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3ae49372"
   },
   "outputs": [],
   "source": [
    "num_purchase['price'] = np.where(num_purchase.price.isnull(), md, num_purchase.price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_purchase['price_category'] = np.where(num_purchase.price>=94, 'very expensive', np.where((num_purchase.price>= 48) & (num_purchase.price<94), 'expensive', np.where((num_purchase.price>= 24) & (num_purchase.price<48), 'medium', 'cheap')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>customerVisitorId</th>\n",
       "      <th>ID</th>\n",
       "      <th>numPurchase</th>\n",
       "      <th>price</th>\n",
       "      <th>customerSessionId</th>\n",
       "      <th>collection</th>\n",
       "      <th>Purchase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cjo06cah000013ea3jqwotb8g</td>\n",
       "      <td>AAM013</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>swansonhealthproduction</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cjo06cah000013ea3jqwotb8g</td>\n",
       "      <td>AAM016</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>swansonhealthproduction</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cjo06cah000013ea3jqwotb8g</td>\n",
       "      <td>AAM019</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>swansonhealthproduction</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cjo06cah000013ea3jqwotb8g</td>\n",
       "      <td>ARG010</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>swansonhealthproduction</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cjo06cah000013ea3jqwotb8g</td>\n",
       "      <td>DB088</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>swansonhealthproduction</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           customerVisitorId      ID  numPurchase  price  customerSessionId  \\\n",
       "0  cjo06cah000013ea3jqwotb8g  AAM013            0    NaN                  2   \n",
       "1  cjo06cah000013ea3jqwotb8g  AAM016            0    NaN                  2   \n",
       "2  cjo06cah000013ea3jqwotb8g  AAM019            0    NaN                  2   \n",
       "3  cjo06cah000013ea3jqwotb8g  ARG010            0    NaN                  2   \n",
       "4  cjo06cah000013ea3jqwotb8g   DB088            0    NaN                  2   \n",
       "\n",
       "                collection  Purchase  \n",
       "0  swansonhealthproduction         0  \n",
       "1  swansonhealthproduction         0  \n",
       "2  swansonhealthproduction         0  \n",
       "3  swansonhealthproduction         0  \n",
       "4  swansonhealthproduction         0  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_purchase.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysis of product price category and number of sessions with regards to purchase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "price_category\n",
       "cheap             1.019692\n",
       "expensive         1.066176\n",
       "medium            1.090868\n",
       "very expensive    1.000000\n",
       "Name: customerSessionId, dtype: float64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_purchase[num_purchase.Purchase==1].groupby('price_category')['customerSessionId'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Groupby Data Filter by Category')"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEWCAYAAACe8xtsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAjiUlEQVR4nO3de5gcVZ3/8feHkMCQRCNkUBMMAdEICAYcUQQRUUHFSwQVFVRYNV52xQuGH+iugisCm5UFRFkQuagIKIaowBpQN3IPTAgQbsEFwiVBCJdACCOQ8P39UadJT6d7pmoy1d2T+byep56uPl116lRVV327zqk6rYjAzMysiA1aXQAzMxt6HDzMzKwwBw8zMyvMwcPMzApz8DAzs8IcPMzMrDAHDytM0sGSrmp1Ocom6WlJW6fxsyV9v8Rl7SnpwbLyNxtsDh5DhKSPS5onaaWkR9L4lyWp1WVbF5LmSvqHpBWSnpI0X9IRkjYqkEdI2mYdyrBYUk8KFpVhQkSMiYh76kzf9id6SbtIulTSckmPS7pe0iE5550r6XNll9GGNgePIUDSYcBJwEzgFcDLgS8CuwGjGswzomkFXHf/EhFjgVcChwEfBy5tcmD8QAoWlWFpWQuStGFZeaf8dwX+AvwV2AbYDPgS8N4yl7uulPE5aYjwjmpzkl4KfA/4ckRcGBErIrMgIg6MiGfTdGdLOjX92lwJvEPStulX5HJJt0n6YFW+vX5d1lZFpV/zh0q6R9KjkmbWHNiS9CNJT0q6U9I7U+JHJc2vWYfDJM3ub10jYmVEzAU+COwK7Jvm30XStWk9HpJ0iqRR6bMr0uw3pyuGAyS9TNLFkpZJeiKNb5F/q/faBtvUpI0G/geYUH2VImmDdMV0t6THJP1a0qZpnskpr89Kup/sxN5omd9K23uxpANT2pskPVwddCTtL+mmBtnMBM6JiOMj4tH0fZkfER9L8zbcPpKOAd4GnJLW7ZSU/jpJl6ermEWSPlZVls0k/SFdOd4g6fs136W3pvQn0+tbqz6bK+kYSVcDzwCHDfT7Y00WER7aeADeA6wCNuxnurOBJ8muRjYAxgL/B3yL7OpkL2AFMCVNPxf4XNX8BwNXVb0P4H+BTYFJwF2V6dO0q4CvAyOBA9KyNwU2Ah4Htq3KawGwf4Ny9ypHVfoVwPFp/I3AW4ANgcnAHcDXasq6TdX7zYD9gU3SdvgNMLuPbbcYeFed9BfzTdv3+2l8T+DBmmm/BlwHbJG2wWnAeemzySmvnwOjgY46y9ozbdMT0vxvB1ZW7a/bgfdWTX8RcFidfDYBVgPv6GN9+9w+db4bo4EHgEPSPtgZeBTYPn1+fho2AbZL016VPtsUeAL4VJr3E+n9ZlXLuh/YPn1e6PvjoXWDrzza33jg0YhYVUmQdE36Fd4jaY+qaX8XEVdHxAvAVGAMcFxEPBcRfwEuJjt48zo+Ih6PiPuBE2vmfQQ4MSKej4gLgEXAvpFdCV0AHJTKuj3ZyfPiQmsNS8lOPET2q/m6iFgVEYvJTsxvbzRjRDwWEb+NiGciYgVwTF/TJ7PTNl0+wF+5XwC+HREPpm1wFPCRmiqqoyK7uurpI59/i4hnI+KvwCVA5Rf+OazZppsC+wC/qjP/y8h+PDzUaAED2D7vBxZHxFlpH9wI/Dat3wiyQPTdlN/tqawV+wJ/i4hfpHnPA+4EPlA1zdkRcVv6fLC+P1YyB4/29xgwvvokFBFvjYhx6bPqffhA1fgE4IEUSCruAyYWWHZ1fvelPCuWREQ0+Pwc4JOSRPaL89fppFDERLJfoEh6bapa+bukp4AfkAXVuiRtIuk0Sfel6a8AxqnvdqBpETEuDdMKlhVgS+CiSgAiuzpaTdY+VfFAvRmrPBERK6veV2/TXwIfkDSGLKBcGRH1AsQTwAtk7Ud1DWD7bAm8uSq4LgcOJGt/6yS7Yqhet9rv4X01+dV+D2u3y2B8f6xkDh7t71rgWeBDOaatPpkvBV5V004xCViSxleSVTNUvKJOfq+qmbe6EXliOrjX+jwirgOeI6s7/yTwixxlf5GkV5FVVV2Zkk4l+7X6moh4CVlVXF+N6YcBU4A3p+krV2eD1QBfryvqB8iqlcZVDRtHxJJ+5qv2stSmUlG9TZeQfRc+THZCrbtNI+KZNN3+fSynv+1TW84HgL/WrNuYiPgSsIysuq26Tan6e7OULPhUq/4errW8df3+WHM4eLS5iFgOHA38RNJHJI1JjbNTyeqiG5lHFiAOlzRS0p5kVQXnp89vAvZLv0K3AT5bJ48ZqXH1VcBXyaoTKjYHDk15fxTYFri06vOfA6cAqyIi1zMhqSxvB34HXF+V31jgKeBpSa8ju3Oo2sPA1lXvxwI9wPJUxfPdPMsv4GFgM2U3M1T8N3CMpC3TunRKyhPwax0taZSkt5FVF/2m6rOfA4cDO5C1eTRyOHCwpBmSNkvleYOkyr7vb/vUbs+LgddK+lTa3yNTI/62EbEamAUclfbf64BPV817aZr3k5I2lHQAWbtIf9VQhb8/1mStbnTxkG8gqya4nuyOlGVkwWE6MCp9fjapQbdqnu3Jbtd8kqzB9cNVn40HLiNrRL+arI6+tsH8UOAesuqxHwIj0mcHp3lOSXnfBexds+xJZNUnR/ezXnOBf6RyrCBrHP02sHHVNHuQXXk8TXY18r2asn6RrI5/OVmVzoSU79OpbF9I61P3pgMKNpin92em7bI8LW8D4BtkbT8rgLuBH6RpJ/e1/DTNnsCDad0fJWtE/lTNNJuQBdFzcnxfdiG7K+xJsuq/ecCn02d9bh+yO93uIqsCOzmlTSFrg1mW1vsvwNT0WWf67CngBuB44M9VZdkdmJ/KMh/YvWb/17thItf3x0PrBqUdZdaLpCCrJvq/Ac7fQdaovnNE/G1QCzeMSbob+EJE/KnVZWlE0vHAKyLiM+uQh78/bc7VVlaWLwE3+MAfPJL2J7tCaPicSCukZ0B2VGYXsirQvqrV8vD3p82V+qSrDU+SFpM1vk5rbUnWH5LmkrUVfCp630HXDsYC55FVhz1CVsX5u4Fm5u/P0OBqKzMzK8zVVmZmVlhbVVuNHz8+Jk+e3OpimJkNGfPnz380Ijqbvdy2Ch6TJ0+mu7u71cUwMxsyJNU+wd8UrrYyM7PCHDzMzKwwBw8zMyvMwcPMzApz8DAzs8JKu9tK0hR698K6NfCdiDhxMJfz5mMu5+EVz734/uVjRzHv2+8ezEWYmVmN0q48ImJRREyNiKlk/83wDOve300vtYED4OEVz/HmYy4fzMWYmVmNZlVbvRO4OyIG9X7k2sDRX7qZmQ2OZgWPj5N1nLYWSdMldUvqXrZsWZOKY2Zm66L04CFpFPBBev8j2osi4vSI6IqIrs7Opj9hb2ZmA9CMK4/3AjdGxMODnfHLx44qlG5mZoOjGcHjEzSoslpX87797rUChe+2MjMrX6kdI0raBHg32X8kl8KBwsys+UoNHhHxDLBZmcswM7Pm8xPmZmZWmIOHmZkV5uBhZmaFOXiYmVlhDh5mZlaYg4eZmRXm4GFmZoU5eJiZWWEOHmZmVpiDh5mZFebgYWZmhTl4mJlZYQ4eZmZWmIOHmZkV5uBhZmaFOXiYmVlhDh5mZlaYg4eZmRXm4GFmZoU5eJiZWWEOHmZmVpiDh5mZFebgYWZmhTl4mJlZYaUGD0njJF0o6U5Jd0jatczlmZlZc2xYcv4nAX+MiI9IGgVsUvLyzMysCUoLHpJeAuwBHAwQEc8Bz5W1PDMza54yq622BpYBZ0laIOkMSaNrJ5I0XVK3pO5ly5aVWBwzMxssZQaPDYGdgVMjYidgJXBE7UQRcXpEdEVEV2dnZ4nFMTOzwVJm8HgQeDAi5qX3F5IFEzMzG+JKCx4R8XfgAUlTUtI7gdvLWp6ZmTVP2XdbfQU4N91pdQ9wSMnLMzOzJig1eETETUBXmcswM7Pm8xPmZmZWmIOHmZkV5uBhZmaFOXiYmVlhDh5mZlaYg4eZmRXm4GFmZoU5eJiZWWEOHmZmVpiDh5mZFebgYWZmhTl4mJlZYQ4eZmZWWL/BQ9Julb+PlXSQpBMkbVl+0czMrF3lufI4FXhG0huAw4H7gJ+XWiozM2treYLHqogI4EPASRFxEjC23GKZmVk7y/NnUCskHQkcBOwhaQQwstximZlZO8tz5XEA8Czw2fS/5BOBmaWWyszM2lq/Vx4pYJxQ9f5+3OZhZjas5bnbaj9Jf5P0pKSnJK2Q9FQzCmdmZu0pT5vHfwAfiIg7yi6MmZkNDXnaPB524DAzs2p5rjy6JV0AzCZrOAcgImaVVSgzM2tveYLHS4BngL2r0gJw8DAzG6by3G11yEAzl7QYWAGsJnvYsGugeTVy4E+v5eq7H3/x/W6v3pRzP7/rYC/GzMyq5LnbagtJF0l6RNLDkn4raYsCy3hHRExtRuAAuPruxznwp9cO9qLMzKxKngbzs4DfAxPIHhD8Q0prudrA0V+6mZkNjjzBozMizoqIVWk4G+jMmX8Al0maL2l6vQkkTZfULal72bJlObM1M7NWyhM8Hk1dsY9Iw0HAYznz3y0idgbeC/yzpD1qJ4iI0yOiKyK6OjvzxiQzM2ulPMHjn4CPAX8HHgI+ktL6FRFL0+sjwEXALgMrZn27vXrTQulmZjY4+g0eEXF/RHwwIjojYvOImBYR9/U3n6TRksZWxslu9b113Yu8xrmf33WtQOG7rczMytfwVl1Jh0fEf0j6EVnbRS8RcWg/eb8cuEhSZTm/iog/rkth63GgMDNrvr6e86h0SdI9kIwj4h7gDQOZ18zM2lvD4BERf0iv51TSJG0AjIkI96prZjaM5XlI8FeSXpLaLW4HFkmaUX7RzMysXeW522q7dKUxDbgUmAR8qsxCmZlZe8sTPEZKGkkWPH4XEc9TpwHdzMyGjzzB4zRgMTAauELSloDbPMzMhrE8veqeDJxclXSfpHeUVyQzM2t3eRrMv5oazCXpZ5JuBPZqQtnMzKxN5eqeJDWY703WIeIhwHGllsrMzNpanuCh9Po+4KyIuLkqzczMhqE8wWO+pMvIgsec1F/VC+UWy8zM2lme/zD/LDAVuCcinpG0GVnVlZmZDVN5rjwC2A6odIQ4Gti4tBKZmVnbyxM8fgLsCnwivV8B/Li0EpmZWdvLU2315ojYWdICgIh4QtKokstlZmZtLM+Vx/OSRpC6JJHUiRvMzcyGtTzB42Syv5DdXNIxwFXAD0otlZmZtbU83ZOcK2k+8E6y5zumRcQd/cxmZmbrsTzdk7wauDcifkz2H+TvljSu7IKZmVn7ylNt9VtgtaRtgDOArYBflVoqMzNra3mCxwsRsQrYDzgpIr4OvLLcYpmZWTvLe7fVJ4BPAxentJHlFcnMzNpdnuBxCNlDgsdExL2StgJ+WW6xzMysneW52+p21nRNQkTci7tkNzMb1vJceZiZmfWSp3uSdZKeTu8GlkTE+wc7/8lHXLJW2uLj9h3sxZgNebMXLGHmnEUsXd7DhHEdzNhnCtN2mtjqYtkQ1fDKQ9Iv0utX13EZXwVKeaiwXuDoK91suJq9YAlHzlrIkuU9BLBkeQ9HzlrI7AVLWl00G6L6qrZ6o6QtgX+S9DJJm1YPeTKXtAWwL9nzIWbWIjPnLKLn+dW90nqeX83MOYtaVCIb6vqqtvpv4I/A1sB8ev/1bKT0/pwIHA6MbTSBpOnAdIBJkyblyNLMilq6vKdQull/Gl55RMTJEbEtcGZEbB0RW1UN/QYOSe8HHomI+X1NFxGnR0RXRHR1dnYWXwMz69eEcR2F0s360+/dVhHxJUlvkPQvadgxZ967AR+UtBg4H9hLkp8PMWuBGftMoWPkiF5pHSNHMGOfKS0qkQ11eTpGPBQ4F9g8DedK+kp/80XEkRGxRURMBj4O/CUiDlrH8vbS6K4q321l1tu0nSZy7H47MHFcBwImjuvg2P128N1WNmCKiL4nkG4Bdo2Ilen9aODaiMh7BYKkPYFv9nerbldXV3R3d+fN1sxs2JM0PyK6mr3cPM95CKi+TWM1vRvP+xURc4G5ReYxM7P2lSd4nAXMk3RRej8N+FlpJTIzs7aXp2+rEyTNBXYnu+I4JCIWlF0wMzNrX7m6J4mIG4EbSy6LmZkNEe4Y0czMCnPwMDOzwvoMHpJGSPpTswpjZmZDQ5/BIyJWA89IemmTymNmZkNAngbzfwALJV0OrKwkRsShjWcxM7P1WZ7gcUkazMzMgHzPeZwjqQOYFBHu/N/MzHJ1jPgB4Cay//ZA0lRJvy+5XGZm1sby3Kp7FLALsBwgIm4CtiqtRGZm1vbyBI9VEfFkTVrfXfGamdl6LU+D+a2SPgmMkPQa4FDgmnKLZWZm7SzPlcdXgO2BZ4HzgKeAr5VYJjMza3N57rZ6Bvi2pOOzt7Gi/GKZmVk7y3O31ZskLQRuIXtY8GZJbyy/aGZm1q7ytHn8DPhyRFwJIGl3sj+Iyv03tGZmtn7J0+axohI4ACLiKsBVV2Zmw1jDKw9JO6fR6yWdRtZYHsAB+P/IzcyGtb6qrX5Y8/67VeN+zsPMbBhrGDwi4h3NLIiZmQ0d/TaYSxoHfBqYXD29u2Q3Mxu+8txtdSlwHbAQeKHc4piZ2VCQJ3hsHBHfKJqxpI2BK4CN0nIujIjv9j1XcZOPWPuvRhYft+9gL8YKmL1gCTPnLGLp8h4mjOtgxj5TmLbTxFYXa9jb6ohLejVWCrjXx4oNUJ5bdX8h6fOSXilp08qQY75ngb0i4g3AVOA9kt6yLoWtVS9w9JVu5Zu9YAlHzlrIkuU9BLBkeQ9HzlrI7AVLWl20Ya02cEB218tWPlZsgPIEj+eAmcC1wPw0dPc3U2SeTm9HpsF3aa3nZs5ZRM/zq3ul9Ty/mplz/D9irdTowPMBaQOVp9rqG8A2EfFo0cwljSALNtsAP46IeXWmmQ5MB5g0aVLRRVibWbq8p1C6mQ1Nea48bgOeGUjmEbE6IqYCWwC7SHp9nWlOj4iuiOjq7OwcyGKsjUwY11Eo3cyGpjzBYzVwk6TTJJ1cGYosJCKWkz2V/p7iRbShZMY+U+gYOaJXWsfIEczYZ0qLSmSQNY4XSTfrT57gMRs4huwPoOZXDX2S1JmeEUFSB/Au4M6BFrSeRndV+W6r1pm200SO3W8HJo7rQMDEcR0cu98Ovtuqxe49bt+1AoXvtrJ1oYhymswk7QicA4wgC1K/jojv9TVPV1dXdHf32xZvZmaJpPkR0dXs5eZ5wvxe6tyUERFb9zVfRNwC7DTwopmZWbvKc7dVdUTbGPgokOc5DzMzW0/12+YREY9VDUsi4kRgr/KLZmZm7SpPtdXOVW83ILsSGVtaiczMrO3lqbaq/l+PVcBi4GOllMbMzIaEfoOH/9fDzMxq5am22gjYn7X/z6PP227NzGz9lafa6nfAk2QPBj5bbnHMzGwoyBM8togIdytiZmYvytM9yTWSdii9JGZmNmTkufLYHTg4PWn+LFmXOBERO5ZaMjMza1t5gsd7Sy+FmZkNKXlu1b2vGQUxM7OhI0+bh5mZWS8OHmZmVpiDh5mZFebgYWZmhTl4mJlZYQ4eZmZWmIOHmZkV5uBhZmaFOXiYmVlhDh5mZlaYg4eZmRXm4GFmZoXl6VV3QCS9Cvg58ArgBeD0iDhpsJcz+YhL1kpbfNy+g70YK8D7pD15v7Sf2QuWMHPOIpYu72HCuA5m7DOFaTtNbHWxcinzymMVcFhEbAu8BfhnSdsN5gLqHQx9pVv5vE/ak/dL+5m9YAlHzlrIkuU9BLBkeQ9HzlrI7AVLWl20XEoLHhHxUETcmMZXAHcAQyOkmpmVbOacRfQ8v7pXWs/zq5k5Z1GLSlRMU9o8JE0GdgLm1flsuqRuSd3Lli1rRnHMzFpu6fKeQuntpvTgIWkM8FvgaxHxVO3nEXF6RHRFRFdnZ2fZxTEzawsTxnUUSm83pQYPSSPJAse5ETGrzGWZmQ0lM/aZQsfIEb3SOkaOYMY+U1pUomJKCx6SBPwMuCMiTihjGY3uFPEdJK3jfdKevF/az7SdJnLsfjswcVwHAiaO6+DY/XYYMndbKSLKyVjaHbgSWEh2qy7AtyLi0kbzdHV1RXd3dynlMTNbH0maHxFdzV5uac95RMRVgMrK38zMWsdPmJuZWWEOHmZmVpiDh5mZFebgYWZmhTl4mJlZYQ4eZmZWmIOHmZkV5uBhZmaFOXiYmVlhDh5mZlaYg4eZmRXm4GFmZoU5eJiZWWEOHmZmVpiDh5mZFebgYWZmhTl4mJlZYQ4eZmZWmIOHmZkV5uBhZmaFOXiYmVlhDh5mZlaYg4eZmRXm4GFmZoVtWFbGks4E3g88EhGvL2s5k4+4ZK20xcftW9biLAfvk/bk/dJ+hvI+KfPK42zgPSXmX3fD95Vu5fM+aU/eL+1nqO+T0oJHRFwBPF5W/mZm1jotb/OQNF1St6TuZcuWtbo4ZmaWQ8uDR0ScHhFdEdHV2dnZ6uKYmVkOLQ8eZmY29Azp4NHoroShcrfC+sj7pD15v7Sfob5PFBHlZCydB+wJjAceBr4bET/ra56urq7o7u4upTxmZusjSfMjoqvZyy3tOY+I+ERZeZuZWWsN6WorMzNrDQcPMzMrzMHDzMwKc/AwM7PCSrvbaiAkLQPuG+Ds44FHB7E4tu68T9qT90v7WZd9smVENP0J67YKHutCUncrblezxrxP2pP3S/sZivvE1VZmZlaYg4eZmRW2PgWP01tdAFuL90l78n5pP0Nun6w3bR5mZtY869OVh5mZNYmDh5mZFdb2wUPS2ZI+0upyWHkkTZB0YavLMRxImiupK41fKmlci4tkTSDpmsHOs7Redc3yioilgH8gNFlEvK/VZRhKJImsnfiFVpelqIh462Dn2XZXHpI+LekWSTdL+kVK3kPSNZLuqb4KkTRD0g1p+qOr0mdLmi/pNknTq9KflvRDSTdK+rOkYf2/t5IOknS9pJsknSbpzWlbbixpdNp+r5e0p6QrJF0k6XZJ/y1pg5TH3pKuTdv0N5LGpPTFko5O6QslvS6lvz0t7yZJCySNlTRZ0q3p83mStq8q41xJb0zlOTPt7wWSPtSKbdYKafvcKekMSbdKOlfSuyRdLelvknZptH0kdUg6P+3XC4COqnwXSxpfvf1T+jclHZXG50r6r7T/75D0Jkmz0nK/3+xtsa4kHS/py1Xvj5J0WBpf63ySts0dkn4C3Aj8m6T/qpr/85JOqLOctY4LSVum7TZe0gaSrkzTVfbvOWnZF0raJOXzRkl/TeezOZJemdLnpnW5XtJdkt6W0revOqZvkfSalP50er1A0vuqynm2pP0ljZA0s2r9v9DvxoyIthmA7YFFwPj0flPgbOA3ZIFuO+D/0md7k93epvTZxcAelfnSawdwK7BZeh/AgWn8O8AprV7nFm7rbYE/ACPT+58Anwa+D/wn8GPgyPTZnsA/gK2BEcDlZFcK44ErgNFpuv8HfCeNLwa+ksa/DJyRxv8A7JbGx5Bd/U4Gbk1pXweOTuOvBO5K4z8ADkrj44C7Kstd34e0fVYBO6Tv+nzgzPTd/xAwu9H2Ab4BnJnSd0z5dFXto/HV2z+lfxM4Ko3PBY5P418Flqb9shHwYOXYGioDsBPw16r3twOTGp1P0rZ5AXhLmn40cHfVcXMNsEPNMvo6Lj4HXAjMAE6r2r9RdVycmfbByJR/Z0o/oGpfzgV+mMbfB/wpjf+INee4UUBHGn86vX4YOKfq8wfIzpPTgX9N6RsB3cBWfW3Ldqu22gu4MCIeBYiIxyUBzI7sUvF2SS9P0+6dhgXp/RjgNWQ77VBJH07pr0rpj5F9CS5I6b8EZpW7Om3tncAbgRvSNu4AHgG+B9xAFiwOrZr++oi4B178l8jd0zTbAVenPEYB11bNU9m+84H90vjVwAmSzgVmRcSDad6KX5MFp+8CHyP74QDZvv6gpG+m9xuTHfR3DGz1h5x7I2IhgKTbgD9HREhaSHby2YL622cP4GSAiLhF0i0DWPbv0+tC4LaIeCiV4x6y4+uxga1S80XEAkmbS5oAdAJPRMT9kg6l/vnkfuC+iLguzb9S0l+A90u6gyyILKxZzFtocFxExBmSPgp8EZhaNc8DEXF1Gv8l2bH3R+D1wOUpnxHAQ1XzVB9fk9P4tcC3JW1Bdnz9raZs/wOcLGkj4D3AFRHRI2lvYEetqdl5aVr/extsyrYLHiKLwLWerZmm8npsRJzWKwNpT+BdwK4R8YykuWQHUj3D+SEXkf0CObJXovQKsgNnJNl2W5k+qt1WkfK4PBr/a2Rlv60mfdci4jhJl5D9WrpO0rvIghDp8yWSHpO0I9kvrcrls4D9I2JR4TVdP1QfAy9UvX+BbNuups72SSed/r7nq+hdhV17vFQvq7Yc7XYOyeNCsivnVwDnp7RG55PJrDkGKs4AvgXcCZxVJ/+Gx0WqjtoivR0DrEjjjY6v2yJi1wbrUe/4+pWkecC+wBxJn4uIv7yYacQ/0jlxH7Lj67yqMn8lIuY0WNZa2q3N48/AxyRtBiBp0z6mnQP8k9bUsU+UtDlZxHwiBY7Xkf0KqNiANQ2znwSuGuwVGEL+DHwkbTMkbSppS7JL938DzgWOr5p+F0lbKWvrOIBs210H7CZpm5THJpJe29dCJb06IhZGxPFkl8avqzPZ+cDhwEurftXNAb6idDaUtNOA1nr91Wj7XAEcmNJeT1Z1VethYHNJm6VfpO9vQnlb6Xzg42Tngspdfo3OJ2uJiHlkV1yfZM3Jt1pfx8XxZMfWd4CfVs0zSVIlSHyC7PhaBHRW0iWNVFV7YD2StgbuiYiTya4Y6+3v84FDgLel9a6s/5ckjUz5vFbS6L6W1Va/GiLiNknHAH+VtJo1l5D1pr1M0rbAtel4eRo4iOxS74vp8nwR2Y6sWAlsL2k+8CTZSXBYiojbJf0rcFkKCM8DvwNWpV8vI4BrJO1F9gvzWuA4snr3K4CLIuIFSQcD56WTDsC/ktW3N/I1Se8g+7V0O9ll9CtrprkQOAn496q0fwdOBG5JJ8jFrP8nuSIabZ9TgbPS8XATcH3tjBHxvKTvAfPIqinubE6RWyOdZ8YCSypVcH2cT1Y3yObXwNSIeKJO/svqHRepsftNZG0bq1ND9SHA/5JVv35G0mnA34BTI+K5VI10sqSXkp2vTwRu62P1DgAOkvQ88HeyauhalwE/B34fEc+ltDPIqr5uTN+fZcC0PpYzvLonkfR0RIxpdTmGmlQV+M2I8MnaDJB0MfBfEfHnQchrMnBxRLx+nQvWRO1WbWVm1rYkjZN0F9AzGIFjKBtWVx5mZjY4fOVhZmaFOXiYmVlhDh5mZlaYg4cNeZK+lx42bOYy95Q06J3NmQ0VbfWch1lRkkZExHdasOg9yZ4FGPSurivS/fZDshdXW//5ysPakvruaXSxpO9Iugr4qKr+80VZr6/XKOuV+Xplvfbm6jFUdXp0lvQBZT39LpD0J0kvT/flfxH4urLeS98mqVPSb9MybpC0W5q/U9LlynpXPU3SfZLGp8++oayX3Fslfa1qvQv34mrWdK3u5dKDh3oDDXoaTeOLgcOrpj2brKuJUcA9wJtS+kvIrq777TGUOj06p9eXseaW9s+xpifToyrlSe9/BeyexicBd6TxU1jTO/F70jqNJ+uUciFZL61jyJ4a3okB9OLqwUMrBldbWTur19Pof6b3F9SZfgrwUETcABART0H23wr032PoWj06p/QtgAtS1xKjaNzL6LuA7bSmh+CXpC4wdifrBpuI+KOkSncWu5N18bIylXEWWV9Dv6d4L65mTefgYe2sXk+jFbU9nULjXpnz9BjaaN4fASdExO9TNy1HNZh/A7KenHt6Zare/c3XLK+Ror24mjWd2zysndXrabQvdwITJL0JILV3bEi+HkMb9ej8UmBJGv9M1fQrgLFV7y8D/qXyRtLUNHoV2f+SVK6AXpbSrwCmKetxdTTZ1cmV9VYq+u/F1azpHDysnVV6Gr2F7F8lT+1r4sh6CD0A+JGkm8n+VGpjsl/ut5P1GHorcBo1V90RcRtQ6dH5ZqDSKH0U8BtJVwKPVs3yB+DDlQZzsiq1rtTgfjtZgzrA0cDekm4E3kv2Zz4rIuJGsraa68l6sz0jIhr2Ik3Wi+vVUacXV7NWcN9W1paGak+jtVKX3KsjYlW6ijo1IqYOIJ9B68XVbDC4zcOsXJOAXyv7z5TngM8XmVnSOLKrk5sdOKyd+MrDzMwKc5uHmZkV5uBhZmaFOXiYmVlhDh5mZlaYg4eZmRX2/wGj92f/HmbRuwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(num_purchase[(num_purchase.collection=='swansonhealthproduction')].price_category, num_purchase[num_purchase.collection=='swansonhealthproduction'].customerSessionId)\n",
    "plt.xlabel('price category')\n",
    "plt.ylabel('number of sessions')\n",
    "plt.title('Groupby Data Filter by Category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2649"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_purchase[(num_purchase.Purchase==1) & (num_purchase.price>=11) & (num_purchase.price<=24)].ID.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "price_category\n",
       "cheap             986914\n",
       "expensive           1550\n",
       "medium              8772\n",
       "very expensive       128\n",
       "Name: ID, dtype: int64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_purchase.groupby('price_category')['ID'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fac11f4fef0>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAASTklEQVR4nO3df2zcd33H8dcrrseOUnC7mCpx04Wh4qlZtQSsdCgSKnRgfqn1QKzJAHUTWvZHN4GGPJopEkFqRaUM1knb0EpbCGpJiUpqyoYIpQOxMZLi1GVu2noNpSSxsyasWMBqMeO+94e/Tpz4rr6z7/y97+f7fEjW3b3ve3fvk5KXv/58P9/P1xEhAEBaVuXdAACg+Qh3AEgQ4Q4ACSLcASBBhDsAJOiCvBuQpNWrV8f69evzbgMACuXw4cM/iYjuas+1RbivX79ew8PDebcBAIVi+8e1nmNYBgASRLgDQIIIdwBIEOEOAAki3AEgQW0xW2apdg6Nau+h45qJUIetbVev0y0DV+XdFgDkrrDhvnNoVPccPHbm8UzEmccEPICyK+ywzN5DxxuqA0CZFDbcZ2qsQ1+rDgBlUthw77AbqgNAmRQ23Lddva6hOgCUSWEPqM4dNGW2DAAs5Ha4hmpfX1+wcBgANMb24Yjoq/ZcYYdlAAC1Ee4AkCDCHQASRLgDQIIIdwBIEOEOAAki3AEgQYQ7ACSosGeoStLQyLh2HxjTxOSU1nZVNNjfq4FNPXm3BQC5K2y4D42Ma8f+UU1Nz0iSxientGP/qCQR8ABKr7DDMrsPjJ0J9jlT0zPafWAsp44AoH0UNtwnJqcaqgNAmRQ23Nd2VRqqA0CZFDbcB/t7VensOKdW6ezQYH9vTh0BQPso7AHVuYOmzJYBgIUKG+7SbMAT5gCwUGGHZQAAtRHuAJAgwh0AEkS4A0CCFg132+tsf8v2k7aP2P5wVr/E9kO2n85uL573mh22j9oes93fyi8AAFiontkyv5L00Yh41PZFkg7bfkjSH0t6OCJus32zpJslfcz2lZK2Stogaa2kb9p+XUTM1Hj/JXv/Z7+n7/7w+TOPt7z2Et37p29s9scAQOEsuuceEScj4tHs/s8lPSmpR9L1kvZkm+2RNJDdv17SfRHxy4j4kaSjkjY3ue8FwS5J3/3h83r/Z7/X7I8CgMJpaMzd9npJmyQdknRpRJyUZn8BSHp1tlmPpOPzXnYiq53/XtttD9sePn36dMONnx/si9UBoEzqDnfbr5D0ZUkfiYifvdSmVWqxoBBxR0T0RURfd3d3vW0AAOpQV7jb7tRssN8bEfuz8nO212TPr5F0KqufkLRu3ssvkzTRnHYBAPWoZ7aMJd0l6cmI+PS8px6UdGN2/0ZJX5lX32r7ZbZfI+kKSY80r2UAwGLq2XPfIumDkt5i+7Hs552SbpP0VttPS3pr9lgRcUTSPklPSPq6pJtaMVNmy2svaagOAGXiiAXD4Suur68vhoeHG34dUyEBlJntwxHRV+25Qq8KSZADQHUsPwAACSLcASBBhDsAJIhwB4AEEe4AkCDCHQASRLgDQIIIdwBIEOEOAAki3AEgQYQ7ACSIcAeABBHuAJAgwh0AElToJX+HRsa1+8CYJiantLarosH+Xg1sWnAtbgAoncKG+9DIuHbsH9XU9OxFnsYnp7Rj/6gkEfAASq+wwzK7D4ydCfY5U9Mz2n1gLKeOAKB9FDbcJyanGqoDQJkUNtzXdlUaqgNAmRQ23Af7e1Xp7DinVuns0GB/b04dAUD7KOwB1bmDpsyWAYCFChvu0mzAE+YAsFBhh2UAALUR7gCQIMIdABJEuANAggh3AEgQ4Q4ACSLcASBBhDsAJIhwB4AEEe4AkCDCHQAStGi4277b9inbj8+r7bI9bvux7Oed857bYfuo7THb/a1qHABQWz0Lh31e0t9L+sJ59b+NiL+ZX7B9paStkjZIWivpm7ZfFxEzagGuoQoA1S265x4R35H0fJ3vd72k+yLilxHxI0lHJW1eRn81zV1DdXxySqGz11AdGhlvxccBQKEsZ8z9z23/ZzZsc3FW65F0fN42J7LaAra32x62PXz69OmGP5xrqAJAbUsN989Ieq2kjZJOSvpUVneVbaPaG0TEHRHRFxF93d3dDTfANVQBoLYlhXtEPBcRMxHxoqTP6uzQywlJ6+ZtepmkieW1WN2rKp0N1QGgTJYU7rbXzHv4B5LmZtI8KGmr7ZfZfo2kKyQ9srwWa/XQWB0AymTR2TK290q6RtJq2yckfVzSNbY3anbI5VlJfyZJEXHE9j5JT0j6laSbWjVT5qcvTDdUB4AyWTTcI2JblfJdL7H9rZJuXU5T9eiwNRMLh/M72HUHgOKeoVot2F+qDgBlUthwBwDURrgDQIIIdwBIEOEOAAki3AEgQYQ7ACSIcAeABBHuAJAgwh0AEkS4A0CCCHcASBDhDgAJItwBIEGEOwAkiHAHgAQR7gCQIMIdABJEuANAggh3AEhQYcO9p6vSUB0AyqSw4b7+N6qHeK06AJRJYcP94DM/bagOAGVS2HCfiWioDgBlUthwtxurA0CZFDbca+2gs+MOAAUOdwBAbYQ7ACSIcAeABBHuAJAgwh0AEkS4A0CCCHcASNCi4W77btunbD8+r3aJ7YdsP53dXjzvuR22j9oes93fqsYBALXVs+f+eUlvP692s6SHI+IKSQ9nj2X7SklbJW3IXvOPtjua1i0AoC4XLLZBRHzH9vrzytdLuia7v0fStyV9LKvfFxG/lPQj20clbZb0vSb1W2pDI+PafWBME5NTWttV0WB/rwY29eTdFoA2tNQx90sj4qQkZbevzuo9ko7P2+5EVlvA9nbbw7aHT58+vcQ2ymNoZFw79o9qfHJKIWl8cko79o9qaGQ879YAtKFmH1CttmxX1dVeIuKOiOiLiL7u7u4mt5Ge3QfGNDU9c05tanpGuw+M5dQRgHa21HB/zvYaScpuT2X1E5LWzdvuMkkTS28PcyYmpxqqAyi3pYb7g5JuzO7fKOkr8+pbbb/M9mskXSHpkeW1CEl6VaWzoTqAclv0gKrtvZo9eLra9glJH5d0m6R9tj8k6Zik90lSRByxvU/SE5J+JemmiJip+sZoCOvXA2hEPbNlttV46toa298q6dblNIWFJl+YbqgOoNw4Q7Ug1nZVv/B3rTqAciPcC2Kwv1eVznPPB6t0dmiwvzenjgC0s0WHZdAe5k5W4iQmAPUg3AtkYFMPYQ6gLgzLAECCCHcASBDhDgAJItwBIEEcUC2QnUOj2nvouGYi1GFr29XrdMvAVXm3BaANEe4FsXNoVPccPHbm8UzEmccEPIDzMSxTEHsPHW+oDqDcCPeCmImqy+LXrAMoN8K9IDpqLP9Yqw6g3Aj3gth29bqG6gDKjQOqBTF30JTZMgDqQbgXyC0DVxHmAOrCsAwAJIhwB4AEMSxTIEMj46znDqAuhHtBDI2Ma/D+H2h6ZnZe+/jklAbv/4EkEfAAFmBYpiA+8dUjZ4J9zvRM6BNfPZJTRwDaGeFeED99YbqhOoByI9wBIEGEe0F0VTobqgMoN8K9IHZdt0Gdq85dR6ZzlbXrug05dQSgnRHuBTGwqUc3bF53ZqGwDls3bF7HTBkAVRHuBTE0Mq57Dh47s8Tv3MU6hkbGc+4MQDsi3Avio/sea6gOoNw4iakgZmpck6NWPRWclQssDeGOtjU0Mq4d+0c1NT0jafas3B37RyVxVi6wGIZl0LZ2Hxg7E+xzpqZntPvAWE4dAcVBuKNtjU9ONVQHcBbhjrbFdWOBpSPc0bbmpn3WWwdw1rLC3faztkdtP2Z7OKtdYvsh209ntxc3p1WUTU9XpaE6gLOasef+5ojYGBF92eObJT0cEVdIejh7DDRssL9Xlc6Oc2qVzg4N9vfm1BFQHK0Ylrle0p7s/h5JAy34DJTAwKYeffI9V6mnqyJrdo/9k++5immQQB2WO889JH3Ddkj6p4i4Q9KlEXFSkiLipO1XV3uh7e2StkvS5Zdfvsw2kKqBTT2EObAEyw33LRExkQX4Q7afqveF2S+COySpr6+PI2SoijNUgaVZVrhHxER2e8r2A5I2S3rO9ppsr32NpFNN6BMlxBmqwNIteczd9oW2L5q7L+ltkh6X9KCkG7PNbpT0leU2iXLiDFVg6Zaz536ppAc8e0LJBZK+GBFft/19Sftsf0jSMUnvW36bKKOJGmei1qoDOGvJ4R4Rz0j63Sr1/5F07XKaAiRpbVel6lIDa5nnDiyKM1TRtpjnDiwd4Y62NbCpR+99Q885lxZ87xuYGgnUg3BH2xoaGdeXHjl+zqUFv/TIcS4tCNSBcEfb2vXgEU2/eO4pENMvhnY9eCSnjoDiINzRtianphuqAziLcAeABBHuaFsXv7yzoTqAswh3tK0r11zUUB3AWYQ72tZ//PD5huoAziLc0bZqLRXKEqLA4gh3AEjQctdzB9BkrGGPZiDcgTbCGvZoFoZlgDbCGvZoFsIdaCPVljh+qTpQC+EOAAki3AEgQYQ7ACSIcAeABDEVEmgjXZXOqksad1VYLC01rT6fgXAH2kh2RcG66yimlTifgWEZoI1MvlDjAiU16iimlTifgT13oI2s7apUndO+tquSQzcrZ+fQqPYemr1eboetbVev0y0DV+XdVstM1DhvoVZ9KdhzB9rIm3+7u6F6CnYOjeqeg8fOuRD6PQePaefQaM6dtU6tX9bN/CVOuANt5FtPnW6onoK9h443VE/BYH+vKp0d59QqnR0a7O9t2mcwLAO0kTIuPzC3x15vPQVzB02ZLQMgWassvVglx1clPkNoYFNPS1f6ZFgGQK46aoR4rTrqQ7gDyNX0i43VUR/CHQASRLgDyFWtpRVYcmF5CHcAudp13QZ1nnf0tHOVteu6DTl1lAZmywDI1UpMCyyjloW77bdL+jtJHZLujIjbWvVZQCo6V1U/kNiZ+N/YrZ4WWEYt+Sdju0PSP0h6h6QrJW2zfWUrPgtIye73bWyoDtTSqj33zZKORsQzkmT7PknXS3qiRZ8HJIEhCjRLq8K9R9L8hSFOSLp6/ga2t0vaLkmXX355i9pAkXVYmqly5mLqJ7cwRIFmaNVIXrX/fuf8N42IOyKiLyL6urvTXfEOS/epP9zYUB3AWa0K9xOS1s17fJmkiRZ9Vil84Peq/3VTq56CgU09uv2GjerpqsiSeroquv2GjezVAnVwtGDlNdsXSPovSddKGpf0fUl/FBFHqm3f19cXw8PDDX/O+pv/ZUHt2dve1fD7FEXZLmgA4KXZPhwRfVWfa0W4Zx/6Tkm3a3Yq5N0RcWutbZca7gBQZi8V7i2b5x4RX5P0tVa9PwCgtsRPjQCAciLcASBBhDsAJIhwB4AEtWy2TENN2Kcl/XgZb7Fa0k+a1E4RlO37SnznsuA7N+Y3I6LqWaBtEe7LZXu41nSgFJXt+0p857LgOzcPwzIAkCDCHQASlEq435F3AyusbN9X4juXBd+5SZIYcwcAnCuVPXcAwDyEOwAkqLDhbvtu26dsP553LyvF9jrb37L9pO0jtj+cd0+tZvvXbT9i+wfZd/5E3j2tBNsdtkds/3PevawU28/aHrX9mO3kl4m13WX7fttPZf+n39jU9y/qmLvtN0n6haQvRMTv5N3PSrC9RtKaiHjU9kWSDksaiIhkr01r25IujIhf2O6U9O+SPhwRB3NuraVs/6WkPkmvjIh3593PSrD9rKS+iCjFSUy290j6t4i40/avSXp5REw26/0Lu+ceEd+R9HzefaykiDgZEY9m938u6UnNXq82WTHrF9nDzuynmHskdbJ9maR3Sboz717QGrZfKelNku6SpIj4v2YGu1TgcC872+slbZJ0KOdWWi4bonhM0ilJD0VE6t/5dkl/JenFnPtYaSHpG7YP296edzMt9luSTkv6XDb8dqftC5v5AYR7Adl+haQvS/pIRPws735aLSJmImKjZq/Fu9l2ssNwtt8t6VREHM67lxxsiYjXS3qHpJuyoddUXSDp9ZI+ExGbJP2vpJub+QGEe8Fk485flnRvROzPu5+VlP3Z+m1Jb8+3k5baIum6bPz5PklvsX1Pvi2tjIiYyG5PSXpA0uZ8O2qpE5JOzPsr9H7Nhn3TEO4Fkh1cvEvSkxHx6bz7WQm2u213Zfcrkn5f0lO5NtVCEbEjIi6LiPWStkr614j4QM5ttZztC7NJAsqGJ94mKdmZcBHx35KO2+7NStdKaurEiJZdQ7XVbO+VdI2k1bZPSPp4RNyVb1ctt0XSByWNZmPQkvTX2fVqU7VG0h7bHZrdGdkXEaWZHlgil0p6YHb/RRdI+mJEfD3fllruLyTdm82UeUbSnzTzzQs7FRIAUBvDMgCQIMIdABJEuANAggh3AEgQ4Q4ACSLcASBBhDsAJOj/ASvs0hah1tJXAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(num_purchase[num_purchase.Purchase==0].customerSessionId, num_purchase[num_purchase.Purchase==0].price)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "59deb013"
   },
   "source": [
    "## Splitting data into train and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "ILxcT9WK9tqi"
   },
   "outputs": [],
   "source": [
    "def random_split(data, user_id, item_id, metric, item_features=None):\n",
    "  dataset = Dataset()\n",
    "  dataset.fit(data[user_id], data[item_id], item_features=item_features)\n",
    "  num_users, num_items = dataset.interactions_shape()\n",
    "  print('Num users: {}, num_items {}.'.format(num_users, num_items))\n",
    "  dataset.fit_partial(users=data[user_id], \n",
    "                     items=data[item_id],\n",
    "                     item_features=item_features)\n",
    "  fe = data.to_dict(orient='records')\n",
    "  (interactions, weights) = dataset.build_interactions((x[user_id], x[item_id], x[metric]) for x in fe)\n",
    "  (train, test) = random_train_test_split(interactions=interactions, test_percentage=0.2, random_state=2)\n",
    "  print('Num values: {}'.format(len(interactions.data)))  \n",
    "  return dataset, train, test, interactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_sparsity(data, user_id, item_id, metric):\n",
    "    item_user_matrix = pd.DataFrame(data.pivot_table(index=[user_id], columns=item_id, values=metric))\n",
    "    item_user_matrix.fillna(0,inplace=True)\n",
    "    print('sparsity in data:',(1.0 - (np.count_nonzero(item_user_matrix) / float(item_user_matrix.size))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num users: 44599, num_items 17480.\n",
      "Num values: 997364\n"
     ]
    }
   ],
   "source": [
    "dataset, train_g, test_g, interactions = random_split(num_purchase, 'customerVisitorId', 'ID', 'Purchase')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(44599, 17480)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_g.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(44599, 17480)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_g.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create dictionary to map user and item index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_id_map, user_features, item_id_map, item_features = dataset.mapping()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_id_map_rev = {int(v):k for k,v in item_id_map.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_id_map_rev = {int(v):k for k,v in user_id_map.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<44599x17480 sparse matrix of type '<class 'numpy.int32'>'\n",
       "\twith 797891 stored elements in COOrdinate format>"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<44599x17480 sparse matrix of type '<class 'numpy.int32'>'\n",
       "\twith 199473 stored elements in COOrdinate format>"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_g"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sparsity in data: 0.999594617954051\n"
     ]
    }
   ],
   "source": [
    "calc_sparsity(num_purchase, 'customerVisitorId', 'ID', 'Purchase')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cgOJeaf1pxHk"
   },
   "source": [
    "## LightFM model training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "XJJvHKlt-oGe"
   },
   "outputs": [],
   "source": [
    "def model_fit_eval(loss, train, test, n_components, k_items, k=5, n=10, item_features=None):\n",
    "  model = LightFM(no_components=n_components, loss=loss, item_alpha=1e-6, random_state=2, k=k, n=n)\n",
    "  %time model.fit(train, epochs=10, num_threads=3, item_features=item_features)\n",
    "  test_auc = auc_score(model, test, train_interactions=train, num_threads=3, item_features=item_features).mean()\n",
    "  print('Collaborative filtering test AUC: %s' % test_auc)\n",
    "  prec = precision_at_k(model, test, train_interactions=train, num_threads=3, k=k_items, item_features=item_features).mean()\n",
    "  print('Collaborative filtering test precision: %s' % prec)\n",
    "  recal = recall_at_k(model, test, train_interactions=train, num_threads=3, k=k_items, item_features=item_features).mean()\n",
    "  print('Collaborative filtering test recall: %s' % recal)\n",
    "  return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 24.6 s, sys: 19.5 ms, total: 24.6 s\n",
      "Wall time: 8.82 s\n",
      "Collaborative filtering test AUC: 0.8918844\n",
      "Collaborative filtering test precision: 0.38604438\n",
      "Collaborative filtering test recall: 0.29490969857731075\n"
     ]
    }
   ],
   "source": [
    "model_g = model_fit_eval('warp', train_g, test_g, 128, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import boto3\n",
    "\n",
    "#s3 = boto3.resource('s3')\n",
    "#s3.meta.client.upload_file('./recdeployment/item_dist_mat.npz', 'groupby', 'interactions.npz')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recommending items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def similar_recommendation(model, interactions, user_id, user_dikt, \n",
    "                               item_dikt, item_dikt_rev, product_dict, threshold = 0, number_rec_items = 5):\n",
    "\n",
    "    #Function to produce user recommendations\n",
    "\n",
    "    n_items = len(item_dikt.keys())\n",
    "    user_x = user_dikt[user_id]\n",
    "    scores = pd.Series(model.predict(user_x,np.arange(n_items)))\n",
    "    scores.index = item_dikt.keys()\n",
    "    scores = list(pd.Series(scores.sort_values(ascending=False).index))\n",
    "    user_item_scores = interactions.toarray()[user_x]\n",
    "    item_index = np.where(user_item_scores>0)[0]\n",
    "    known_items = pd.DataFrame({ 'items': [item_dikt_rev[i] for i in item_index], 'scores':user_item_scores[item_index]}).sort_values('scores', ascending=False)\n",
    "    \n",
    "    scores = [x for x in scores if x not in known_items['items'].values]\n",
    "    score_list = scores[0:number_rec_items]\n",
    "\n",
    "    print(\"Items that were liked by the User:\")\n",
    "    counter = 1\n",
    "    for i in range(len(known_items)):\n",
    "        item = known_items.loc[i]['items']\n",
    "        if str(product_dict[item]['name']) != 'nan':\n",
    "            print(str(counter) + '- ' + str(product_dict[item]['name']) + ', ' + str(product_dict[item]['price']) + ', ' + product_dict[item]['collection'])\n",
    "        else:\n",
    "            print(str(counter) + '- ' + str(product_dict[item]['collection']))\n",
    "        counter+=1\n",
    "\n",
    "    print(\"\\n Recommended Items:\")\n",
    "    counter = 1\n",
    "    for i in score_list:\n",
    "        if str(product_dict[i]['name']) != 'nan':\n",
    "            print(str(counter) + '- ' + product_dict[i]['name'] + ', ' + str(product_dict[i]['price']) + ', ' + product_dict[i][\"collection\"]) \n",
    "        else:\n",
    "            print(str(counter) + '- ' + product_dict[i]['collection'])\n",
    "        counter+=1\n",
    "#     return score_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Items that were liked by the User:\n",
      "1- zinc picolinate - body preferred form, 2.99, Swanson Premium\n",
      "\n",
      " Recommended Items:\n",
      "1- zinc citrate, 2.84, Swanson Premium\n",
      "2- vitamin c with rose hips, 11.49, Swanson Premium\n",
      "3- full spectrum saw palmetto, 9.49, Swanson Premium\n",
      "4- full spectrum ashwagandha, 5.79, Swanson Premium\n",
      "5- omega-3 fish oil - lemon flavor, 6.99, Swanson Premium\n"
     ]
    }
   ],
   "source": [
    "similar_recommendation(model, interactions, 'cjo06esrd00013gblo1n3haw7', user_id_map, \n",
    "                               item_id_map, item_id_map_rev, product_dic, threshold = 0, number_rec_items = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Items that were liked by the User:\n",
      "1- long-lasting deodorant stick - refreshing lavender, 5.89, NOW Foods\n",
      "\n",
      " Recommended Items:\n",
      "1- 12 oz pwdr, 19.78, Swanson Ultra\n",
      "2- mct oil - 100% pure pharmaceutical grade, 7.98, Swanson Ultra\n",
      "3- acti-joint plus with krill oil, 13.18, Swanson Ultra\n",
      "4- vitamin d-3 & k-2, 5.99, NOW Foods\n",
      "5- chlorella complex, 17.99, Swanson Kyoto Brand\n"
     ]
    }
   ],
   "source": [
    "similar_recommendation(model, interactions, 'cjo09x89u00013gb80mn9xb57', user_id_map, \n",
    "                               item_id_map, item_id_map_rev, product_dic, threshold = 0, number_rec_items = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Items that were liked by the User:\n",
      "1- 400 iu (10 mcg) 250 sgels, 3.99, Swanson Premium\n",
      "2- longjax eurycoma longifolia jack extract, 18.99, Swanson Passion\n",
      "3- high immunoglobulin colostrum, 18.39, default\n",
      "\n",
      " Recommended Items:\n",
      "1- vitamin d3 - highest potency, 10.99, default\n",
      "2- vitamin d3 - high potency, 4.79, default\n",
      "3- super stress vitamin b-complex with vitamin c, 15.49, Swanson Premium\n",
      "4- coq10 - high potency, 8.99, Swanson Ultra\n",
      "5- zinc picolinate - body preferred form, 2.99, Swanson Premium\n"
     ]
    }
   ],
   "source": [
    "similar_recommendation(model_g, 'cjo6507qf00013ac37tbxxu8z', user_id_map, \n",
    "                               item_id_map, item_id_map_rev, dic, threshold = 0, number_rec_items = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def item_emdedding_distance_matrix(model, item_dikt):\n",
    "\n",
    "#     Function to create item-item distance embedding matrix\n",
    "\n",
    "    df_item_norm_sparse = csr_matrix(model.item_embeddings)\n",
    "    similarities = cosine_similarity(df_item_norm_sparse)\n",
    "    item_emdedding_distance_matrix = pd.DataFrame(similarities)\n",
    "    item_emdedding_distance_matrix.columns = item_dikt.keys()\n",
    "    item_emdedding_distance_matrix.index = item_dikt.keys()\n",
    "    return item_emdedding_distance_matrix\n",
    "\n",
    "def also_bought_recommendation(item_emdedding_distance_matrix, item_id, \n",
    "                             product_dict, n_items = 5):\n",
    "\n",
    "#     Function to create item-item recommendation\n",
    "\n",
    "    recommended_items = list(pd.Series(item_emdedding_distance_matrix.loc[item_id,:]. \\\n",
    "                                  sort_values(ascending = False).head(n_items+1). \\\n",
    "                                  index[1:n_items+1]))\n",
    "    \n",
    "    print(\"Item of interest:\")\n",
    "    print(str(product_dict[item_id]['name']) + ', ' + str(product_dict[item_id]['price']) + ', ' + product_dict[item_id]['collection'])\n",
    "    print(\"\\n\")\n",
    "    print(\"Items that are frequently bought together:\")\n",
    "    counter = 1\n",
    "    for i in recommended_items:\n",
    "        if i in product_dict.keys():\n",
    "            print(str(counter) + '- ' + str(product_dict[i]['name']) + ', ' + str(product_dict[i]['price']) + ', ' + product_dict[i]['collection'])\n",
    "        else:\n",
    "            print(str(counter) + '- ' + 'swansonhealthproduct')\n",
    "        counter+=1\n",
    "    #return recommended_items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_dist = item_emdedding_distance_matrix(model, item_id_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item of interest:\n",
      "zinc picolinate - body preferred form, 2.99, Swanson Premium\n",
      "\n",
      "\n",
      "Items that are frequently bought together:\n",
      "1- zinc citrate, 2.84, Swanson Premium\n",
      "2- vitamin d3 - high potency, 4.79, default\n",
      "3- full spectrum saw palmetto, 9.49, Swanson Premium\n",
      "4- vitamin c with rose hips, 11.49, Swanson Premium\n",
      "5- triple magnesium complex, 11.99, Swanson Premium\n"
     ]
    }
   ],
   "source": [
    "also_bought_recommendation(item_dist, 'SW1113', dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item of interest:\n",
      "fat burner - 2 pack, 12.49, Swanson Best Weight-Control Formulas\n",
      "\n",
      "\n",
      "Items that are frequently bought together:\n",
      "1- fat burner, 7.99, Swanson Best Weight-Control Formulas\n",
      "2- swansonhealthproduct\n",
      "3- super fat burners extreme, 11.61, Natural Balance\n",
      "4- swansonhealthproduct\n",
      "5- full spectrum pterocarpus marsupium, 10.79, Swanson Premium\n"
     ]
    }
   ],
   "source": [
    "also_bought_recommendation(item_dist, 'SWD015', dic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "v0Gvro1N9O0_"
   },
   "source": [
    "## Kaggle Cosmetics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "UZixmptq9L6F"
   },
   "outputs": [],
   "source": [
    "all_features = pd.read_pickle(\"s3://groupby/data/dfCosmetics.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 20583955 entries, 0 to 20583954\n",
      "Data columns (total 12 columns):\n",
      " #   Column          Dtype              \n",
      "---  ------          -----              \n",
      " 0   event_time      datetime64[ns, UTC]\n",
      " 1   event_type      object             \n",
      " 2   product_id      int64              \n",
      " 3   category_id     int64              \n",
      " 4   price           float64            \n",
      " 5   user_id         int64              \n",
      " 6   user_session    object             \n",
      " 7   year            object             \n",
      " 8   month           object             \n",
      " 9   day             object             \n",
      " 10  year_month_day  object             \n",
      " 11  day_seq         int64              \n",
      "dtypes: datetime64[ns, UTC](1), float64(1), int64(4), object(6)\n",
      "memory usage: 2.0+ GB\n"
     ]
    }
   ],
   "source": [
    "all_features.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features2 = all_features[(all_features.price>=11) & (all_features.price<=24)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features2 = all_features[all_features.category_id == 1487580013950664926]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "1ZGWxfmJ9TNA"
   },
   "outputs": [],
   "source": [
    "GroupUS = all_features.sort_values('event_time').groupby(['user_id', 'product_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "v_xd8DGs9UE9"
   },
   "outputs": [],
   "source": [
    "NumEventTime = GroupUS['event_type'].value_counts().unstack(fill_value = 0).rename(columns={'view':'NumTimesViewed','cart':'NumTimesCarted',\n",
    "                                                                                        'purchase':'NumTimesPurchased','remove_from_cart' : 'NumTimesRemoved'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "Fs5dbokX9XiB"
   },
   "outputs": [],
   "source": [
    "data = NumEventTime.reset_index()\n",
    "data['Purchase'] = data.NumTimesPurchased.apply(lambda x: 1 if x!=0 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "other_data = GroupUS.agg({'price':'mean', 'user_session':'nunique', 'category_id':'first'}).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.merge(other_data, how='inner', on=['user_id', 'product_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([9.557002e+06, 3.105470e+05, 1.182760e+05, 4.792400e+04,\n",
       "        3.099900e+04, 2.572200e+04, 7.511000e+03, 2.069000e+03,\n",
       "        4.323000e+03, 7.198000e+03]),\n",
       " array([5.00000e-02, 3.28230e+01, 6.55960e+01, 9.83690e+01, 1.31142e+02,\n",
       "        1.63915e+02, 1.96688e+02, 2.29461e+02, 2.62234e+02, 2.95007e+02,\n",
       "        3.27780e+02]),\n",
       " <BarContainer object of 10 artists>)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEDCAYAAAAlRP8qAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAPAElEQVR4nO3cf+xdd13H8efLdsMfQwf0C5ltxwopYEPYwFqIPxB/QTtMKokmLURgjjRLGME/TKghIoZ/QIJKwqCp2ADGrIkypUplEhVngsg67H6U2VG3yb6U0E4EBBJnx9s/7il8ubvfe0+72957Pzwfyc33ns/53HtfOTd99dxzzz2pKiRJi+/7Zh1AkjQdFrokNcJCl6RGWOiS1AgLXZIaYaFLUiNmWuhJDiQ5leSeHnP/MMnR7nZfkq9chIiStDAyy/PQk7wY+Drwoap67jk87g3A86vqNy5YOElaMDPdQ6+q24AvrxxL8swkH0tyR5J/TvKcEQ/dDdx8UUJK0oJYO+sAI+wHbqiqzyV5IfBe4OfPrkzydGAT8A8zyidJc2muCj3JZcBPAn+e5OzwE4am7QL+oqoevZjZJGnezVWhMzgE9JWqumbMnF3A6y9OHElaHHN12mJVfQ14IMmvAWTg6rPrkzwbeBLwLzOKKElza9anLd7MoJyfnWQ5yfXAq4Drk9wJHAN2rnjIbuBgeYlISXqMiactJjkA/DJwatSphRkc7H43cC3wTeC1VfWZC5BVkjRGnz30DwDbx6zfAWzubnuA9z3+WJKkczXxS9Gqui3JVWOm7GTww6ACPpXk8iRXVNUXxz3vunXr6qqrxj2tJGnYHXfc8XBVLY1aN42zXNYDD61YXu7GHlPoSfYw2Ivnyiuv5MiRI1N4eUn63pHkP1dbN40vRTNibOSB+araX1Vbq2rr0tLI/2AkSedpGoW+DGxcsbwBODmF55UknYNpFPoh4NXdOeMvAr466fi5JGn6Jh5D784VfwmwLsky8LvAJQBVtQ84zOCUxRMMTlu87kKFlSStrs9ZLrsnrC/8Kb4kzdxc/fRfknT+LHRJaoSFLkmNsNAlqRHzdj30Xq7a+9GZvfaDb3/5zF5bksZxD12SGmGhS1IjLHRJaoSFLkmNsNAlqREWuiQ1wkKXpEZY6JLUCAtdkhphoUtSIyx0SWqEhS5JjbDQJakRFrokNcJCl6RGWOiS1AgLXZIaYaFLUiMsdElqhIUuSY2w0CWpERa6JDXCQpekRljoktQIC12SGmGhS1IjLHRJaoSFLkmNsNAlqRG9Cj3J9iTHk5xIsnfE+h9J8tdJ7kxyLMl1048qSRpnYqEnWQPcBOwAtgC7k2wZmvZ64LNVdTXwEuBdSS6dclZJ0hh99tC3ASeq6v6qegQ4COwcmlPAE5MEuAz4MnBmqkklSWP1KfT1wEMrlpe7sZXeA/wYcBK4G3hjVX1r+ImS7ElyJMmR06dPn2dkSdIofQo9I8ZqaPllwFHgR4FrgPck+eHHPKhqf1VtraqtS0tL5xhVkjROn0JfBjauWN7AYE98peuAW2rgBPAA8JzpRJQk9dGn0G8HNifZ1H3RuQs4NDTn88AvACR5GvBs4P5pBpUkjbd20oSqOpPkRuBWYA1woKqOJbmhW78PeBvwgSR3MzhE86aqevgC5pYkDZlY6ABVdRg4PDS2b8X9k8BLpxtNknQu/KWoJDXCQpekRljoktQIC12SGmGhS1IjLHRJaoSFLkmNsNAlqREWuiQ1wkKXpEZY6JLUCAtdkhphoUtSIyx0SWqEhS5JjbDQJakRFrokNcJCl6RGWOiS1AgLXZIaYaFLUiMsdElqhIUuSY2w0CWpERa6JDXCQpekRljoktQIC12SGmGhS1IjLHRJaoSFLkmNsNAlqREWuiQ1wkKXpEZY6JLUiF6FnmR7kuNJTiTZu8qclyQ5muRYkn+abkxJ0iRrJ01Isga4CfglYBm4PcmhqvrsijmXA+8FtlfV55M89QLllSStos8e+jbgRFXdX1WPAAeBnUNzXgncUlWfB6iqU9ONKUmapE+hrwceWrG83I2t9CzgSUk+keSOJK8e9URJ9iQ5kuTI6dOnzy+xJGmkPoWeEWM1tLwW+HHg5cDLgN9J8qzHPKhqf1VtraqtS0tL5xxWkrS6icfQGeyRb1yxvAE4OWLOw1X1DeAbSW4Drgbum0pKSdJEffbQbwc2J9mU5FJgF3BoaM5HgJ9JsjbJDwIvBO6dblRJ0jgT99Cr6kySG4FbgTXAgao6luSGbv2+qro3yceAu4BvAe+vqnsuZHBJ0nfrc8iFqjoMHB4a2ze0/E7gndOLJkk6F/5SVJIaYaFLUiMsdElqhIUuSY2w0CWpERa6JDXCQpekRljoktQIC12SGmGhS1IjLHRJaoSFLkmNsNAlqREWuiQ1wkKXpEZY6JLUCAtdkhphoUtSIyx0SWqEhS5JjbDQJakRFrokNcJCl6RGWOiS1AgLXZIaYaFLUiMsdElqhIUuSY2w0CWpERa6JDXCQpekRljoktQIC12SGmGhS1IjehV6ku1Jjic5kWTvmHk/keTRJL86vYiSpD4mFnqSNcBNwA5gC7A7yZZV5r0DuHXaISVJk/XZQ98GnKiq+6vqEeAgsHPEvDcAHwZOTTGfJKmnPoW+HnhoxfJyN/ZtSdYDrwD2jXuiJHuSHEly5PTp0+eaVZI0Rp9Cz4ixGlr+I+BNVfXouCeqqv1VtbWqti4tLfWMKEnqY22POcvAxhXLG4CTQ3O2AgeTAKwDrk1ypqr+ahohJUmT9Sn024HNSTYBXwB2Aa9cOaGqNp29n+QDwN9Y5pJ0cU0s9Ko6k+RGBmevrAEOVNWxJDd068ceN5ckXRx99tCpqsPA4aGxkUVeVa99/LEkSefKX4pKUiMsdElqhIUuSY2w0CWpERa6JDXCQpekRljoktQIC12SGmGhS1IjLHRJaoSFLkmNsNAlqREWuiQ1wkKXpEZY6JLUCAtdkhphoUtSIyx0SWqEhS5JjbDQJakRFrokNcJCl6RGWOiS1AgLXZIaYaFLUiMsdElqhIUuSY2w0CWpERa6JDXCQpekRljoktQIC12SGmGhS1IjLHRJakSvQk+yPcnxJCeS7B2x/lVJ7upun0xy9fSjSpLGmVjoSdYANwE7gC3A7iRbhqY9APxsVT0PeBuwf9pBJUnj9dlD3wacqKr7q+oR4CCwc+WEqvpkVf13t/gpYMN0Y0qSJulT6OuBh1YsL3djq7ke+NtRK5LsSXIkyZHTp0/3TylJmqhPoWfEWI2cmPwcg0J/06j1VbW/qrZW1dalpaX+KSVJE63tMWcZ2LhieQNwcnhSkucB7wd2VNV/TSeeJKmvPnvotwObk2xKcimwCzi0ckKSK4FbgF+vqvumH1OSNMnEPfSqOpPkRuBWYA1woKqOJbmhW78PeAvwFOC9SQDOVNXWCxdbkjSszyEXquowcHhobN+K+68DXjfdaJKkc+EvRSWpERa6JDXCQpekRljoktQIC12SGmGhS1IjLHRJaoSFLkmNsNAlqREWuiQ1wkKXpEZY6JLUCAtdkhphoUtSIyx0SWqEhS5JjbDQJakRFrokNcJCl6RGWOiS1AgLXZIaYaFLUiMsdElqhIUuSY2w0CWpERa6JDXCQpekRljoktQIC12SGmGhS1IjLHRJasTaWQdYNFft/ehMXvfBt798Jq8raXG4hy5JjbDQJakRFrokNaLXMfQk24F3A2uA91fV24fWp1t/LfBN4LVV9ZkpZ/2eNqtj9+Dxe2lRTNxDT7IGuAnYAWwBdifZMjRtB7C5u+0B3jflnJKkCfrsoW8DTlTV/QBJDgI7gc+umLMT+FBVFfCpJJcnuaKqvjj1xLroZvnp4HuNn4b0ePQp9PXAQyuWl4EX9pizHviuQk+yh8EePMDXkxw/p7TfsQ54+DwfO2uLmn1Rc8MCZc87vmtxYXKPsKjZFyH301db0afQM2KszmMOVbUf2N/jNccHSo5U1dbH+zyzsKjZFzU3LG72Rc0Ni5t9UXOf1ecsl2Vg44rlDcDJ85gjSbqA+hT67cDmJJuSXArsAg4NzTkEvDoDLwK+6vFzSbq4Jh5yqaozSW4EbmVw2uKBqjqW5IZu/T7gMINTFk8wOG3xugsXGZjCYZsZWtTsi5obFjf7ouaGxc2+qLkByODEFEnSovOXopLUCAtdkhqxcIWeZHuS40lOJNk76zzjJHkwyd1JjiY50o09OcnHk3yu+/ukWecESHIgyakk96wYWzVrkt/u3oPjSV42m9Sr5n5rki902/1okmtXrJuX3BuT/GOSe5McS/LGbnwRtvlq2ed6uyf5/iSfTnJnl/v3uvG53+a9VdXC3Bh8KfsfwDOAS4E7gS2zzjUm74PAuqGx3wf2dvf3Au+Ydc4uy4uBFwD3TMrK4BIQdwJPADZ178maOcr9VuC3Rsydp9xXAC/o7j8RuK/LtwjbfLXsc73dGfxe5rLu/iXAvwIvWoRt3ve2aHvo374MQVU9Apy9DMEi2Ql8sLv/QeBXZhflO6rqNuDLQ8OrZd0JHKyq/62qBxic3bTtYuQctkru1cxT7i9WdwG7qvof4F4Gv65ehG2+WvbVzEX2Gvh6t3hJdysWYJv3tWiFvtolBuZVAX+X5I7usgcAT6vuHP3u71Nnlm6y1bIuwvtwY5K7ukMyZz9Cz2XuJFcBz2ewx7hQ23woO8z5dk+yJslR4BTw8apauG0+zqIVeq9LDMyRn6qqFzC4GuXrk7x41oGmZN7fh/cBzwSuYXA9oXd143OXO8llwIeB36yqr42bOmJs3rLP/Xavqker6hoGv2bfluS5Y6bPTe6+Fq3QF+oSA1V1svt7CvhLBh/XvpTkCoDu76nZJZxotaxz/T5U1Ze6f7jfAv6Y73xMnqvcSS5hUIh/VlW3dMMLsc1HZV+U7Q5QVV8BPgFsZ0G2eR+LVuh9LkMwF5L8UJInnr0PvBS4h0He13TTXgN8ZDYJe1kt6yFgV5InJNnE4Dr4n55BvpHO/uPsvILBdoc5yp0kwJ8A91bVH6xYNffbfLXs877dkywluby7/wPALwL/zgJs895m/a3sud4YXGLgPgbfOL951nnG5HwGg2/I7wSOnc0KPAX4e+Bz3d8nzzprl+tmBh+T/4/Bnsn147ICb+7eg+PAjjnL/afA3cBdDP5RXjGHuX+awcf3u4Cj3e3aBdnmq2Wf6+0OPA/4ty7fPcBbuvG53+Z9b/70X5IasWiHXCRJq7DQJakRFrokNcJCl6RGWOiS1AgLXZIaYaFLUiP+H3mBGS2gXJUIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(data.price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['price_category'] = np.where(data.price>=99, 'very expensive', np.where((data.price >= 66) & (data.price < 99), 'expensive', np.where((data.price >= 33) & (data.price<66), 'medium', 'cheap')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>NumTimesCarted</th>\n",
       "      <th>NumTimesPurchased</th>\n",
       "      <th>NumTimesRemoved</th>\n",
       "      <th>NumTimesViewed</th>\n",
       "      <th>Purchase</th>\n",
       "      <th>price</th>\n",
       "      <th>user_session</th>\n",
       "      <th>category_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>465496</td>\n",
       "      <td>5769989</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>10.00</td>\n",
       "      <td>1</td>\n",
       "      <td>1487580008447738866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>465496</td>\n",
       "      <td>5865524</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>10.00</td>\n",
       "      <td>1</td>\n",
       "      <td>1487580008447738866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>465496</td>\n",
       "      <td>5865526</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>10.00</td>\n",
       "      <td>1</td>\n",
       "      <td>1487580008447738866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1120748</td>\n",
       "      <td>5240</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4.05</td>\n",
       "      <td>1</td>\n",
       "      <td>1487580009202712597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1180452</td>\n",
       "      <td>5881337</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>25.40</td>\n",
       "      <td>1</td>\n",
       "      <td>1487580012096782476</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  product_id  NumTimesCarted  NumTimesPurchased  NumTimesRemoved  \\\n",
       "0   465496     5769989               0                  0                0   \n",
       "1   465496     5865524               0                  0                0   \n",
       "2   465496     5865526               0                  0                0   \n",
       "3  1120748        5240               0                  0                0   \n",
       "4  1180452     5881337               0                  0                0   \n",
       "\n",
       "   NumTimesViewed  Purchase  price  user_session          category_id  \n",
       "0               1         0  10.00             1  1487580008447738866  \n",
       "1               1         0  10.00             1  1487580008447738866  \n",
       "2               1         0  10.00             1  1487580008447738866  \n",
       "3               1         0   4.05             1  1487580009202712597  \n",
       "4               1         0  25.40             1  1487580012096782476  "
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Kaggle Cosmetics Data Filter by Price')"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAfXklEQVR4nO3deZgdVbnv8e+PJpCQgSQmcDIIYYhRUBCICgKKgAyCEr2goGhEBEcCovGAegE9csATQUEBQaYoiETAMOgxYBQFQTADMYSQCwKZCEmABBKIkIT3/lGri51md3f1sPfu3f37PM9+umpV1V5v7entWqtqlSICMzMzgM1qHYCZmXUdTgpmZpZzUjAzs5yTgpmZ5ZwUzMws56RgZmY5JwV7A0nXSvp+reOoFknbSVorqaHWsVSapP0lLSiZf0rSwRWs7xxJ11Xq+Vupe62kHWtRdz1zUujCmn5hJR0raZWk99cyrpYoM0HSw5JekrRE0m8kvaPWsTVq+rpGxKKI6BcRGzuxjkj7v1bSc5KmS/pEG7Y/QNKSDtQ/KsWwtuQxJyLuiYgxzWxTsx/wItL7ti7ty3JJ10jq19z66T19opoxdgdOCnVC0njgEuCIiPhLreNpwUXAqcAEYDDwFmAqcEQNY6qV3SOiHzAGuBb4qaSzqxzDwPTj2C8idq9kRZI2r+TzJx9Or+mewLuA79Qoju4rIvzoog/gKeBg4GTgWWBsybKdgD8Bz6Vl15P9ADQu3xOYDawBfgPcCHy/ZPk3gWXA08DngQB2TsuubbLukcBDwGrgPmC3ZuIdDWwE3t3CPm0N/AJYCSwk+1JvlpbtDPwFeCHt040l2wXwZeCxtE//lV6D+4EXgSnAFq3FDPwSeA1YB6xNr8Oo9Pybp3UGA9ek12YVMDWVDwHuSM/5PHBPY+xl9jN/PUvKjgb+DbwpzZ8AzE/78wTwhVTeN8X3WopxLTAceHfa39Xpvftp6T43qWuTfSopPwBYUuYzdhjwKrA+1Ten5P26KtW3FPg+0JCWfRb4G/Cj9Hp8v0wc5wA3kX3+1gCzyJIlwETg5ibr/wT4cUvfh5L5ScAdJa/3V9Ln48mm7wHQB7iA7DP3AnAv0Cct2zt9RlYDc4ADav3dr+Wj5gH40cKbk30JbgaWN36RSpbtDHwQ2BIYCvy18csEbJE+/KcCvYCPpS/899Pyw4BngF2Brch+KMsmBbLksgJ4D9AAjE9xbVkm3i8CC1vZp18AtwL90w/X/wNOTMtuAL5NdgTbG9ivZLsAbgMGpLhfAaYDO6YfrkeA8UViLvPjMopNk8LvyH7EBqXX7/2p/DzgZ6msF7A/oGb2s1xS6AVsAA5P80eQJTYB7wdeBvZMyw6g5Mc7le2VfsA2TzHPB05rpv5N9qmkfJPnLX0tyH7Ar2uy/lTgcrJEtQ3wIK8nr8+m/TklxdSnTBznkCWao9P+fwN4Mk0PA14i/TOTnmMFsFcL34fGWN8MzAP+q+T1vossofcpKWv8TF8C3A2MSJ+J95J9d0aQ/WP1IbLP3QfT/NBaf/9r9ah5AH608OZkX4IXyX5Ey/5HWrLuOGB2mn4f2X91Kll+L6//0F8NnFeybGeaTwqXNX7xStZfQPqhbFL+beDvLcTYQPZjvktJ2ReAu9P0L4ArgJFltg1g35L5mcB/lsxfwOtJscWYaSEppB+q14BBZWL4Xnovdm5uH5vE+4b1yJLxp5rZZipwapo+gCZJocz6pwG/bWZZ4z6tLnl8o+nz0kJSALZN71efkrLjgD+n6c8Ci1qJ8ZzSzwTZD+8yYP80/7/ASWn6SOCRVr4Pa9O+LAQuZdMEcGC59yDVuY4m/1ildf4T+GWTsmmkfzB64sN9Cl3fF8na5a+UpMZCSdtI+rWkpZJeBK4ja96ArKlhaaRPeLK4ZHp4k/nS6aa2B74uaXXjg+y/tOFl1n2O7Ee1OUN4/Sim0UKy/9Yga8oR8KCkeZI+12T75SXT68rMN3Y6tiXmpt4MPB8Rq8osmwQ8Dtwp6QlJZxR4vpykXmRHdc+n+cMl/V3S8ynGD/H6e1hu+7dIukPSM+k9/++W1k+GRMTA9PhhW+Ilex17ActKXsfLyY4YGrX02XnDOhHxGrCE19+LycDxafp4sqPWloxL+7J9RHw5ItYViGUI2ZHnv8os2x44pslnZT9a/hx3a04KXd8K4CCypopLS8rPI/tPaLeIGED2hWpMGsuAEaVJhOzHjpLlI5tZ1tRi4NySH5aBEbFVRNxQZt3pwEhJY5t5rmfJmhK2Lynbjuyohoh4JiJOiojhZEcQl0rauYXY2htztLLtYEkDmy6IiDUR8fWI2BH4MHC6pIPaENdRZM0tD0rakqxp8IfAthExEPg9r7+H5WK8DHgUGJ3e82+VrN8Zmta5mOxIoTSxDIiIXVvYppz88yVpM7LP3tOpaCqwm6S3kx0pXN/e4FuI5VmyvpydyixbTHakUPpZ6RsR53cgjrrmpFAHIuJp4EDgMEk/SsX9SYfSkkaQddo1up+sw/erkjaXdBRZJ2WjKcAJkt4maSvgrBaq/znwRUnvSaeb9pV0hKT+ZeJ8jCxx3ZBOqdxCUu90Ku0ZkZ3yOQU4V1J/SdsDp5Md5SDpGEmNyWoV2Ze8PaeJthbzcrK+iDeIiGVkTRqXShokqZek96X4jpS0c0q2L6bYWo1P0mBJnyJr1/5BRDxHdsS0JVmH+wZJhwOHlGy2HHiTpK1LyvqnetdKeivwpYKvR1HLgVHph7vxtbgTuEDSAEmbSdqpHadE7yXpY+msoNPIEs3fUx3/JuuI/hXwYEQs6qR9yaWjk6uBCyUNl9QgaZ+UmK8DPizp0FTeO312R7b8rN2Xk0KdiIjFZInhaEnnAd8l61B9gaxj9JaSdV8l61w+kaz99Xiys2ZeScv/F7gY+DNZc8j9adNXytQ7AziJ7EyXVWn9z7YQ6oS07iWp7n8BHwVuT8tPIetcfIKsn+NXZF9YyE4xfEDSWrJO5VMj4skWX5gyCsR8HvCd1FzwjTJP8WmyI5pHyY7UTkvlo4E/kiXj+4FLI+LuFkKZk/blcbIzvL4WEWelGNeQvVZTUoyfJNvnxn14lKzj/YkU53CyPoFPkp3F83OyzvDO9Jv09zlJs9L0Z8gS2CMpzptoe9PKrcAn0vafBj4WEetLlk8G3kHrTUcd8Q1gLvAPsua7H5D10y0mO4L7FlmCXkz2D1aP/W3Ups3O1l1JegD4WURcU2bZ24CHyc7O2VD14KxHk7QdWQL+j4h4sdbx9HQ9Nht2d5LeL+k/UvPReGA34A8lyz+amncGkf3XdLsTglVbaqo6Hfi1E0LX4Cv/uq8xZE0T/ciacI5ObcSNvkB26ulGsgvGvlztAK1nk9SXrB9jIdm1M9YFuPnIzMxybj4yM7NcXTcfDRkyJEaNGlXrMMzM6srMmTOfjYih5ZbVdVIYNWoUM2bMqHUYZmZ1RdLC5pa5+cjMzHJOCmZmlnNSMDOznJOCmZnlnBTMzCxX12cfmXVFU2cvZdK0BTy9eh3DB/Zh4qFjGLfHiNY3NOsCnBTMOtHU2Us585a5rFufjai9dPU6zrxlLoATg9UFNx+ZdaJJ0xbkCaHRuvUbmTRtQY0iMmsbJwWzTvT06nVtKjfrapwUzDrR8IF92lRu1tU4KZh1oomHjqFPr4ZNyvr0amDioWNqFJFZ27ij2awTNXYm++wjq1dOCmadbNweI5wErG65+cjMzHJOCmZmlnNSMDOznJOCmZnlnBTMzCznpGBmZjknBTMzyzkpmJlZzknBzMxyTgpmZpZzUjAzs5yTgpmZ5SqWFCRdLWmFpIdLygZLukvSY+nvoJJlZ0p6XNICSYdWKi4zM2teJY8UrgUOa1J2BjA9IkYD09M8knYBjgV2TdtcKqkBMzOrqoolhYj4K/B8k+KjgMlpejIwrqT81xHxSkQ8CTwOvLtSsZmZWXnV7lPYNiKWAaS/26TyEcDikvWWpLI3kHSypBmSZqxcubKiwZqZ9TRdpaNZZcqi3IoRcUVEjI2IsUOHDq1wWGZmPUu1k8JyScMA0t8VqXwJ8OaS9UYCT1c5NjOzHq/aSeE2YHyaHg/cWlJ+rKQtJe0AjAYerHJsZmY9XsXu0SzpBuAAYIikJcDZwPnAFEknAouAYwAiYp6kKcAjwAbgKxGxsVKxmZlZeRVLChFxXDOLDmpm/XOBcysVj5mZta6rdDSbmVkX4KRgZmY5JwUzM8s5KZiZWc5JwczMck4KZmaWc1IwM7Ock4KZmeWcFMzMLOekYGZmOScFMzPLOSmYmVnOScHMzHJOCmZmlnNSMDOznJOCmZnlnBTMzCznpGBmZjknBTMzyzkpmJlZzknBzMxyTgpmZpZzUjAzs5yTgpmZ5ZwUzMws56RgZma5VpOCpH0l9U3Tx0u6UNL2lQ/NzMyqrciRwmXAy5J2B74JLAR+0ZFKJX1N0jxJD0u6QVJvSYMl3SXpsfR3UEfqMDOztiuSFDZERABHARdFxEVA//ZWKGkEMAEYGxFvBxqAY4EzgOkRMRqYnubNzKyKiiSFNZLOBI4HfiepAejVwXo3B/pI2hzYCniaLOlMTssnA+M6WIeZmbVRkaTwCeAV4MSIeAYYAUxqb4URsRT4IbAIWAa8EBF3AttGxLK0zjJgm3LbSzpZ0gxJM1auXNneMMzMrIxWk0JEPBMRF0bEPWl+UUS0u08h9RUcBewADAf6Sjq+6PYRcUVEjI2IsUOHDm1vGGZmVkaRs48+ljp/X5D0oqQ1kl7sQJ0HA09GxMqIWA/cArwXWC5pWKpzGLCiA3WYmVk7FGk++h/gIxGxdUQMiIj+ETGgA3UuAvaWtJUkAQcB84HbgPFpnfHArR2ow8zM2mHzAussj4j5nVVhRDwg6SZgFrABmA1cAfQDpkg6kSxxHNNZdZqZWTFFksIMSTcCU8k6nAGIiFvaW2lEnA2c3aT4FbKjBjMzq5EiSWEA8DJwSElZkPUFmJlZN9JqUoiIE6oRiJmZ1V6Rs49GSvqtpBWSlku6WdLIagRnZmbVVeTso2vIzgwaTnbh2u2pzMzMupkiSWFoRFwTERvS41rAV42ZmXVDRZLCs2nI7Ib0OB54rtKBmZlZ9RVJCp8DPg48QzZW0dGpzMzMupkiZx8tAj5ShVjMzKzGmk0Kkr4ZEf8j6Sdk1yVsIiImVDQyMzOrupaOFBqHtphRjUDMzKz2mk0KEXF7+tt44xskbQb0i4iOjJJqZmZdVJGL134laYCkvsAjwAJJEysfmpmZVVuRs492SUcG44DfA9sBn65kUGZmVhtFkkIvSb3IksKt6cY4b+h4NjOz+lckKVwOPAX0Bf4qaXvAfQpmZt1QkesULgYuLilaKOkDlQvJzMxqpUhH86mpo1mSrpI0CziwCrGZmVmVFRrmInU0H0I2EN4JwPkVjcrMzGqiSFJQ+vsh4JqImFNSZmZm3UiRpDBT0p1kSWGapP7Aa5UNy8zMaqHIPZpPBN4JPBERL0t6E1kTkpmZdTNFjhQC2AVoHACvL9C7YhGZmVnNFEkKlwL7AMel+TXAJRWLyMzMaqZI89F7ImJPSbMBImKVpC0qHJeZmdVAkSOF9ZIaSENbSBqKO5rNzLqlIknhYuC3wDaSzgXuBf67olGZmVlNFBnm4npJM4GDyK5PGBcR81vZzMzM6lCRYS52Ap6MiEuAh4EPShrYkUolDZR0k6RHJc2XtI+kwZLukvRY+juoI3WYmVnbFWk+uhnYKGln4EpgB+BXHaz3IuAPEfFWYHeyW3+eAUyPiNHA9DRvZmZVVCQpvBYRG4CPARdFxNeAYe2tUNIA4H3AVQAR8WpErAaOAhpv/TmZ7P4NZmZWRUXPPjoO+AxwRyrr1YE6dwRWAtdImi3pynSrz20jYhlA+rtNuY0lnSxphqQZK1eu7EAYZmbWVJGkcALZxWvnRsSTknYArutAnZsDewKXRcQewEu0oakoIq6IiLERMXbo0KEdCMPMzJpqNSlExCMRMSEibkjzT0ZER4bOXgIsiYgH0vxNZEliuaRhAOnvig7UYWZm7VDkSKFTRcQzwGJJY1LRQcAjwG3A+FQ2Hri12rGZmfV0RYa5qIRTgOvTcBlPkDVRbQZMkXQisAg4pkaxmZn1WM0mBUm/jIhPSzo1Ii7qzEoj4iFgbJlFB3VmPWZm1jYtHSnsJWl74HOSfkGTu61FxPMVjcysTk2dvZRJ0xbw9Op1DB/Yh4mHjmHcHiNqHZZZIS0lhZ8BfyA7hXQmmyaFSOVmVmLq7KWcectc1q3fCMDS1es485a5AE4MVhea7WiOiIsj4m3A1RGxY0TsUPJwQjArY9K0BXlCaLRu/UYmTVtQo4jM2qbIgHhfkrQ7sH8q+mtE/LOyYZnVp6dXr2tTuVlXU2RAvAnA9WRXGG9DdtbQKZUOzKweDR/Yp03lZl1NkesUPk9297WzIuIsYG/gpMqGZVafJh46hj69GjYp69OrgYmHjmlmC7Oupch1CgJKG0k30uRMJDPLNHYm++wjq1dFksI1wAOSfpvmx5FGODWzNxq3xwgnAatbRTqaL5R0N7Af2RHCCRExu9KBmZlZ9RUa5iIiZgGzKhyLWbfgi9esntVq7COzbskXr1m9q/ooqWbdmS9es3rXYlKQ1CDpj9UKxqze+eI1q3ctJoWI2Ai8LGnrKsVjVtd88ZrVuyLNR/8G5kq6StLFjY9KB2ZWj3zxmtW7Ih3Nv0sPM2uFL16zelfkOoXJkvoA20WEe8vMWuGL16yeFRkQ78PAQ2T3VkDSOyXdVuG4zMysBoo0H50DvBu4G7JbaUraoYIxmdU1X7xm9axIUtgQES9Im96Ns0LxmNU1X7xm9a7I2UcPS/ok0CBptKSfAPdVOC6zuuSL16zeFUkKpwC7Aq8ANwAvAqdVMCazurW0mYvUmis362qKnH30MvBtST/IZmNN5cMyq08NEhvjja2rDfItSKw+FDn76F2S5gL/JLuIbY6kvSofmln9KZcQWio362qKNB9dBXw5IkZFxCjgK2Q33jGzJgZt1atN5WZdTZGksCYi7mmciYh7ATchmZXR3AGBDxSsXjTbpyBpzzT5oKTLyTqZA/gE6ZoFM9vUC+vWt6ncrKtpqaP5gibzZ5dMd/j/HkkNwAxgaUQcKWkwcCMwCngK+HhErOpoPWbVNHxgn7JnGnmUVKsXzTYfRcQHWngc2Al1nwrML5k/A5geEaOB6WnerK54lFSrd0XOPhooaYKkCztr6GxJI4EjgCtLio8CJqfpycC4jtRhVgvj9hjByEG9NykbOai3r2a2ulGko/n3ZE06c4GZJY+O+DHwTeC1krJtI2IZQPq7TbkNJZ0saYakGStXruxgGGad61M/v5/HVry0SdljK17iUz+/v0YRmbVNkbGPekfE6Z1VoaQjgRURMVPSAW3dPiKuAK4AGDt2rM/psC7lb/96vk3lZl1NkaTwS0knAXeQDXUBQES091O+L/ARSR8CegMDJF0HLJc0LCKWSRoGrGjn85uZWTsVaT56FZgE3M/rTUcz2lthRJwZESPThXDHAn+KiOOB24DxabXxwK3trcPMzNqnyJHC6cDOEfFshWM5H5gi6URgEXBMhesz63T77jS4bFPRvjsNrkE0Zm1X5EhhHvByJSqPiLsj4sg0/VxEHBQRo9NfN8Ja3bn+pH3ekAD23Wkw15+0T40iMmubIkcKG4GHJP2ZTfsUJlQsKrM65gRg9axIUpiaHmZm1s0VuZ/C5NbWMbPX+R7NVs9aTQqSnqTMWEcRsWNFIjKrY1NnL2XiTXNYvzH7yixdvY6JN80BfI9mqw9Fmo/Glkz3JjsryKdSmJXx3dvn5Qmh0fqNwXdvn+ekYHWh1bOP0llBjY+lEfFjoDMGxDPrdla9XH6I7ObKzbqaIs1He5bMbkZ25NC/YhGZmVnNFGk+Kr2vwgbSvQ4qEo1ZnRPlbzaiagdi1k5Fzj76QDUCMesOmhuh0SM3Wr0o0ny0JfB/yIbPztePiO9VLiyz+tQgsbHMDZkb5GMFqw9Fmo9uBV4gGwjvlVbWNevRyiWElsrNupoiSWFkRBxW8UjMugEfKVi9KzIg3n2S3lHxSMy6AR8pWL0rcqSwH/DZdGXzK6QTLCJit4pGZmZmVVckKRxe8SjMzKxLKHJK6sJqBGJmZrVXpE/BzMx6CCcFMzPLOSmYmVnOScHMzHJOCmZmlnNSMDOznJOCmZnlnBTMzCznpGBmZjknBTMzyzkpmJlZrupJQdKbJf1Z0nxJ8ySdmsoHS7pL0mPp76Bqx2Zm1tPV4khhA/D1iHgbsDfwFUm7AGcA0yNiNDA9zZuZWRVVPSlExLKImJWm1wDzgRHAUcDktNpkYFy1YzMz6+lq2qcgaRSwB/AAsG1ELIMscQDbNLPNyZJmSJqxcuXKqsVqZtYT1CwpSOoH3AycFhEvFt0uIq6IiLERMXbo0KGVC9DMrAeqSVKQ1IssIVwfEbek4uWShqXlw4AVtYjNzKwnq8XZRwKuAuZHxIUli24Dxqfp8cCt1Y7NzKynK3KP5s62L/BpYK6kh1LZt4DzgSmSTgQWAcfUIDYzsx6t6kkhIu4F1Mzig6oZi5mZbcpXNJuZWc5JwczMck4KZmaWc1IwM7Ock4KZmeWcFMzMLOekYGZmOScFMzPLOSmYmVnOScHMzHJOCmZmlnNSMDOznJOCmZnlnBTMzCznpGBmZjknBTMzyzkpmJlZzknBzMxyTgpmZpZzUjAzs5yTgpmZ5ZwUzMws56RgZmY5JwUzM8s5KZiZWc5JwczMck4KZmaW63JJQdJhkhZIelzSGbWOx8ysJ+lSSUFSA3AJcDiwC3CcpF1qG5WZWc/RpZIC8G7g8Yh4IiJeBX4NHFXjmMzMeoyulhRGAItL5pekspykkyXNkDRj5cqVVQ3OzKy762pJQWXKYpOZiCsiYmxEjB06dGiVwjIr5qnzj2hTuVlXs3mtA2hiCfDmkvmRwNM1isWsXZwArJ51tSOFfwCjJe0gaQvgWOC2GsdkZtZjdKkjhYjYIOmrwDSgAbg6IubVOCwzsx6jSyUFgIj4PfD7WsdhZtYTdbXmIzMzqyEnBTMzyykiWl+ri5K0ElhY6zjMmjEEeLbWQZiVsX1ElD2nv66TgllXJmlGRIytdRxmbeHmIzMzyzkpmJlZzknBrHKuqHUAZm3lPgUzM8v5SMHMzHJOCmZmlnNSMGsjSddKOrrWcZhVgpOCmZnlnBTMWiHpM5L+KWmOpF+m4vdJuk/SE6VHDZImSvpHWv+7JeVTJc2UNE/SySXlayVdIGmWpOmSfOcoqyknBbMWSNoV+DZwYETsDpyaFg0D9gOOBM5P6x4CjCa71/g7gb0kvS+t/7mI2AsYC0yQ9KZU3heYFRF7An8Bzq74Tpm1wEnBrGUHAjdFxLMAEfF8Kp8aEa9FxCPAtqnskPSYDcwC3kqWJCBLBHOAv5PdXbCx/DXgxjR9HVmiMauZLnc/BbMuRjS5T3jySpN1Gv+eFxGXb/IE0gHAwcA+EfGypLuB3s3U5wuHrKZ8pGDWsunAxxubeyQNbmHdacDnJPVL646QtA2wNbAqJYS3AnuXbLMZ0Ngn8Ung3s7eAbO28JGCWQsiYp6kc4G/SNpI1jTU3Lp3SnobcL8kgLXA8cAfgC9K+iewgKwJqdFLwK6SZgIvAJ+ozJ6YFeNhLsxqSNLaiOhX6zjMGrn5yMzMcj5SMDOznI8UzMws56RgZmY5JwUzM8s5KViPJOl7kg6ucp0HSHpvNes0aytfp2A9jqSGiDirBlUfQHbtwn2VqkDZBRKKiNcqVYd1bz5SsG5D0ihJj0qanEYpvUnSVmnZU5LOknQvcEzpPREkvSuNeDpH0oOS+ktqkDSpZMTTLzRT5xtGUJX0YUkPSJot6Y+StpU0Cvgi8DVJD0naX9JQSTenOv4had+0/VBJd6WRUy+XtFDSkLTsdEkPp8dpJfs9X9KlZGMu/V9JPyqJ8SRJF1boZbfuJiL88KNbPIBRZGMH7Zvmrwa+kaafAr5Zsu61ZMNLbAE8AbwrlQ8gO4I+GfhOKtsSmAHs0KS+XcmuUB6S5genv4N4/XTvzwMXpOlzGuNJ878C9kvT2wHz0/RPgTPT9GFpn4YAewFzyUZW7QfMA/ZI+/0asHfapi/wL6BXmr8PeEet3x8/6uPh5iPrbhZHxN/S9HXABOCHaf7GMuuPAZZFxD8AIuJFyIfB3q3kXglbk41s+mTJts2NoDoSuFHSMLKkU7pNqYOBXdKQGAADJPUnGyn1o+k5/yBpVVq+H/DbiHgpxXgLsD9wG7AwIv6etnlJ0p+AIyXNJ0sOc5uJwWwTTgrW3TS9GrN0/qUy6zc3CqqAUyJiWgt1NbftT4ALI+K2NELqOc1svxnZyKnrNnnSkixRpr7mNN23K4FvAY8C17Swndkm3Kdg3c12kvZJ08fR+qijjwLDJb0LIPUnbE424umXJPVK5W+R1LfJts2NoLo1sDRNjy9Zfw3Qv2T+TuCrjTOS3pkm7wU+nsoOIWuOAvgrME7SVimWjwL3lNupiHiA7L4NnwRuaOU1MMs5KVh3Mx8Yn0YkHQxc1tLKEfEq2cikP0k3wbmL7F4HVwKPALMkPQxcTpMj64iYBzSOoDoHaOzMPQf4jaR7gGdLNrkd+GhjRzNZ09bY1FH9CFlHNMB3gUMkzQIOB5YBayJiFllfyIPAA8CVEdHsqK3AFOBvEbGqhXXMNuGxj6zbSGf43BERb691LB0haUtgY0RsSEc9l0XEO9vxPHcAP4qI6Z0do3Vf7lMw63q2A6ZI2gx4FTipLRtLGkh2NDHHCcHaykcKZmaWc5+CmZnlnBTMzCznpGBmZjknBTMzyzkpmJlZ7v8D+PM0NRrJU80AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(data[(data.Purchase==1) & (data.price >= 11) & (data.price<=24)].price_category, data[(data.Purchase==1) &  (data.price >= 11) & (data.price<=24)].user_session)\n",
    "plt.xlabel('price category')\n",
    "plt.ylabel('number of sessions')\n",
    "plt.title('Kaggle Cosmetics Data Filter by Price')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_id              38505\n",
       "product_id            4366\n",
       "NumTimesCarted          32\n",
       "NumTimesPurchased       10\n",
       "NumTimesRemoved         22\n",
       "NumTimesViewed          53\n",
       "Purchase                 1\n",
       "price                 3640\n",
       "user_session            41\n",
       "category_id            252\n",
       "price_category           1\n",
       "dtype: int64"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[(data.Purchase==1) & (data.price >= 11) & (data.price<=24)].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ec2-user/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/numpy/lib/histograms.py:839: RuntimeWarning: invalid value encountered in greater_equal\n",
      "  keep = (tmp_a >= first_edge)\n",
      "/home/ec2-user/anaconda3/envs/tensorflow2_p36/lib/python3.6/site-packages/numpy/lib/histograms.py:840: RuntimeWarning: invalid value encountered in less_equal\n",
      "  keep &= (tmp_a <= last_edge)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(array([1.23361e+05, 1.46570e+04, 2.08600e+03, 4.64000e+02, 9.60000e+01,\n",
       "        4.90000e+01, 5.00000e+00, 1.00000e+00, 8.00000e+00, 9.00000e+00]),\n",
       " array([  0.66 ,  23.914,  47.168,  70.422,  93.676, 116.93 , 140.184,\n",
       "        163.438, 186.692, 209.946, 233.2  ]),\n",
       " <BarContainer object of 10 artists>)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAD4CAYAAADy46FuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAATXUlEQVR4nO3dcYxdZX7e8e9TO6FktxADBlHbqmmxmgBqd4PFul0p2tYVdrtRTCWQZtUEq7HkFDntporU2ukfVIksgdqGFqkguYFiyHbBcnaF1S3ZtUyiVSRidthFMcahjAIFBxdPakpoK0hNfv3jvqPeGe68tufaHuz5fqSre+7vvO+57zk68HDec+aSqkKSpPn8ucUegCTp082gkCR1GRSSpC6DQpLUZVBIkrqWL/YAzrfrrruu1q5du9jDkKRLyksvvfTHVbVy1LrLLijWrl3L5OTkYg9Dki4pSf7bfOucepIkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHVddn+ZPa61O7+1KN/75gNfXpTvlaQz8YpCktR1xqBI8niSk0leGar9qyR/kOT3k3wzyY8OrduVZCrJa0k2DdVvT3KkrXs4SVr9iiTPtPrhJGuH+mxN8np7bT1fOy1JOntnc0XxBLB5Tu0gcFtV/TXgvwK7AJLcAkwAt7Y+jyRZ1vo8CmwH1rXXzDa3Ae9V1c3AQ8CDbVvXAPcDXwDuAO5PsuLcd1GSNI4zBkVVfRc4Naf2nao63T7+HrC6LW8Bnq6qj6rqDWAKuCPJjcBVVfVCVRXwJHDXUJ+9bXk/sLFdbWwCDlbVqap6j0E4zQ0sSdIFdj7uUfwc8FxbXgW8PbTueKutastz67P6tPB5H7i2s61PSLI9yWSSyenp6bF2RpI021hBkeRfAKeBr82URjSrTn2hfWYXq/ZU1fqqWr9y5cj/74YkaYEWHBTt5vJPAf+gTSfB4L/61ww1Ww280+qrR9Rn9UmyHLiawVTXfNuSJF1ECwqKJJuBfw78dFX9n6FVB4CJ9iTTTQxuWr9YVSeAD5JsaPcf7gWeHeoz80TT3cDzLXi+DdyZZEW7iX1nq0mSLqIz/sFdkq8DXwKuS3KcwZNIu4ArgIPtKdffq6p/VFVHk+wDXmUwJbWjqj5um7qPwRNUVzK4pzFzX+Mx4KkkUwyuJCYAqupUkl8Fvtfa/UpVzbqpLkm68M4YFFX1lRHlxzrtdwO7R9QngdtG1D8E7plnW48Dj59pjJKkC8e/zJYkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdZ0xKJI8nuRkkleGatckOZjk9fa+YmjdriRTSV5LsmmofnuSI23dw0nS6lckeabVDydZO9Rna/uO15NsPW97LUk6a2dzRfEEsHlObSdwqKrWAYfaZ5LcAkwAt7Y+jyRZ1vo8CmwH1rXXzDa3Ae9V1c3AQ8CDbVvXAPcDXwDuAO4fDiRJ0sVxxqCoqu8Cp+aUtwB72/Je4K6h+tNV9VFVvQFMAXckuRG4qqpeqKoCnpzTZ2Zb+4GN7WpjE3Cwqk5V1XvAQT4ZWJKkC2yh9yhuqKoTAO39+lZfBbw91O54q61qy3Prs/pU1WngfeDazrY+Icn2JJNJJqenpxe4S5KkUc73zeyMqFWnvtA+s4tVe6pqfVWtX7ly5VkNVJJ0dhYaFO+26STa+8lWPw6sGWq3Gnin1VePqM/qk2Q5cDWDqa75tiVJuogWGhQHgJmnkLYCzw7VJ9qTTDcxuGn9Ypue+iDJhnb/4d45fWa2dTfwfLuP8W3gziQr2k3sO1tNknQRLT9TgyRfB74EXJfkOIMnkR4A9iXZBrwF3ANQVUeT7ANeBU4DO6rq47ap+xg8QXUl8Fx7ATwGPJVkisGVxETb1qkkvwp8r7X7laqae1NdknSBnTEoquor86zaOE/73cDuEfVJ4LYR9Q9pQTNi3ePA42caoyTpwvEvsyVJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoaKyiS/NMkR5O8kuTrSf58kmuSHEzyentfMdR+V5KpJK8l2TRUvz3Jkbbu4SRp9SuSPNPqh5OsHWe8kqRzt+CgSLIK+CfA+qq6DVgGTAA7gUNVtQ441D6T5Ja2/lZgM/BIkmVtc48C24F17bW51bcB71XVzcBDwIMLHa8kaWHGnXpaDlyZZDnwI8A7wBZgb1u/F7irLW8Bnq6qj6rqDWAKuCPJjcBVVfVCVRXw5Jw+M9vaD2ycudqQJF0cCw6Kqvoj4F8DbwEngPer6jvADVV1orU5AVzfuqwC3h7axPFWW9WW59Zn9amq08D7wLULHbMk6dyNM/W0gsF/8d8E/EXgM0l+ptdlRK069V6fuWPZnmQyyeT09HR/4JKkczLO1NPfAd6oqumq+r/AN4C/CbzbppNo7ydb++PAmqH+qxlMVR1vy3Prs/q06a2rgVNzB1JVe6pqfVWtX7ly5Ri7JEmaa5ygeAvYkORH2n2DjcAx4ACwtbXZCjzblg8AE+1JppsY3LR+sU1PfZBkQ9vOvXP6zGzrbuD5dh9DknSRLF9ox6o6nGQ/8H3gNPADYA/wWWBfkm0MwuSe1v5okn3Aq639jqr6uG3uPuAJ4ErgufYCeAx4KskUgyuJiYWOV5K0MAsOCoCquh+4f075IwZXF6Pa7wZ2j6hPAreNqH9ICxpJ0uLwL7MlSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV1jBUWSH02yP8kfJDmW5G8kuSbJwSSvt/cVQ+13JZlK8lqSTUP125McaeseTpJWvyLJM61+OMnaccYrSTp3415R/Dvgt6rqx4C/DhwDdgKHqmodcKh9JsktwARwK7AZeCTJsradR4HtwLr22tzq24D3qupm4CHgwTHHK0k6RwsOiiRXAT8JPAZQVX9aVf8T2ALsbc32Ane15S3A01X1UVW9AUwBdyS5Ebiqql6oqgKenNNnZlv7gY0zVxuSpItjnCuKvwxMA/8xyQ+S/HqSzwA3VNUJgPZ+fWu/Cnh7qP/xVlvVlufWZ/WpqtPA+8C1cweSZHuSySST09PTY+ySJGmucYJiOfATwKNV9Xngf9OmmeYx6kqgOvVen9mFqj1Vtb6q1q9cubI/aknSORknKI4Dx6vqcPu8n0FwvNumk2jvJ4farxnqvxp4p9VXj6jP6pNkOXA1cGqMMUuSztGCg6Kq/jvwdpK/2kobgVeBA8DWVtsKPNuWDwAT7UmmmxjctH6xTU99kGRDu/9w75w+M9u6G3i+3ceQJF0ky8fs/4+BryX5YeAPgX/IIHz2JdkGvAXcA1BVR5PsYxAmp4EdVfVx2859wBPAlcBz7QWDG+VPJZlicCUxMeZ4JUnnaKygqKqXgfUjVm2cp/1uYPeI+iRw24j6h7SgkSQtDv8yW5LUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpK6xgyLJsiQ/SPKf2+drkhxM8np7XzHUdleSqSSvJdk0VL89yZG27uEkafUrkjzT6oeTrB13vJKkc3M+rii+Chwb+rwTOFRV64BD7TNJbgEmgFuBzcAjSZa1Po8C24F17bW51bcB71XVzcBDwIPnYbySpHMwVlAkWQ18Gfj1ofIWYG9b3gvcNVR/uqo+qqo3gCngjiQ3AldV1QtVVcCTc/rMbGs/sHHmakOSdHGMe0Xxb4F/BvzZUO2GqjoB0N6vb/VVwNtD7Y632qq2PLc+q09VnQbeB64dc8ySpHOw4KBI8lPAyap66Wy7jKhVp97rM3cs25NMJpmcnp4+y+FIks7GOFcUXwR+OsmbwNPA307yG8C7bTqJ9n6ytT8OrBnqvxp4p9VXj6jP6pNkOXA1cGruQKpqT1Wtr6r1K1euHGOXJElzLTgoqmpXVa2uqrUMblI/X1U/AxwAtrZmW4Fn2/IBYKI9yXQTg5vWL7bpqQ+SbGj3H+6d02dmW3e37/jEFYUk6cJZfgG2+QCwL8k24C3gHoCqOppkH/AqcBrYUVUftz73AU8AVwLPtRfAY8BTSaYYXElMXIDxSpI6zktQVNXvAL/Tlv8HsHGedruB3SPqk8BtI+of0oJGkrQ4/MtsSVKXQSFJ6jIoJEldBoUkqcugkCR1GRSSpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVLXgoMiyZokv53kWJKjSb7a6tckOZjk9fa+YqjPriRTSV5LsmmofnuSI23dw0nS6lckeabVDydZO8a+SpIWYJwritPAL1XVjwMbgB1JbgF2Aoeqah1wqH2mrZsAbgU2A48kWda29SiwHVjXXptbfRvwXlXdDDwEPDjGeCVJC7DgoKiqE1X1/bb8AXAMWAVsAfa2ZnuBu9ryFuDpqvqoqt4ApoA7ktwIXFVVL1RVAU/O6TOzrf3AxpmrDUnSxXFe7lG0KaHPA4eBG6rqBAzCBLi+NVsFvD3U7XirrWrLc+uz+lTVaeB94NoR3789yWSSyenp6fOxS5KkZuygSPJZ4DeBX6yqP+k1HVGrTr3XZ3ahak9Vra+q9StXrjzTkCVJ52CsoEjyQwxC4mtV9Y1WfrdNJ9HeT7b6cWDNUPfVwDutvnpEfVafJMuBq4FT44xZknRuxnnqKcBjwLGq+rWhVQeArW15K/DsUH2iPcl0E4Ob1i+26akPkmxo27x3Tp+Zbd0NPN/uY0iSLpLlY/T9IvCzwJEkL7faLwMPAPuSbAPeAu4BqKqjSfYBrzJ4YmpHVX3c+t0HPAFcCTzXXjAIoqeSTDG4kpgYY7ySpAVYcFBU1e8y+h4CwMZ5+uwGdo+oTwK3jah/SAsaSdLi8C+zJUldBoUkqcugkCR1jXMzW+fR2p3fWrTvfvOBLy/ad0v69POKQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnLoJAkdRkUkqQug0KS1GVQSJK6DApJUpdBIUnqMigkSV0GhSSpy6CQJHUZFJKkLoNCktRlUEiSugwKSVKXQSFJ6lq+2APQ4lu781uL8r1vPvDlRfleSefmkriiSLI5yWtJppLsXOzxSNJS8qkPiiTLgH8P/F3gFuArSW5Z3FFJ0tJxKUw93QFMVdUfAiR5GtgCvLqoo9LYnPKSLg2XQlCsAt4e+nwc+MJwgyTbge3t4/9K8toCv+s64I8X2Pdycdkfgzx4Vs0u++NwFjwGA0vlOPyl+VZcCkGREbWa9aFqD7Bn7C9KJqtq/bjbuZR5DAY8Dh6DGR6HS+AeBYMriDVDn1cD7yzSWCRpybkUguJ7wLokNyX5YWACOLDIY5KkJeNTP/VUVaeT/ALwbWAZ8HhVHb1AXzf29NVlwGMw4HHwGMxY8schVXXmVpKkJetSmHqSJC0ig0KS1GVQsLR/IiTJm0mOJHk5yWSrXZPkYJLX2/uKxR7n+ZTk8SQnk7wyVJt3n5PsaufGa0k2Lc6oz795jsO/TPJH7Xx4OcnfG1p32R2HJGuS/HaSY0mOJvlqqy+586FnyQeFPxECwN+qqs8NPSu+EzhUVeuAQ+3z5eQJYPOc2sh9bufCBHBr6/NIO2cuB0/wyeMA8FA7Hz5XVf8FLuvjcBr4par6cWADsKPt61I8H+a15IOCoZ8Iqao/BWZ+ImQp2wLsbct7gbsWbyjnX1V9Fzg1pzzfPm8Bnq6qj6rqDWCKwTlzyZvnOMznsjwOVXWiqr7flj8AjjH4NYgldz70GBSjfyJk1SKNZTEU8J0kL7WfQgG4oapOwOAfJOD6RRvdxTPfPi/F8+MXkvx+m5qamXK57I9DkrXA54HDeD7MYlCcxU+EXOa+WFU/wWDqbUeSn1zsAX3KLLXz41HgrwCfA04A/6bVL+vjkOSzwG8Cv1hVf9JrOqJ22RyH+RgUS/wnQqrqnfZ+Evgmg8vod5PcCNDeTy7eCC+a+fZ5SZ0fVfVuVX1cVX8G/Af+/7TKZXsckvwQg5D4WlV9o5U9H4YYFEv4J0KSfCbJX5hZBu4EXmGw/1tbs63As4szwotqvn0+AEwkuSLJTcA64MVFGN9FMfMvx+bvMzgf4DI9DkkCPAYcq6pfG1rl+TDkU/8THhfaRf6JkE+bG4BvDv5ZYTnwn6rqt5J8D9iXZBvwFnDPIo7xvEvydeBLwHVJjgP3Aw8wYp+r6miSfQz+/yengR1V9fGiDPw8m+c4fCnJ5xhMp7wJ/Dxc1sfhi8DPAkeSvNxqv8wSPB96/AkPSVKXU0+SpC6DQpLUZVBIkroMCklSl0EhSeoyKCRJXQaFJKnr/wENzwN4uBHNmgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(hits.price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rwbxllg6-ZG_",
    "outputId": "bd75aaca-6b20-47a7-c7d4-9949063b3e5e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num users: 1637770, num_items 53903.\n",
      "Num values: 10111571\n"
     ]
    }
   ],
   "source": [
    "dataset_k, train, test, int_k = random_split(data, 'user_id', 'product_id', 'Purchase')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "calc_sparsity(data, 'user_id', 'product_id', 'Purchase')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/"
    },
    "id": "2FGaACB2_Pb1",
    "outputId": "a6a3ad96-c18c-4fd4-d15d-64b657771ac5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32\n",
      "CPU times: user 3min 12s, sys: 188 ms, total: 3min 12s\n",
      "Wall time: 1min 15s\n"
     ]
    }
   ],
   "source": [
    "for n in [32, 64, 192]: \n",
    "    print(n)\n",
    "    model_c = model_fit_eval('warp', train, test, n, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 37 s, sys: 81.9 ms, total: 37 s\n",
      "Wall time: 14.4 s\n",
      "Collaborative filtering test AUC: 0.9077277\n",
      "Collaborative filtering test precision: 0.043549385\n",
      "Collaborative filtering test recall: 0.1560249026255479\n"
     ]
    }
   ],
   "source": [
    " model_c = model_fit_eval('warp-kos', train, test, 128, k=7, n=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Kaggle Cosmetics + Groupby"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>event_type</th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>NumTimesCarted</th>\n",
       "      <th>NumTimesPurchased</th>\n",
       "      <th>NumTimesRemoved</th>\n",
       "      <th>NumTimesViewed</th>\n",
       "      <th>Purchase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9794320</td>\n",
       "      <td>5706475</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10280338</td>\n",
       "      <td>5876344</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14297993</td>\n",
       "      <td>5804820</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14297993</td>\n",
       "      <td>5882654</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20554973</td>\n",
       "      <td>5706479</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167790</th>\n",
       "      <td>622083330</td>\n",
       "      <td>5924482</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167791</th>\n",
       "      <td>622083780</td>\n",
       "      <td>5846096</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167792</th>\n",
       "      <td>622086358</td>\n",
       "      <td>5908170</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167793</th>\n",
       "      <td>622086382</td>\n",
       "      <td>5804820</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167794</th>\n",
       "      <td>622086800</td>\n",
       "      <td>5765359</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>167795 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "event_type    user_id  product_id  NumTimesCarted  NumTimesPurchased  \\\n",
       "0             9794320     5706475               0                  0   \n",
       "1            10280338     5876344               0                  0   \n",
       "2            14297993     5804820               0                  0   \n",
       "3            14297993     5882654               0                  0   \n",
       "4            20554973     5706479               0                  0   \n",
       "...               ...         ...             ...                ...   \n",
       "167790      622083330     5924482               0                  0   \n",
       "167791      622083780     5846096               0                  0   \n",
       "167792      622086358     5908170               0                  0   \n",
       "167793      622086382     5804820               0                  0   \n",
       "167794      622086800     5765359               0                  0   \n",
       "\n",
       "event_type  NumTimesRemoved  NumTimesViewed  Purchase  \n",
       "0                         0               1         0  \n",
       "1                         0               1         0  \n",
       "2                         0               1         0  \n",
       "3                         0               1         0  \n",
       "4                         0               2         0  \n",
       "...                     ...             ...       ...  \n",
       "167790                    0               1         0  \n",
       "167791                    0               1         0  \n",
       "167792                    0               1         0  \n",
       "167793                    0               1         0  \n",
       "167794                    0               1         0  \n",
       "\n",
       "[167795 rows x 7 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_purchase2 = num_purchase.rename(columns={'customerVisitorId':\"user_id\", \"ID\":\"product_id\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>numPurchase</th>\n",
       "      <th>price</th>\n",
       "      <th>Purchase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cjo06cah000013ea3jqwotb8g</td>\n",
       "      <td>AAM013</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cjo06cah000013ea3jqwotb8g</td>\n",
       "      <td>AAM016</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>cjo06cah000013ea3jqwotb8g</td>\n",
       "      <td>AAM019</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>cjo06cah000013ea3jqwotb8g</td>\n",
       "      <td>ARG010</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>cjo06cah000013ea3jqwotb8g</td>\n",
       "      <td>DB088</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997359</th>\n",
       "      <td>cki8fds6p000139dnzl8n11ln</td>\n",
       "      <td>SW106</td>\n",
       "      <td>5</td>\n",
       "      <td>9.76</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997360</th>\n",
       "      <td>cki8h8bl300013bdvycc084d4</td>\n",
       "      <td>SWD015</td>\n",
       "      <td>0</td>\n",
       "      <td>12.49</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997361</th>\n",
       "      <td>cki8i50go000139dbjuozjlpv</td>\n",
       "      <td>SWA051</td>\n",
       "      <td>3</td>\n",
       "      <td>14.79</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997362</th>\n",
       "      <td>cki8pfl1y000139ebdeezn6pn</td>\n",
       "      <td>SWA051</td>\n",
       "      <td>3</td>\n",
       "      <td>14.79</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997363</th>\n",
       "      <td>cki8qpz0f000139dbxyagqagm</td>\n",
       "      <td>SW1113</td>\n",
       "      <td>21</td>\n",
       "      <td>2.54</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>997364 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          user_id product_id  numPurchase  price  Purchase\n",
       "0       cjo06cah000013ea3jqwotb8g     AAM013            0    NaN         0\n",
       "1       cjo06cah000013ea3jqwotb8g     AAM016            0    NaN         0\n",
       "2       cjo06cah000013ea3jqwotb8g     AAM019            0    NaN         0\n",
       "3       cjo06cah000013ea3jqwotb8g     ARG010            0    NaN         0\n",
       "4       cjo06cah000013ea3jqwotb8g      DB088            0    NaN         0\n",
       "...                           ...        ...          ...    ...       ...\n",
       "997359  cki8fds6p000139dnzl8n11ln      SW106            5   9.76         1\n",
       "997360  cki8h8bl300013bdvycc084d4     SWD015            0  12.49         0\n",
       "997361  cki8i50go000139dbjuozjlpv     SWA051            3  14.79         1\n",
       "997362  cki8pfl1y000139ebdeezn6pn     SWA051            3  14.79         1\n",
       "997363  cki8qpz0f000139dbxyagqagm     SW1113           21   2.54         1\n",
       "\n",
       "[997364 rows x 5 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_purchase2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined = pd.concat([data[['user_id', 'product_id', 'Purchase']], num_purchase2[['user_id', 'product_id', 'Purchase']]], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>event_type</th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>Purchase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9794320</td>\n",
       "      <td>5706475</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10280338</td>\n",
       "      <td>5876344</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14297993</td>\n",
       "      <td>5804820</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14297993</td>\n",
       "      <td>5882654</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20554973</td>\n",
       "      <td>5706479</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997359</th>\n",
       "      <td>cki8fds6p000139dnzl8n11ln</td>\n",
       "      <td>SW106</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997360</th>\n",
       "      <td>cki8h8bl300013bdvycc084d4</td>\n",
       "      <td>SWD015</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997361</th>\n",
       "      <td>cki8i50go000139dbjuozjlpv</td>\n",
       "      <td>SWA051</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997362</th>\n",
       "      <td>cki8pfl1y000139ebdeezn6pn</td>\n",
       "      <td>SWA051</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997363</th>\n",
       "      <td>cki8qpz0f000139dbxyagqagm</td>\n",
       "      <td>SW1113</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1165159 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "event_type                    user_id product_id  Purchase\n",
       "0                             9794320    5706475         0\n",
       "1                            10280338    5876344         0\n",
       "2                            14297993    5804820         0\n",
       "3                            14297993    5882654         0\n",
       "4                            20554973    5706479         0\n",
       "...                               ...        ...       ...\n",
       "997359      cki8fds6p000139dnzl8n11ln      SW106         1\n",
       "997360      cki8h8bl300013bdvycc084d4     SWD015         0\n",
       "997361      cki8i50go000139dbjuozjlpv     SWA051         1\n",
       "997362      cki8pfl1y000139ebdeezn6pn     SWA051         1\n",
       "997363      cki8qpz0f000139dbxyagqagm     SW1113         1\n",
       "\n",
       "[1165159 rows x 3 columns]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num users: 151533, num_items 17776.\n",
      "Num values: 1165159\n"
     ]
    }
   ],
   "source": [
    "train_comb, test_comb = random_split(combined, 'user_id', 'product_id', 'Purchase')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 31 s, sys: 49.7 ms, total: 31 s\n",
      "Wall time: 11.4 s\n",
      "Collaborative filtering test AUC: 0.94967175\n",
      "Collaborative filtering test precision: 0.20758086\n",
      "Collaborative filtering test recall: 0.31428044888114814\n"
     ]
    }
   ],
   "source": [
    " model_comb = model_fit_eval('warp', train_comb, test_comb, 128 )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Benchmark other data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lightfm.datasets import fetch_movielens\n",
    "\n",
    "data = fetch_movielens(min_rating=5.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 987 ms, sys: 1.46 ms, total: 989 ms\n",
      "Wall time: 337 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<lightfm.lightfm.LightFM at 0x7f2a6582ae80>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = LightFM(loss='warp', no_components=192, item_alpha=1e-6, random_state=2)\n",
    "%time model.fit(data['train'], epochs=10, num_threads=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collaborative filtering test AUC: 0.91579705\n",
      "Collaborative filtering test precision: 0.073447816\n",
      "Collaborative filtering test recall: 0.13724497284602966\n"
     ]
    }
   ],
   "source": [
    "  test_auc = auc_score(model, data['test'], train_interactions=data['train'], num_threads=3).mean()\n",
    "  print('Collaborative filtering test AUC: %s' % test_auc)\n",
    "  prec = precision_at_k(model, data['test'], train_interactions=data['train'], num_threads=3, k=5).mean()\n",
    "  print('Collaborative filtering test precision: %s' % prec)\n",
    "  recal = recall_at_k(model, data['test'], train_interactions=data['train'], num_threads=3, k=5).mean()\n",
    "  print('Collaborative filtering test recall: %s' % recal)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "Groupby - Recommendation.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "1312c30ed5864aaca1944d4141b0db75": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2665cb8f053b4095b56af209565206cc": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "337eb8ba003743a695ba6446c95d8fc8": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "618db2266cf64af8acf0eff931ece47b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "6ec87fdb66ea43a0a34dd5fc4224a861": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_2665cb8f053b4095b56af209565206cc",
      "placeholder": "​",
      "style": "IPY_MODEL_337eb8ba003743a695ba6446c95d8fc8",
      "value": " 100/100 [07:53&lt;00:00,  4.73s/it]"
     }
    },
    "7d7201afc99c4b56b4ae85cea7db6e8b": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_93b572d1795349ed99e60a46dc7669a3",
       "IPY_MODEL_6ec87fdb66ea43a0a34dd5fc4224a861"
      ],
      "layout": "IPY_MODEL_8785929d29d644eda9468a8f69ec0802"
     }
    },
    "8785929d29d644eda9468a8f69ec0802": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "93b572d1795349ed99e60a46dc7669a3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1312c30ed5864aaca1944d4141b0db75",
      "max": 100,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_618db2266cf64af8acf0eff931ece47b",
      "value": 100
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}