cross_validation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fine-Scale Prediction of People's Home Location using Social Media Footprints\n",
    "\n",
    "_**Authors:** Hamdi Kavak, Daniele Vernon-Bido, and Jose Padilla_\n",
    "\n",
    "_**Submitted:** SBP-BRIMS 2018 on January 11, 2018._\n",
    "\n",
    "## Repeated 5-fold validation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### - Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Home Location Prediction Paper ###########\n",
    "# Task: Training and test\n",
    "# Author: Hamdi Kavak\n",
    "# Created: January 03, 2017\n",
    "#########################################\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.model_selection import KFold\n",
    "from random import shuffle\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### - Validation function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate(training_df, feature_columns, class_column, number_of_repeats=5, k_fold=5, evaluation_treshold_diff=None):\n",
    "    \n",
    "    # evaluates all points including non-home tweets for each individial. \n",
    "    # gets the highest score using weights and intercept values and eliminates when highest score is below the treshold for each individual again.\n",
    "    # returns \n",
    "    # - average accuracy for all runs\n",
    "    # - average coverage (based on the treshold)\n",
    "    # - min, max, median, and average max points\n",
    "    # - creates an average weights and intercept values and report how it performes (accuracy and coverage again)\n",
    "    \n",
    "    # get all unique user ids\n",
    "    user_ids = training_df.user_id.unique()\n",
    "    \n",
    "    # create a k-fold object\n",
    "    kf = KFold(n_splits=k_fold)\n",
    "    \n",
    "    max_points = []\n",
    "    covarege_percentages = []\n",
    "    accuracy_list = []\n",
    "    f1_list = []\n",
    "    precision_list = []\n",
    "    recall_list = []\n",
    "    weights_list = []\n",
    "    intercept_list = []\n",
    "    \n",
    "    for i in range(number_of_repeats):\n",
    "        print 'repeat:',i\n",
    "        #shuffle user ids for repeated k-fold cross validation\n",
    "        shuffle(user_ids)\n",
    "        \n",
    "        # splitting is per user so we actually splitting user ids\n",
    "        for train_index, test_index in kf.split(user_ids):\n",
    "            training_ids = user_ids[train_index]\n",
    "            test_ids = user_ids[test_index]\n",
    "            \n",
    "            # get actual training and test sets\n",
    "            df_training = df[df.user_id.isin(training_ids)]\n",
    "            df_test = df[df.user_id.isin(test_ids)]\n",
    "        \n",
    "            # train the model\n",
    "            model = SVC(kernel='linear',class_weight='balanced')\n",
    "            y_training = df_training[class_name]\n",
    "            x_training = df_training[feature_names]\n",
    "            model.fit(x_training, y_training)\n",
    "            \n",
    "            weights_list.append(model.coef_.ravel())\n",
    "            intercept_list.append(model.intercept_)\n",
    "            \n",
    "                        \n",
    "            treshold = model.intercept_ if evaluation_treshold_diff == None else avg_intercept + evaluation_treshold_diff \n",
    "            # print 'weight:', model.coef_.ravel(), ' - intercept:', model.intercept_, ' - treshold:', treshold\n",
    "            \n",
    "            users_covered = 0\n",
    "            expected_list = []\n",
    "            predicted_list = []\n",
    "            \n",
    "            # go through all test users\n",
    "            for user_id in test_ids:\n",
    "                \n",
    "                df_test_user = df_test.loc[(df_test['user_id'] == user_id)]\n",
    "                \n",
    "                max_score = -999999\n",
    "                current_expected = False\n",
    "                \n",
    "                for i,row in df_test_user.iterrows():\n",
    "                    # prediction function\n",
    "                    \n",
    "                    total_score = sum(model.coef_.ravel()*row[feature_names].values)\n",
    "\n",
    "                    if (total_score > max_score):\n",
    "                        max_score = total_score\n",
    "                        current_expected = row[class_name]\n",
    "\n",
    "                if max_score >= treshold:\n",
    "                    users_covered = users_covered + 1\n",
    "                    # lets record this user's prediction \n",
    "                    \n",
    "                    predicted_list.append(max_score >= model.intercept_)\n",
    "                    expected_list.append(current_expected)\n",
    "                    \n",
    "                max_points.append(max_score)\n",
    "                \n",
    "            accuracy_score = metrics.accuracy_score(expected_list, predicted_list)\n",
    "            f1_score = metrics.f1_score(expected_list, predicted_list)\n",
    "            precision_score = metrics.precision_score(expected_list, predicted_list)\n",
    "            recall_score = metrics.recall_score(expected_list, predicted_list)\n",
    "            \n",
    "            accuracy_list.append(accuracy_score)\n",
    "            f1_list.append(f1_score)\n",
    "            precision_list.append(precision_score)\n",
    "            recall_list.append(recall_score)\n",
    "            \n",
    "            coverage_perc = users_covered*100.0/len(test_ids)\n",
    "            covarege_percentages.append(coverage_perc)\n",
    "    \n",
    "    \n",
    "    \n",
    "    avg_weight = np.mean(weights_list, axis=0)\n",
    "    avg_intercept = np.mean(intercept_list)\n",
    "    \n",
    "    #print 'avg_weight:',avg_weight\n",
    "    \n",
    "    final_users_covered = 0\n",
    "    final_expected_list = []\n",
    "    final_predicted_list = []\n",
    "    \n",
    "    treshold = avg_intercept if evaluation_treshold_diff == None else avg_intercept + evaluation_treshold_diff \n",
    "\n",
    "    for usr_id in user_ids:\n",
    "        df_test_user = training_df.loc[(training_df['user_id'] == usr_id)]\n",
    "        \n",
    "        max_score = -999999\n",
    "        current_expected = False\n",
    "                \n",
    "        for i,row in df_test_user.iterrows():\n",
    "            # prediction function\n",
    "            total_score = sum(avg_weight*row[feature_names].values)\n",
    "            #print 'row[feature_names].values:', row[feature_names].values, ' - total_score:', total_score\n",
    "            \n",
    "            if (total_score > max_score):\n",
    "                max_score = total_score\n",
    "                current_expected = row[class_name]\n",
    "\n",
    "        if max_score >= treshold:\n",
    "            final_users_covered = final_users_covered + 1\n",
    "            # lets record this user's prediction \n",
    "\n",
    "            final_predicted_list.append(max_score >= avg_intercept)\n",
    "            final_expected_list.append(current_expected)\n",
    "    \n",
    "    \n",
    "    final_return_dict = {}\n",
    "    \n",
    "    final_return_dict['k_fold'] = {'avg_coverage':np.mean(covarege_percentages),\n",
    "                                   'avg_accuracy':np.mean(accuracy_list),\n",
    "                                   'avg_f1':np.mean(f1_list),\n",
    "                                   'avg_precision':np.mean(precision_list),\n",
    "                                   'avg_recall':np.mean(recall_list),\n",
    "                                   'max_point_stat':{'min':np.min(max_points),\n",
    "                                                     'max':np.max(max_points),\n",
    "                                                     'avg':np.max(max_points)}}\n",
    "    \n",
    "    final_return_dict['avg_model'] = {'coverage':final_users_covered*100.0/len(user_ids),\n",
    "                                      'accuracy':metrics.accuracy_score(final_expected_list, final_predicted_list),\n",
    "                                      'f1':metrics.f1_score(final_expected_list, final_predicted_list),\n",
    "                                      'precision':metrics.precision_score(final_expected_list, final_predicted_list),\n",
    "                                      'recall':metrics.recall_score(final_expected_list, final_predicted_list),\n",
    "                                      'weights':avg_weight,\n",
    "                                      'intercept':avg_intercept}\n",
    "    \n",
    "    return final_return_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### - Prepare feature pairs to be evaluated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feature_dict = {}\n",
    "\n",
    "feature_dict['end_of_day_ratio']={\n",
    "    'name':'End of Day Ratio',\n",
    "    'code': 'EDR'\n",
    "}\n",
    "feature_dict['checkin_ratio']={\n",
    "    'name':'Checkin Ratio',\n",
    "    'code': 'CR'\n",
    "}\n",
    "feature_dict['end_of_inactive_day_ratio']={\n",
    "    'name':'End of Inactive Day Ratio',\n",
    "    'code': 'EIDR'\n",
    "}\n",
    "feature_dict['midnight_ratio']={\n",
    "    'name':'Midnight Ratio',\n",
    "    'code': 'MR'\n",
    "}\n",
    "feature_dict['page_rank']={\n",
    "    'name':'PageRank',\n",
    "    'code': 'PR'\n",
    "}\n",
    "feature_dict['reverse_page_rank']={\n",
    "    'name':'Reverse PageRank',\n",
    "    'code': 'RPR'\n",
    "}\n",
    "feature_dict['is_residential']={\n",
    "    'name':'Land Use',\n",
    "    'code': 'LU'\n",
    "}\n",
    "feature_dict['kilometer_distance_to_most_checked_in']={\n",
    "    'name':'Kilometer Distance to most checked in',\n",
    "    'code': 'KM'\n",
    "}\n",
    "\n",
    "feature_pairs = [['end_of_day_ratio'],\n",
    "                 ['checkin_ratio'],\n",
    "                 ['end_of_inactive_day_ratio'],\n",
    "                 ['midnight_ratio'],\n",
    "                 ['page_rank'],\n",
    "                 ['reverse_page_rank'],\n",
    "                 ['kilometer_distance_to_most_checked_in'],\n",
    "                 ['is_residential'],\n",
    "                 ['end_of_day_ratio','is_residential'],\n",
    "                 ['checkin_ratio','is_residential'],\n",
    "                 ['end_of_inactive_day_ratio','is_residential'],\n",
    "                 ['midnight_ratio','is_residential'],\n",
    "                 ['page_rank','is_residential'],\n",
    "                 ['reverse_page_rank','is_residential'],\n",
    "                 ['kilometer_distance_to_most_checked_in','is_residential'],\n",
    "                 ['end_of_day_ratio', 'checkin_ratio'],\n",
    "                 ['end_of_day_ratio', 'end_of_inactive_day_ratio'],\n",
    "                 ['end_of_day_ratio', 'midnight_ratio'],\n",
    "                 ['end_of_day_ratio', 'page_rank'],\n",
    "                 ['end_of_day_ratio', 'reverse_page_rank'],\n",
    "                 ['end_of_day_ratio', 'kilometer_distance_to_most_checked_in'],\n",
    "                 ['checkin_ratio', 'end_of_inactive_day_ratio'],\n",
    "                 ['checkin_ratio', 'midnight_ratio'],\n",
    "                 ['checkin_ratio', 'page_rank'],\n",
    "                 ['checkin_ratio', 'reverse_page_rank'],\n",
    "                 ['checkin_ratio', 'kilometer_distance_to_most_checked_in'],\n",
    "                 ['end_of_inactive_day_ratio', 'midnight_ratio'],\n",
    "                 ['end_of_inactive_day_ratio', 'page_rank'],\n",
    "                 ['end_of_inactive_day_ratio', 'reverse_page_rank'],\n",
    "                 ['end_of_inactive_day_ratio', 'kilometer_distance_to_most_checked_in'],\n",
    "                 ['midnight_ratio', 'page_rank'],\n",
    "                 ['midnight_ratio', 'reverse_page_rank'],\n",
    "                 ['end_of_day_ratio', 'checkin_ratio','end_of_inactive_day_ratio','midnight_ratio','page_rank','reverse_page_rank','kilometer_distance_to_most_checked_in']]\n",
    "\n",
    "\n",
    "training_set_folder = 'training_test_set/'\n",
    "feature_scores_folder = 'feature_scores/'\n",
    "class_name = 'is_home'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### - Evaluate features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "78812  rows loaded.\n",
      "['end_of_day_ratio']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['checkin_ratio']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['end_of_inactive_day_ratio']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['midnight_ratio']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['page_rank']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['reverse_page_rank']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['kilometer_distance_to_most_checked_in']\n",
      "repeat: 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hdl/anaconda/lib/python2.7/site-packages/numpy/lib/function_base.py:1110: RuntimeWarning: Mean of empty slice.\n",
      "  avg = a.mean(axis)\n",
      "/Users/hdl/anaconda/lib/python2.7/site-packages/numpy/core/_methods.py:80: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['is_residential']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['end_of_day_ratio', 'is_residential']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['checkin_ratio', 'is_residential']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['end_of_inactive_day_ratio', 'is_residential']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['midnight_ratio', 'is_residential']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['page_rank', 'is_residential']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['reverse_page_rank', 'is_residential']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['kilometer_distance_to_most_checked_in', 'is_residential']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['end_of_day_ratio', 'checkin_ratio']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['end_of_day_ratio', 'end_of_inactive_day_ratio']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['end_of_day_ratio', 'midnight_ratio']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['end_of_day_ratio', 'page_rank']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['end_of_day_ratio', 'reverse_page_rank']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['end_of_day_ratio', 'kilometer_distance_to_most_checked_in']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['checkin_ratio', 'end_of_inactive_day_ratio']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['checkin_ratio', 'midnight_ratio']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['checkin_ratio', 'page_rank']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['checkin_ratio', 'reverse_page_rank']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['checkin_ratio', 'kilometer_distance_to_most_checked_in']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['end_of_inactive_day_ratio', 'midnight_ratio']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['end_of_inactive_day_ratio', 'page_rank']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['end_of_inactive_day_ratio', 'reverse_page_rank']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['end_of_inactive_day_ratio', 'kilometer_distance_to_most_checked_in']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['midnight_ratio', 'page_rank']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['midnight_ratio', 'reverse_page_rank']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n",
      "['end_of_day_ratio', 'checkin_ratio', 'end_of_inactive_day_ratio', 'midnight_ratio', 'page_rank', 'reverse_page_rank', 'kilometer_distance_to_most_checked_in']\n",
      "repeat: 0\n",
      "repeat: 1\n",
      "repeat: 2\n",
      "repeat: 3\n",
      "repeat: 4\n"
     ]
    }
   ],
   "source": [
    "filenames = ['training_test_set......csv']\n",
    "\n",
    "for afile in filenames:\n",
    "    df = pd.read_csv(training_set_folder+afile)\n",
    "    \n",
    "    num_of_rows = len(df.index)\n",
    "    print num_of_rows, ' rows loaded.'\n",
    "    \n",
    "     \n",
    "    all_results = []\n",
    "    rows_list = []\n",
    "\n",
    "    for feature_names in feature_pairs:\n",
    "        print feature_names\n",
    "        result = evaluate(df, feature_names, class_name)\n",
    "        all_results.append({'features':feature_names, 'results':result})\n",
    "\n",
    "        kfold_result = result['k_fold']\n",
    "\n",
    "        dict1 = {}\n",
    "        \n",
    "        # first column\n",
    "        column = feature_dict[feature_names[0]]['name']\n",
    "        for i in range(1,len(feature_names)):\n",
    "            column = column + '+' + feature_dict[feature_names[i]]['name']\n",
    "        dict1['Feature'] = column   \n",
    "        \n",
    "        # second column\n",
    "        column = feature_dict[feature_names[0]]['code']\n",
    "        for i in range(1,len(feature_names)):\n",
    "            column = column + '+' + feature_dict[feature_names[i]]['code']\n",
    "        dict1['Short Form'] = column\n",
    "        \n",
    "        # third column\n",
    "        dict1['Accuracy'] = kfold_result['avg_accuracy']\n",
    "\n",
    "        # fourth column\n",
    "        dict1['Precision'] = kfold_result['avg_precision']\n",
    "\n",
    "        # fifth column\n",
    "        dict1['Recall'] = kfold_result['avg_recall']\n",
    "\n",
    "        # sixth column\n",
    "        dict1['F1'] = kfold_result['avg_f1']\n",
    "\n",
    "        # seventh column\n",
    "        dict1['Coverage'] = kfold_result['avg_coverage']\n",
    "\n",
    "        rows_list.append(dict1)\n",
    "    \n",
    "    df_features = pd.DataFrame(rows_list, columns=['Feature', 'Short Form', 'Accuracy',\n",
    "                                                   'Precision','Recall','F1','Coverage'])\n",
    "   \n",
    "    # print output\n",
    "    df_features.to_csv(feature_scores_folder + 'weighted_score_all_features_' + afile, index=False)\n",
    "    f = open(feature_scores_folder+\"feature_training_outputs_w.txt\", \"a+\")\n",
    "    f.write(\"\\n\\nBy number \" + afile+ \" \\n\")\n",
    "    f.write(''.join(str(all_results)))\n",
    "    f.close()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}