Validation2_Regression_POI.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import copy\n",
    "import statsmodels.api as sm\n",
    "import statsmodels.multivariate.multivariate_ols as mols\n",
    "import scipy.stats as ss\n",
    "from statsmodels.multivariate.pca import PCA\n",
    "\n",
    "\n",
    "def readdata(path):\n",
    "    F1=open(path,\"r\")\n",
    "    lines=F1.readlines()\n",
    "    F1.close()\n",
    "    data=[]\n",
    "    for i in lines:\n",
    "        user=i.split(',')\n",
    "        user2=[]\n",
    "        for j in range(len(user)):\n",
    "            try:\n",
    "                user2.append(float(user[j]))\n",
    "            except:\n",
    "                a=1\n",
    "        data.append(user2)\n",
    "    return data\n",
    "\n",
    "##input data\n",
    "path=\"E:/research/POIStatistics.csv\"\n",
    "data=readdata(path)\n",
    "data=np.array(data)\n",
    "\n",
    "classnum=[2586,3230,8600,8587,6260,3690,3865,3598,3279,4776,10123,12478,22461,6635]\n",
    "\n",
    "Y=data[:,-1]\n",
    "X=data[:,:-1]\n",
    "\n",
    "for i in range(len(X)):\n",
    "    for j in range(len(X[0])):\n",
    "        #X[i][j]=X[i][j]/classnum[j]\n",
    "        X[i][j]=(X[i][j]-np.mean(np.array(X[:,j])))/np.var(np.array(X[:,j]))\n",
    "for i in range(len(Y)):\n",
    "    Y[i]=Y[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SpearmanrResult(correlation=1.0, pvalue=0.0)\n",
      "__________________________\n",
      "SpearmanrResult(correlation=0.28653752247430764, pvalue=1.8075339627512917e-18)\n",
      "SpearmanrResult(correlation=1.0, pvalue=0.0)\n",
      "__________________________\n",
      "SpearmanrResult(correlation=0.63726723942457542, pvalue=1.0334966824592594e-103)\n",
      "SpearmanrResult(correlation=0.26873312189274312, pvalue=2.361577540015911e-16)\n",
      "SpearmanrResult(correlation=1.0, pvalue=0.0)\n",
      "__________________________\n",
      "SpearmanrResult(correlation=0.26352223480111292, pvalue=9.1942637420508981e-16)\n",
      "SpearmanrResult(correlation=0.39769905888772705, pvalue=1.7787469687310239e-35)\n",
      "SpearmanrResult(correlation=0.3601816956975667, pvalue=5.8876331088218658e-29)\n",
      "SpearmanrResult(correlation=1.0, pvalue=0.0)\n",
      "__________________________\n",
      "SpearmanrResult(correlation=0.58759723571675204, pvalue=1.1667688172027294e-84)\n",
      "SpearmanrResult(correlation=0.30344146103884079, pvalue=1.2663114629409681e-20)\n",
      "SpearmanrResult(correlation=0.60586436937987986, pvalue=2.8195178478300512e-91)\n",
      "SpearmanrResult(correlation=0.38362597155469741, pvalue=6.2520656400378505e-33)\n",
      "SpearmanrResult(correlation=1.0, pvalue=0.0)\n",
      "__________________________\n",
      "SpearmanrResult(correlation=0.67266473168485397, pvalue=1.3777288268037701e-119)\n",
      "SpearmanrResult(correlation=0.38905730336292604, pvalue=6.7308517834495951e-34)\n",
      "SpearmanrResult(correlation=0.69798407158527365, pvalue=2.2616358418463886e-132)\n",
      "SpearmanrResult(correlation=0.37878876393674565, pvalue=4.3941966024746316e-32)\n",
      "SpearmanrResult(correlation=0.60871717948622983, pvalue=2.3861351898304061e-92)\n",
      "SpearmanrResult(correlation=1.0, pvalue=0.0)\n",
      "__________________________\n",
      "SpearmanrResult(correlation=0.54722681550635666, pvalue=1.9322529990913755e-71)\n",
      "SpearmanrResult(correlation=0.21630594605672354, pvalue=5.4552591472630501e-11)\n",
      "SpearmanrResult(correlation=0.63331330861725343, pvalue=4.5663797241431418e-102)\n",
      "SpearmanrResult(correlation=0.30010415650718919, pvalue=3.4622148391784726e-20)\n",
      "SpearmanrResult(correlation=0.61124541306018076, pvalue=2.6186172775426241e-93)\n",
      "SpearmanrResult(correlation=0.59161050816112126, pvalue=4.4612098228208981e-86)\n",
      "SpearmanrResult(correlation=1.0, pvalue=0.0)\n",
      "__________________________\n",
      "SpearmanrResult(correlation=0.52378281948496241, pvalue=1.4848887152165365e-64)\n",
      "SpearmanrResult(correlation=0.23974357169988278, pvalue=3.1256813994294701e-13)\n",
      "SpearmanrResult(correlation=0.55547688330479417, pvalue=5.3795099858507689e-74)\n",
      "SpearmanrResult(correlation=0.35853634387202948, pvalue=1.0881403450146892e-28)\n",
      "SpearmanrResult(correlation=0.51990549782571749, pvalue=1.8168268998542165e-63)\n",
      "SpearmanrResult(correlation=0.53941243548860351, pvalue=4.3819174188973423e-69)\n",
      "SpearmanrResult(correlation=0.52507277992524282, pvalue=6.4079525950470169e-65)\n",
      "SpearmanrResult(correlation=1.0, pvalue=0.0)\n",
      "__________________________\n",
      "SpearmanrResult(correlation=0.65982432077076647, pvalue=1.4053571383642982e-113)\n",
      "SpearmanrResult(correlation=0.40316295040693467, pvalue=1.6905672292341808e-36)\n",
      "SpearmanrResult(correlation=0.61259004023461772, pvalue=8.0196661824662589e-94)\n",
      "SpearmanrResult(correlation=0.42805735151113355, pvalue=2.106759630078258e-41)\n",
      "SpearmanrResult(correlation=0.66918504014614022, pvalue=6.2702247350266609e-118)\n",
      "SpearmanrResult(correlation=0.65161306783506334, pvalue=6.8729729686061694e-110)\n",
      "SpearmanrResult(correlation=0.55914388371672885, pvalue=3.7307782047395949e-75)\n",
      "SpearmanrResult(correlation=0.56824430647445245, pvalue=4.2883951791864813e-78)\n",
      "SpearmanrResult(correlation=1.0, pvalue=0.0)\n",
      "__________________________\n",
      "SpearmanrResult(correlation=0.66574143095649907, pvalue=2.6070118554171581e-116)\n",
      "SpearmanrResult(correlation=0.32622334924693325, pvalue=9.2523845270016202e-24)\n",
      "SpearmanrResult(correlation=0.68898085059364278, pvalue=1.1255488065138774e-127)\n",
      "SpearmanrResult(correlation=0.39653648955122173, pvalue=2.9183116847224361e-35)\n",
      "SpearmanrResult(correlation=0.57252577678079442, pvalue=1.6495345420008295e-79)\n",
      "SpearmanrResult(correlation=0.69278482236809347, pvalue=1.224638139293472e-129)\n",
      "SpearmanrResult(correlation=0.56076060999252264, pvalue=1.1382278985141937e-75)\n",
      "SpearmanrResult(correlation=0.57668020166280043, pvalue=6.6735707655078465e-81)\n",
      "SpearmanrResult(correlation=0.63948704463421147, pvalue=1.2023061950171561e-104)\n",
      "SpearmanrResult(correlation=1.0, pvalue=0.0)\n",
      "__________________________\n",
      "SpearmanrResult(correlation=0.45107925647644836, pvalue=2.5598111569902583e-46)\n",
      "SpearmanrResult(correlation=0.44941653837432727, pvalue=5.9703237992228789e-46)\n",
      "SpearmanrResult(correlation=0.52957494186206822, pvalue=3.3151738360595142e-66)\n",
      "SpearmanrResult(correlation=0.6781759319044427, pvalue=2.9242981245898548e-122)\n",
      "SpearmanrResult(correlation=0.57529809296060852, pvalue=1.9498549966431959e-80)\n",
      "SpearmanrResult(correlation=0.55617627511474299, pvalue=3.2420160627891416e-74)\n",
      "SpearmanrResult(correlation=0.3998536129879996, pvalue=7.0690124874953285e-36)\n",
      "SpearmanrResult(correlation=0.53639239883835255, pvalue=3.4322043445792304e-68)\n",
      "SpearmanrResult(correlation=0.62480501827780044, pvalue=1.3178033977624026e-98)\n",
      "SpearmanrResult(correlation=0.54232284237387962, pvalue=5.909840530947158e-70)\n",
      "SpearmanrResult(correlation=1.0, pvalue=0.0)\n",
      "__________________________\n",
      "SpearmanrResult(correlation=0.51132575060792262, pvalue=4.136611178168878e-61)\n",
      "SpearmanrResult(correlation=0.22410900095761438, pvalue=1.041772506415269e-11)\n",
      "SpearmanrResult(correlation=0.61367405803381658, pvalue=3.0763701920348229e-94)\n",
      "SpearmanrResult(correlation=0.48579429521929457, pvalue=1.7664211902957787e-54)\n",
      "SpearmanrResult(correlation=0.65653974058404196, pvalue=4.3397600313735034e-112)\n",
      "SpearmanrResult(correlation=0.56395177442605893, pvalue=1.0720099632857042e-76)\n",
      "SpearmanrResult(correlation=0.66052086484057393, pvalue=6.7523757044735415e-114)\n",
      "SpearmanrResult(correlation=0.50568444323182304, pvalue=1.3501393882484602e-59)\n",
      "SpearmanrResult(correlation=0.59186728214067752, pvalue=3.6147526344798255e-86)\n",
      "SpearmanrResult(correlation=0.52311697504153298, pvalue=2.2880757753071109e-64)\n",
      "SpearmanrResult(correlation=0.53743236720045351, pvalue=1.6934525523723826e-68)\n",
      "SpearmanrResult(correlation=1.0, pvalue=0.0)\n",
      "__________________________\n",
      "SpearmanrResult(correlation=0.56093471720335464, pvalue=1.0012380497034765e-75)\n",
      "SpearmanrResult(correlation=0.47101338808237214, pvalue=6.870648786322008e-51)\n",
      "SpearmanrResult(correlation=0.64013478205940177, pvalue=6.3963620161081674e-105)\n",
      "SpearmanrResult(correlation=0.70836824901429918, pvalue=5.1803759865648171e-138)\n",
      "SpearmanrResult(correlation=0.59701571648030849, pvalue=5.1111131183220688e-88)\n",
      "SpearmanrResult(correlation=0.65715481953269894, pvalue=2.2904955930797068e-112)\n",
      "SpearmanrResult(correlation=0.48367840783342536, pvalue=5.9132582484770999e-54)\n",
      "SpearmanrResult(correlation=0.5465377611577299, pvalue=3.1352982693150755e-71)\n",
      "SpearmanrResult(correlation=0.69361097976664177, pvalue=4.5451218884297037e-130)\n",
      "SpearmanrResult(correlation=0.64690784391914891, pvalue=7.9419798182708859e-108)\n",
      "SpearmanrResult(correlation=0.81464497281272374, pvalue=1.1101714185790542e-214)\n",
      "SpearmanrResult(correlation=0.60013986848543444, pvalue=3.7130615380378113e-89)\n",
      "SpearmanrResult(correlation=1.0, pvalue=0.0)\n",
      "__________________________\n",
      "SpearmanrResult(correlation=0.49539896551312618, pvalue=6.5825630171961992e-57)\n",
      "SpearmanrResult(correlation=0.30641432067611607, pvalue=5.1123995637817118e-21)\n",
      "SpearmanrResult(correlation=0.62231306458403035, pvalue=1.2975613029978506e-97)\n",
      "SpearmanrResult(correlation=0.44583039402929309, pvalue=3.6504898755604094e-45)\n",
      "SpearmanrResult(correlation=0.56945355899616334, pvalue=1.7169876933650972e-78)\n",
      "SpearmanrResult(correlation=0.57851870599017197, pvalue=1.5903708248198075e-81)\n",
      "SpearmanrResult(correlation=0.64690330070366342, pvalue=7.9781628692819758e-108)\n",
      "SpearmanrResult(correlation=0.56829648760883256, pvalue=4.1226347056725704e-78)\n",
      "SpearmanrResult(correlation=0.58419910806474251, pvalue=1.78578823694983e-83)\n",
      "SpearmanrResult(correlation=0.56296983082695173, pvalue=2.223852874776002e-76)\n",
      "SpearmanrResult(correlation=0.57391280729976202, pvalue=5.6819354685031853e-80)\n",
      "SpearmanrResult(correlation=0.71944541495318726, pvalue=2.611696031827978e-144)\n",
      "SpearmanrResult(correlation=0.59093235094528929, pvalue=7.7693241984543493e-86)\n",
      "SpearmanrResult(correlation=1.0, pvalue=0.0)\n",
      "__________________________\n"
     ]
    }
   ],
   "source": [
    "##Spearmanr\n",
    "for i in range(14):\n",
    "    for j in range(14):\n",
    "        if(i>=j):\n",
    "            #print(ss.pearsonr(X[:,i],X[:,j])[0])\n",
    "            print(ss.spearmanr(X[:,i],X[:,j]))\n",
    "    print(\"__________________________\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(900, 14)\n",
      "(9, 900)\n",
      "                            OLS Regression Results                            \n",
      "==============================================================================\n",
      "Dep. Variable:                      y   R-squared:                       0.213\n",
      "Model:                            OLS   Adj. R-squared:                  0.205\n",
      "Method:                 Least Squares   F-statistic:                     26.69\n",
      "Date:                Tue, 10 Mar 2020   Prob (F-statistic):           4.96e-41\n",
      "Time:                        11:57:33   Log-Likelihood:                -9058.5\n",
      "No. Observations:                 900   AIC:                         1.814e+04\n",
      "Df Residuals:                     890   BIC:                         1.819e+04\n",
      "Df Model:                           9                                         \n",
      "Covariance Type:            nonrobust                                         \n",
      "==============================================================================\n",
      "                 coef    std err          t      P>|t|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "const       1185.9587    220.235      5.385      0.000     753.719    1618.199\n",
      "x1           -86.0568    232.381     -0.370      0.711    -542.135     370.021\n",
      "x2           281.4493    264.400      1.064      0.287    -237.472     800.370\n",
      "x3          -759.4258    303.461     -2.503      0.013   -1355.009    -163.842\n",
      "x4           838.8727    230.716      3.636      0.000     386.062    1291.684\n",
      "x5             3.2491    288.454      0.011      0.991    -562.881     569.379\n",
      "x6           684.9608    376.732      1.818      0.069     -54.426    1424.348\n",
      "x7          -876.1245    355.257     -2.466      0.014   -1573.365    -178.884\n",
      "x8          -651.7276   1860.459     -0.350      0.726   -4303.126    2999.671\n",
      "x9         -2585.3888    237.296    -10.895      0.000   -3051.113   -2119.665\n",
      "==============================================================================\n",
      "Omnibus:                      546.329   Durbin-Watson:                   1.114\n",
      "Prob(Omnibus):                  0.000   Jarque-Bera (JB):            23048.725\n",
      "Skew:                          -2.116   Prob(JB):                         0.00\n",
      "Kurtosis:                      27.428   Cond. No.                         24.4\n",
      "==============================================================================\n",
      "\n",
      "Warnings:\n",
      "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n"
     ]
    }
   ],
   "source": [
    "#regression analysis\n",
    "selected=[3,4,5,7,8,9,10,12,13]\n",
    "newX=[]\n",
    "print(np.array(X).shape)   \n",
    "for i in selected:\n",
    "    newX.append(copy.deepcopy(X[:,i]))\n",
    "print(np.array(newX).shape)   \n",
    "\n",
    "Final_X=sm.add_constant(np.array(newX).T)\n",
    "est=sm.OLS(np.array(Y),Final_X).fit()\n",
    "print(est.summary())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}