ALR_VAME.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import matplotlib\n",
    "import vame\n",
    "import numpy as np\n",
    "import glob\n",
    "from vame.util import auxiliary as aux\n",
    "from vame.analysis import behavior_structure as bs\n",
    "from vame.custom import helperFunctions as hf\n",
    "from vame.custom import ACWS_videowriter as avw\n",
    "import polars as pl\n",
    "import pandas as pd\n",
    "from vame.util import auxiliary as aux\n",
    "from vame.analysis import behavior_structure as bs\n",
    "from vame.custom import helperFunctions as hf\n",
    "from vame.custom import ALR_helperFunctions as AlHf\n",
    "from vame.custom import ALR_plottingFunctions as AlPf\n",
    "from vame.custom import ACWS_videowriter as avw\n",
    "from vame.custom import ALR_kinematics as kin\n",
    "from vame.custom import ALR_analysis as an\n",
    "from vame.analysis import community_analysis as ca\n",
    "from importlib import reload\n",
    "from icecream import ic\n",
    "from scipy.stats import shapiro\n",
    "import matplotlib.pyplot as plt\n",
    "import six\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysis\n",
    "## Checking the Normality of the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "reload(AlHf)\n",
    "#df = pd.read_csv(\"/Users/adanredwine/Downloads/CombinedMotifUsage-drawn-sweep-88-40-200_100prcnt.csv\")\n",
    "df = pd.read_csv(\"/Users/adanredwine/Desktop/ALR_1-VAME-Project-Feb3-2023/CombinedMotifUsage-kmeans-30.csv\")\n",
    "meta_data = AlHf.create_meta_data_df(df)\n",
    "normality_results_Group = AlHf.check_normality_by_group(df, meta_data)\n",
    "normality_results_GroupTime = AlHf.check_normality_by_group_and_time(df, meta_data)\n",
    "normality_results_GroupTimeMotif = AlHf.check_normality_group_timepoint_motif(df, meta_data)\n",
    "normality_results_StateTimeMotif = AlHf.check_normality_state_timepoint_motif(df, meta_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Group</th>\n",
       "      <th>W</th>\n",
       "      <th>p</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Sham</td>\n",
       "      <td>0.740767</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ABX</td>\n",
       "      <td>0.735181</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Injured</td>\n",
       "      <td>0.825460</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Treated</td>\n",
       "      <td>0.787291</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Group         W    p\n",
       "0     Sham  0.740767  0.0\n",
       "1      ABX  0.735181  0.0\n",
       "2  Injured  0.825460  0.0\n",
       "3  Treated  0.787291  0.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normality_results_Group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Group</th>\n",
       "      <th>Time_Point</th>\n",
       "      <th>W</th>\n",
       "      <th>p</th>\n",
       "      <th>Note</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Sham</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>0.864440</td>\n",
       "      <td>3.493903e-17</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Sham</td>\n",
       "      <td>Baseline_2</td>\n",
       "      <td>0.786921</td>\n",
       "      <td>2.297405e-21</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sham</td>\n",
       "      <td>Week_02</td>\n",
       "      <td>0.709611</td>\n",
       "      <td>1.639493e-24</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Sham</td>\n",
       "      <td>Week_04</td>\n",
       "      <td>0.694650</td>\n",
       "      <td>4.811304e-25</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Sham</td>\n",
       "      <td>Week_06</td>\n",
       "      <td>0.700416</td>\n",
       "      <td>7.674122e-25</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Sham</td>\n",
       "      <td>Week_08</td>\n",
       "      <td>0.761729</td>\n",
       "      <td>1.786731e-22</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Sham</td>\n",
       "      <td>Week_11</td>\n",
       "      <td>0.755931</td>\n",
       "      <td>1.022052e-22</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Sham</td>\n",
       "      <td>Week_13</td>\n",
       "      <td>0.755759</td>\n",
       "      <td>1.005356e-22</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Sham</td>\n",
       "      <td>Week_15</td>\n",
       "      <td>0.717889</td>\n",
       "      <td>3.299606e-24</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Sham</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>0.696749</td>\n",
       "      <td>5.698159e-25</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ABX</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>0.823333</td>\n",
       "      <td>1.425235e-19</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ABX</td>\n",
       "      <td>Baseline_2</td>\n",
       "      <td>0.746546</td>\n",
       "      <td>4.226199e-23</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>ABX</td>\n",
       "      <td>Week_02</td>\n",
       "      <td>0.740472</td>\n",
       "      <td>2.418088e-23</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>ABX</td>\n",
       "      <td>Week_04</td>\n",
       "      <td>0.780274</td>\n",
       "      <td>1.146351e-21</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>ABX</td>\n",
       "      <td>Week_06</td>\n",
       "      <td>0.790297</td>\n",
       "      <td>3.290914e-21</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ABX</td>\n",
       "      <td>Week_08</td>\n",
       "      <td>0.732914</td>\n",
       "      <td>1.223982e-23</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>ABX</td>\n",
       "      <td>Week_11</td>\n",
       "      <td>0.729975</td>\n",
       "      <td>9.429990e-24</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ABX</td>\n",
       "      <td>Week_13</td>\n",
       "      <td>0.585704</td>\n",
       "      <td>2.077685e-28</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ABX</td>\n",
       "      <td>Week_15</td>\n",
       "      <td>0.743195</td>\n",
       "      <td>3.102068e-23</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>ABX</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No matching data for group/time point</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Injured</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>0.874406</td>\n",
       "      <td>1.588204e-16</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Injured</td>\n",
       "      <td>Baseline_2</td>\n",
       "      <td>0.811352</td>\n",
       "      <td>3.434206e-20</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Injured</td>\n",
       "      <td>Week_02</td>\n",
       "      <td>0.852146</td>\n",
       "      <td>6.007435e-18</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Injured</td>\n",
       "      <td>Week_04</td>\n",
       "      <td>0.896996</td>\n",
       "      <td>6.968870e-15</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Injured</td>\n",
       "      <td>Week_06</td>\n",
       "      <td>0.796103</td>\n",
       "      <td>6.169110e-21</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Injured</td>\n",
       "      <td>Week_08</td>\n",
       "      <td>0.836457</td>\n",
       "      <td>7.373482e-19</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Injured</td>\n",
       "      <td>Week_11</td>\n",
       "      <td>0.721916</td>\n",
       "      <td>4.663590e-24</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Injured</td>\n",
       "      <td>Week_13</td>\n",
       "      <td>0.829568</td>\n",
       "      <td>3.075589e-19</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Injured</td>\n",
       "      <td>Week_15</td>\n",
       "      <td>0.854211</td>\n",
       "      <td>8.013647e-18</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Injured</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>0.793497</td>\n",
       "      <td>4.645586e-21</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Treated</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>0.881835</td>\n",
       "      <td>5.199098e-16</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Treated</td>\n",
       "      <td>Baseline_2</td>\n",
       "      <td>0.792734</td>\n",
       "      <td>4.277226e-21</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Treated</td>\n",
       "      <td>Week_02</td>\n",
       "      <td>0.773318</td>\n",
       "      <td>5.632589e-22</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Treated</td>\n",
       "      <td>Week_04</td>\n",
       "      <td>0.839349</td>\n",
       "      <td>1.073165e-18</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Treated</td>\n",
       "      <td>Week_06</td>\n",
       "      <td>0.788012</td>\n",
       "      <td>2.579223e-21</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Treated</td>\n",
       "      <td>Week_08</td>\n",
       "      <td>0.790965</td>\n",
       "      <td>3.535474e-21</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Treated</td>\n",
       "      <td>Week_11</td>\n",
       "      <td>0.815734</td>\n",
       "      <td>5.733374e-20</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Treated</td>\n",
       "      <td>Week_13</td>\n",
       "      <td>0.732985</td>\n",
       "      <td>1.231679e-23</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Treated</td>\n",
       "      <td>Week_15</td>\n",
       "      <td>0.730806</td>\n",
       "      <td>1.014942e-23</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Treated</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>0.761031</td>\n",
       "      <td>1.669526e-22</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Group  Time_Point         W             p  \\\n",
       "0      Sham  Baseline_1  0.864440  3.493903e-17   \n",
       "1      Sham  Baseline_2  0.786921  2.297405e-21   \n",
       "2      Sham     Week_02  0.709611  1.639493e-24   \n",
       "3      Sham     Week_04  0.694650  4.811304e-25   \n",
       "4      Sham     Week_06  0.700416  7.674122e-25   \n",
       "5      Sham     Week_08  0.761729  1.786731e-22   \n",
       "6      Sham     Week_11  0.755931  1.022052e-22   \n",
       "7      Sham     Week_13  0.755759  1.005356e-22   \n",
       "8      Sham     Week_15  0.717889  3.299606e-24   \n",
       "9      Sham    Drug_Trt  0.696749  5.698159e-25   \n",
       "10      ABX  Baseline_1  0.823333  1.425235e-19   \n",
       "11      ABX  Baseline_2  0.746546  4.226199e-23   \n",
       "12      ABX     Week_02  0.740472  2.418088e-23   \n",
       "13      ABX     Week_04  0.780274  1.146351e-21   \n",
       "14      ABX     Week_06  0.790297  3.290914e-21   \n",
       "15      ABX     Week_08  0.732914  1.223982e-23   \n",
       "16      ABX     Week_11  0.729975  9.429990e-24   \n",
       "17      ABX     Week_13  0.585704  2.077685e-28   \n",
       "18      ABX     Week_15  0.743195  3.102068e-23   \n",
       "19      ABX    Drug_Trt       NaN           NaN   \n",
       "20  Injured  Baseline_1  0.874406  1.588204e-16   \n",
       "21  Injured  Baseline_2  0.811352  3.434206e-20   \n",
       "22  Injured     Week_02  0.852146  6.007435e-18   \n",
       "23  Injured     Week_04  0.896996  6.968870e-15   \n",
       "24  Injured     Week_06  0.796103  6.169110e-21   \n",
       "25  Injured     Week_08  0.836457  7.373482e-19   \n",
       "26  Injured     Week_11  0.721916  4.663590e-24   \n",
       "27  Injured     Week_13  0.829568  3.075589e-19   \n",
       "28  Injured     Week_15  0.854211  8.013647e-18   \n",
       "29  Injured    Drug_Trt  0.793497  4.645586e-21   \n",
       "30  Treated  Baseline_1  0.881835  5.199098e-16   \n",
       "31  Treated  Baseline_2  0.792734  4.277226e-21   \n",
       "32  Treated     Week_02  0.773318  5.632589e-22   \n",
       "33  Treated     Week_04  0.839349  1.073165e-18   \n",
       "34  Treated     Week_06  0.788012  2.579223e-21   \n",
       "35  Treated     Week_08  0.790965  3.535474e-21   \n",
       "36  Treated     Week_11  0.815734  5.733374e-20   \n",
       "37  Treated     Week_13  0.732985  1.231679e-23   \n",
       "38  Treated     Week_15  0.730806  1.014942e-23   \n",
       "39  Treated    Drug_Trt  0.761031  1.669526e-22   \n",
       "\n",
       "                                     Note  \n",
       "0                                     NaN  \n",
       "1                                     NaN  \n",
       "2                                     NaN  \n",
       "3                                     NaN  \n",
       "4                                     NaN  \n",
       "5                                     NaN  \n",
       "6                                     NaN  \n",
       "7                                     NaN  \n",
       "8                                     NaN  \n",
       "9                                     NaN  \n",
       "10                                    NaN  \n",
       "11                                    NaN  \n",
       "12                                    NaN  \n",
       "13                                    NaN  \n",
       "14                                    NaN  \n",
       "15                                    NaN  \n",
       "16                                    NaN  \n",
       "17                                    NaN  \n",
       "18                                    NaN  \n",
       "19  No matching data for group/time point  \n",
       "20                                    NaN  \n",
       "21                                    NaN  \n",
       "22                                    NaN  \n",
       "23                                    NaN  \n",
       "24                                    NaN  \n",
       "25                                    NaN  \n",
       "26                                    NaN  \n",
       "27                                    NaN  \n",
       "28                                    NaN  \n",
       "29                                    NaN  \n",
       "30                                    NaN  \n",
       "31                                    NaN  \n",
       "32                                    NaN  \n",
       "33                                    NaN  \n",
       "34                                    NaN  \n",
       "35                                    NaN  \n",
       "36                                    NaN  \n",
       "37                                    NaN  \n",
       "38                                    NaN  \n",
       "39                                    NaN  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normality_results_GroupTime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_motifs_Group = len(normality_results_GroupTimeMotif)\n",
    "normal_motifs_Group = normality_results_GroupTimeMotif[normality_results_GroupTimeMotif['p'] > 0.05]\n",
    "non_normal_motifs_Group = normality_results_GroupTimeMotif[normality_results_GroupTimeMotif['p'] <= 0.05]\n",
    "percent_normal_Group = (len(normal_motifs_Group) / total_motifs_Group) * 100\n",
    "\n",
    "total_motifs_State = len(normality_results_StateTimeMotif)\n",
    "normal_motifs_State = normality_results_StateTimeMotif[normality_results_StateTimeMotif['p'] > 0.05]\n",
    "non_normal_motifs_State = normality_results_StateTimeMotif[normality_results_StateTimeMotif['p'] <= 0.05]\n",
    "percent_normal_State = (len(normal_motifs_State) / total_motifs_State) * 100\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "55.75"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "percent_normal_Group "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44.583333333333336"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "percent_normal_State"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In both observation angles, only about half of the data is normal. A log transformation will be done to the values. Additionally, log values will be normalized to baseline and sham values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "AlPf.plot_distribution_of_motif_3(df, meta_data, motif=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#clusters = np.load(\"/Users/adanredwine/Desktop/ALR_1-VAME-Project-Feb3-2023/clusters_all_data-11.npy\")\n",
    "clusters = np.load(\"/Users/adanredwine/Desktop/ALR_1-VAME-Project-Feb3-2023/clusters_wSpeed_all_data-11_kmeans-30.npy\")\n",
    "clustered_motifs =  AlHf.group_motifs_by_cluster(clusters)\n",
    "df_long = AlHf.melt_df(df, meta_data)\n",
    "df_long = AlHf.assign_clusters(df_long, clustered_motifs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "reload(AlHf)\n",
    "df_long_norm_base = AlHf.normalize_to_baseline(df_long)\n",
    "df_long_logNorm_base = AlHf.normalize_to_baseline_log(df_long)\n",
    "df_long_norm_sham = AlHf.normalize_to_baseline_sham(df_long)\n",
    "df_long_logNorm_sham = AlHf.normalize_to_baseline_sham_log(df_long)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Log transformation is a powerful statistical tool used to stabilize the variance, make the data more closely conform to normality, and reduce the effects of outliers. Here's how it affects the data:\n",
    "\n",
    "1. Normalization: Log transformation can help in normalizing data that is positively skewed. Many biological, economic, and other data sets naturally exhibit a right-skewed distribution, and log transformation can make such data conform more closely to a normal distribution.\n",
    "\n",
    "2. Variance Stabilization: In some datasets, the variance increases with the mean. Log transformation can stabilize the variance across levels of an independent variable, which is an assumption of many parametric statistical tests.\n",
    "\n",
    "3. Multiplicative to Additive: Log transformation turns multiplicative relationships into additive relationships, which can simplify the modeling of relationships between variables.\n",
    "\n",
    "4. Handling Outliers: It can diminish the influence of outliers due to the compression effect on the scale. Outliers will have less impact on the parameter estimates in a statistical model.\n",
    "\n",
    "5. Interpretation Changes: After log transformation, the interpretation of the results changes. For example, in a log-transformed linear regression, the coefficients represent the percentage change in the dependent variable for a one percent change in the independent variable, rather than the absolute change.\n",
    "\n",
    "6. Zero Values: Log transformation cannot be directly applied to zero or negative values since the logarithm of zero and negative numbers is undefined. In such cases, a constant may be added to all values before transformation.\n",
    "\n",
    "7. Back-Transformation: After analysis, results may need to be back-transformed to the original scale, which can be non-trivial, especially for measures of variability or for the interpretation of interaction effects.\n",
    "\n",
    "It's important to note that log transformation is not always the appropriate or necessary transformation, and its suitability depends on the specific characteristics and requirements of the data and analysis."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Whether to use normalized values or raw values for statistical tests depends on the context of your study and what you aim to compare.\n",
    "\n",
    "Here are some considerations to help you decide:\n",
    "\n",
    "1. Raw Values: Using raw values can be appropriate when you want to assess the actual measured quantities and their differences are meaningful. Raw values are also useful when you want to maintain the original scale and variance of the data, which can be important for certain statistical tests.\n",
    "\n",
    "2. Normalized Values: Normalization is often used when you want to control for baseline differences between subjects or groups, or when you want to compare relative changes rather than absolute values. For example, if you're interested in the percentage change from baseline or if the baseline values vary significantly between subjects, normalization can help to account for these differences.\n",
    "\n",
    "For repeated measures ANOVA and mixed-effects models, which are designed to handle repeated measurements from the same subjects, using normalized values can be beneficial if you're interested in relative changes over time and if there's a need to control for individual baseline differences.\n",
    "\n",
    "For non-parametric tests like the Friedman test or Kruskal-Wallis test, which are less sensitive to the scale of the data, normalized values can be used if the goal is to compare relative changes or rankings.\n",
    "\n",
    "If focus is on the change relative to a baseline or control condition, and you want to account for individual variability at the starting point, then normalized values may be more appropriate. However, if the actual measured values and their scale are important for the research question, then raw values should be used.\n",
    "\n",
    "Always consider the assumptions and requirements of the statistical test you're using, and ensure that the method of normalization is appropriate for the test. If in doubt, it's a good idea to consult with a statistician or methodologist."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reload(AlHf)\n",
    "result = AlHf.run_mixed_effects_model(df_long, dependent_var='Normalized_Value', groups='Animal_ID', re_formula='1', fe_formula='Time_Point * Group', maxiter=100)\n",
    "print(result.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/adanredwine/miniconda3/envs/VAME-2/lib/python3.11/site-packages/statsmodels/regression/mixed_linear_model.py:1635: UserWarning: Random effects covariance is singular\n",
      "  warnings.warn(msg)\n",
      "/Users/adanredwine/miniconda3/envs/VAME-2/lib/python3.11/site-packages/statsmodels/regression/mixed_linear_model.py:2238: ConvergenceWarning: The MLE may be on the boundary of the parameter space.\n",
      "  warnings.warn(msg, ConvergenceWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converged with lbfgs\n",
      "                       Mixed Linear Model Regression Results\n",
      "====================================================================================\n",
      "Model:                  MixedLM       Dependent Variable:       Log_Normalized_Value\n",
      "No. Observations:       14040         Method:                   REML                \n",
      "No. Groups:             48            Scale:                    1.1807              \n",
      "Min. group size:        270           Log-Likelihood:           -21127.5331         \n",
      "Max. group size:        300           Converged:                Yes                 \n",
      "Mean group size:        292.5                                                       \n",
      "------------------------------------------------------------------------------------\n",
      "                                          Coef.  Std.Err.   z    P>|z| [0.025 0.975]\n",
      "------------------------------------------------------------------------------------\n",
      "Intercept                                  0.029    0.057  0.505 0.614 -0.083  0.141\n",
      "Time_Point[T.Baseline_2]                  -0.000    0.081 -0.000 1.000 -0.159  0.159\n",
      "Time_Point[T.Drug_Trt]                    -0.055                                    \n",
      "Time_Point[T.Week_02]                     -0.091    0.081 -1.126 0.260 -0.250  0.068\n",
      "Time_Point[T.Week_04]                     -0.050    0.081 -0.616 0.538 -0.209  0.109\n",
      "Time_Point[T.Week_06]                     -0.106    0.081 -1.312 0.189 -0.265  0.052\n",
      "Time_Point[T.Week_08]                     -0.082    0.081 -1.016 0.310 -0.241  0.076\n",
      "Time_Point[T.Week_11]                     -0.091    0.081 -1.122 0.262 -0.250  0.068\n",
      "Time_Point[T.Week_13]                     -0.066    0.081 -0.812 0.417 -0.224  0.093\n",
      "Time_Point[T.Week_15]                     -0.061    0.081 -0.751 0.452 -0.220  0.098\n",
      "Group[T.Injured]                          -0.031    0.081 -0.381 0.703 -0.190  0.128\n",
      "Group[T.Sham]                             -0.036    0.081 -0.450 0.653 -0.195  0.122\n",
      "Group[T.Treated]                          -0.046    0.081 -0.571 0.568 -0.205  0.113\n",
      "Time_Point[T.Baseline_2]:Group[T.Injured]  0.000    0.115  0.000 1.000 -0.224  0.224\n",
      "Time_Point[T.Drug_Trt]:Group[T.Injured]   -0.025                                    \n",
      "Time_Point[T.Week_02]:Group[T.Injured]     0.026    0.115  0.227 0.820 -0.198  0.250\n",
      "Time_Point[T.Week_04]:Group[T.Injured]    -0.018    0.115 -0.154 0.878 -0.242  0.207\n",
      "Time_Point[T.Week_06]:Group[T.Injured]    -0.036    0.115 -0.314 0.754 -0.260  0.189\n",
      "Time_Point[T.Week_08]:Group[T.Injured]     0.081    0.115  0.705 0.481 -0.144  0.305\n",
      "Time_Point[T.Week_11]:Group[T.Injured]     0.090    0.115  0.783 0.433 -0.135  0.314\n",
      "Time_Point[T.Week_13]:Group[T.Injured]    -0.009    0.115 -0.080 0.936 -0.234  0.215\n",
      "Time_Point[T.Week_15]:Group[T.Injured]    -0.009    0.115 -0.078 0.937 -0.233  0.216\n",
      "Time_Point[T.Baseline_2]:Group[T.Sham]     0.000    0.115  0.000 1.000 -0.224  0.224\n",
      "Time_Point[T.Drug_Trt]:Group[T.Sham]      -0.027                                    \n",
      "Time_Point[T.Week_02]:Group[T.Sham]       -0.061    0.115 -0.537 0.591 -0.286  0.163\n",
      "Time_Point[T.Week_04]:Group[T.Sham]       -0.143    0.115 -1.246 0.213 -0.367  0.082\n",
      "Time_Point[T.Week_06]:Group[T.Sham]        0.023    0.115  0.205 0.838 -0.201  0.248\n",
      "Time_Point[T.Week_08]:Group[T.Sham]       -0.004    0.115 -0.031 0.975 -0.228  0.221\n",
      "Time_Point[T.Week_11]:Group[T.Sham]        0.068    0.115  0.590 0.555 -0.157  0.292\n",
      "Time_Point[T.Week_13]:Group[T.Sham]        0.007    0.115  0.058 0.954 -0.218  0.231\n",
      "Time_Point[T.Week_15]:Group[T.Sham]       -0.077    0.115 -0.671 0.502 -0.301  0.148\n",
      "Time_Point[T.Baseline_2]:Group[T.Treated]  0.000    0.115  0.000 1.000 -0.224  0.224\n",
      "Time_Point[T.Drug_Trt]:Group[T.Treated]   -0.004                                    \n",
      "Time_Point[T.Week_02]:Group[T.Treated]     0.044    0.115  0.386 0.699 -0.180  0.269\n",
      "Time_Point[T.Week_04]:Group[T.Treated]    -0.017    0.115 -0.145 0.885 -0.241  0.208\n",
      "Time_Point[T.Week_06]:Group[T.Treated]     0.130    0.115  1.135 0.256 -0.094  0.355\n",
      "Time_Point[T.Week_08]:Group[T.Treated]    -0.030    0.115 -0.265 0.791 -0.255  0.194\n",
      "Time_Point[T.Week_11]:Group[T.Treated]     0.083    0.115  0.724 0.469 -0.142  0.307\n",
      "Time_Point[T.Week_13]:Group[T.Treated]    -0.004    0.115 -0.032 0.975 -0.228  0.221\n",
      "Time_Point[T.Week_15]:Group[T.Treated]    -0.045    0.115 -0.392 0.695 -0.269  0.180\n",
      "Group Var                                  0.000    0.000                           \n",
      "====================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/adanredwine/miniconda3/envs/VAME-2/lib/python3.11/site-packages/statsmodels/regression/mixed_linear_model.py:2706: RuntimeWarning: invalid value encountered in sqrt\n",
      "  sdf[0:self.k_fe, 1] = np.sqrt(np.diag(self.cov_params()[0:self.k_fe]))\n"
     ]
    }
   ],
   "source": [
    "reload(AlHf)\n",
    "result_logNorm_base = AlHf.run_mixed_effects_model(df_long_logNorm_base, dependent_var='Log_Normalized_Value', groups='Animal_ID', re_formula='1', fe_formula='Time_Point * Group', maxiter=100)\n",
    "print(result_logNorm_base.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/adanredwine/miniconda3/envs/VAME-2/lib/python3.11/site-packages/statsmodels/regression/mixed_linear_model.py:1635: UserWarning: Random effects covariance is singular\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converged with lbfgs\n",
      "                       Mixed Linear Model Regression Results\n",
      "====================================================================================\n",
      "Model:                  MixedLM       Dependent Variable:       Log_Normalized_Value\n",
      "No. Observations:       14040         Method:                   REML                \n",
      "No. Groups:             48            Scale:                    0.9990              \n",
      "Min. group size:        270           Log-Likelihood:           -20002.2062         \n",
      "Max. group size:        300           Converged:                Yes                 \n",
      "Mean group size:        292.5                                                       \n",
      "------------------------------------------------------------------------------------\n",
      "                                          Coef.  Std.Err.   z    P>|z| [0.025 0.975]\n",
      "------------------------------------------------------------------------------------\n",
      "Intercept                                 -0.007    0.067 -0.108 0.914 -0.139  0.125\n",
      "Time_Point[T.Baseline_2]                  -0.121    0.074 -1.627 0.104 -0.267  0.025\n",
      "Time_Point[T.Drug_Trt]                    -0.005                                    \n",
      "Time_Point[T.Week_02]                      0.001    0.074  0.012 0.991 -0.145  0.147\n",
      "Time_Point[T.Week_04]                      0.082    0.074  1.103 0.270 -0.064  0.228\n",
      "Time_Point[T.Week_06]                     -0.084    0.074 -1.128 0.259 -0.230  0.062\n",
      "Time_Point[T.Week_08]                     -0.057    0.074 -0.765 0.444 -0.203  0.089\n",
      "Time_Point[T.Week_11]                     -0.128    0.074 -1.721 0.085 -0.274  0.018\n",
      "Time_Point[T.Week_13]                     -0.067    0.074 -0.902 0.367 -0.213  0.079\n",
      "Time_Point[T.Week_15]                      0.016    0.074  0.218 0.827 -0.130  0.162\n",
      "Group[T.Injured]                           0.130    0.095  1.365 0.172 -0.057  0.316\n",
      "Group[T.Sham]                              0.007    0.095  0.076 0.939 -0.179  0.194\n",
      "Group[T.Treated]                           0.129    0.095  1.360 0.174 -0.057  0.316\n",
      "Time_Point[T.Baseline_2]:Group[T.Injured]  0.138    0.105  1.312 0.190 -0.068  0.345\n",
      "Time_Point[T.Drug_Trt]:Group[T.Injured]    0.015                                    \n",
      "Time_Point[T.Week_02]:Group[T.Injured]     0.095    0.105  0.903 0.367 -0.111  0.302\n",
      "Time_Point[T.Week_04]:Group[T.Injured]     0.051    0.105  0.489 0.625 -0.155  0.258\n",
      "Time_Point[T.Week_06]:Group[T.Injured]     0.033    0.105  0.315 0.753 -0.173  0.240\n",
      "Time_Point[T.Week_08]:Group[T.Injured]     0.150    0.105  1.422 0.155 -0.057  0.356\n",
      "Time_Point[T.Week_11]:Group[T.Injured]     0.159    0.105  1.508 0.132 -0.048  0.365\n",
      "Time_Point[T.Week_13]:Group[T.Injured]     0.060    0.105  0.568 0.570 -0.147  0.266\n",
      "Time_Point[T.Week_15]:Group[T.Injured]     0.060    0.105  0.571 0.568 -0.146  0.267\n",
      "Time_Point[T.Baseline_2]:Group[T.Sham]     0.121    0.105  1.151 0.250 -0.085  0.328\n",
      "Time_Point[T.Drug_Trt]:Group[T.Sham]       0.005                                    \n",
      "Time_Point[T.Week_02]:Group[T.Sham]       -0.001    0.105 -0.008 0.993 -0.207  0.206\n",
      "Time_Point[T.Week_04]:Group[T.Sham]       -0.082    0.105 -0.780 0.436 -0.289  0.124\n",
      "Time_Point[T.Week_06]:Group[T.Sham]        0.084    0.105  0.798 0.425 -0.122  0.291\n",
      "Time_Point[T.Week_08]:Group[T.Sham]        0.057    0.105  0.541 0.588 -0.149  0.264\n",
      "Time_Point[T.Week_11]:Group[T.Sham]        0.128    0.105  1.217 0.224 -0.078  0.335\n",
      "Time_Point[T.Week_13]:Group[T.Sham]        0.067    0.105  0.638 0.523 -0.139  0.274\n",
      "Time_Point[T.Week_15]:Group[T.Sham]       -0.016    0.105 -0.154 0.877 -0.223  0.190\n",
      "Time_Point[T.Baseline_2]:Group[T.Treated]  0.105    0.105  0.995 0.320 -0.102  0.311\n",
      "Time_Point[T.Drug_Trt]:Group[T.Treated]    0.020                                    \n",
      "Time_Point[T.Week_02]:Group[T.Treated]     0.097    0.105  0.917 0.359 -0.110  0.303\n",
      "Time_Point[T.Week_04]:Group[T.Treated]     0.036    0.105  0.340 0.734 -0.171  0.242\n",
      "Time_Point[T.Week_06]:Group[T.Treated]     0.182    0.105  1.731 0.083 -0.024  0.389\n",
      "Time_Point[T.Week_08]:Group[T.Treated]     0.022    0.105  0.209 0.835 -0.185  0.228\n",
      "Time_Point[T.Week_11]:Group[T.Treated]     0.135    0.105  1.285 0.199 -0.071  0.342\n",
      "Time_Point[T.Week_13]:Group[T.Treated]     0.049    0.105  0.463 0.643 -0.158  0.255\n",
      "Time_Point[T.Week_15]:Group[T.Treated]     0.008    0.105  0.071 0.943 -0.199  0.214\n",
      "Group Var                                  0.021    0.005                           \n",
      "====================================================================================\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/adanredwine/miniconda3/envs/VAME-2/lib/python3.11/site-packages/statsmodels/regression/mixed_linear_model.py:2706: RuntimeWarning: invalid value encountered in sqrt\n",
      "  sdf[0:self.k_fe, 1] = np.sqrt(np.diag(self.cov_params()[0:self.k_fe]))\n"
     ]
    }
   ],
   "source": [
    "reload(AlHf)\n",
    "result_logNorm_sham = AlHf.run_mixed_effects_model(df_long_logNorm_sham, dependent_var='Log_Normalized_Value', groups='Animal_ID', re_formula='1', fe_formula='Time_Point * Group', maxiter=100)\n",
    "print(result_logNorm_sham.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### **Notes on the mixed effects models:**\n",
    "\n",
    "\n",
    "<span style=\"color:blue;\">Formula: </span> Adjust the fe_formula and re_formula according to specific model needs. The fe_formula represents the fixed effects you want to include in the model, and re_formula represents the random effects.\n",
    "\n",
    "*In mixed-effects models, the terms \"fixed effects\" and \"random effects\" have specific meanings:*\n",
    "**Fixed Effects:**\n",
    "Fixed effects are the variables of interest that you expect to have a systematic and consistent impact on the dependent variable across all units of observation. They are the factors you believe will influence your outcome variable in a predictable way. In your model, fixed effects would include:\n",
    "\n",
    "- Time Points: These represent the different time measurements in your study. You are likely interested in how the dependent variable changes over these time points.\n",
    "- Groups: These represent the different treatment or condition groups in your study. You are likely interested in the differences in the dependent variable between these groups.\n",
    "\n",
    "The coefficients for fixed effects are estimated directly and apply to the entire population. They are akin to the coefficients in a standard regression model.\n",
    "**Random Effects:**\n",
    "Random effects are variables that introduce variability into the model but are not the primary variables of interest. Instead, they account for the random variation at different levels of the data hierarchy. In your model, the random effects might include:\n",
    "\n",
    "- Animal_ID: This represents the individual animals in your study. Each animal might respond differently to the treatments or conditions, and these individual differences are not the main focus of your study. However, they are important to account for because they can influence the dependent variable.\n",
    "\n",
    "Random effects are assumed to be randomly distributed across different levels of the data (e.g., subjects or clusters). They allow the model to account for the fact that observations within the same group (like the same animal) are not independent and may be more similar to each other than to observations from different groups.\n",
    "\n",
    "In summary, fixed effects are your main variables of interest that you hypothesize will affect the outcome consistently, while random effects are used to model the variability in the data that is not explained by the fixed effects. Random effects allow for individual differences and are useful when data is collected in a nested or hierarchical structure (like multiple measurements from the same subjects).\n",
    "\n",
    "\n",
    "```diff\n",
    "+ ### How to Color Text in Markdown\n",
    "+ \n",
    "+ To color text in markdown, you typically need to use HTML tags, as standard Markdown does not support text color directly. Here's an example of how to change the color of text using HTML:\n",
    "+\n",
    "+ ```html\n",
    "+ <span style=\"color:red;\">This text will be red</span>\n",
    "+ ```\n",
    "+\n",
    "+ Replace `red` with any other color name or hex code to change the text color accordingly.\n",
    "```\n",
    "\n",
    "\n",
    "The output you're seeing is the summary of a mixed-effects linear model that you've run using the statsmodels library in Python. This model is used to analyze data that has both fixed effects (such as time points and treatment groups) and random effects (such as individual variability in animals). Here's a breakdown of the output and the warnings:\n",
    "\n",
    "<span style=\"color:red;\">Warnings:</span>\n",
    "\n",
    "1. Convergence Warning: This warning indicates that the optimization algorithm used to find the maximum likelihood estimates did not converge. This means that the algorithm did not successfully find the best-fit parameters within the specified number of iterations or tolerance levels. It suggests that the results may not be reliable.\n",
    "\n",
    "2. Retrying MixedLM optimization with lbfgs: This is a follow-up message indicating that the algorithm is retrying with a different optimization method (lbfgs) to attempt to find a solution.\n",
    "\n",
    "3. Random effects covariance is singular: This warning suggests that the variance-covariance matrix for the random effects has some issues, possibly due to a lack of variability in the random effects or overparameterization of the model.\n",
    "**Model Summary:**\n",
    "- Dependent Variable: The outcome variable that the model is trying to predict (Normalized_Value).\n",
    "- Method: The method used to estimate the parameters (REML stands for Restricted Maximum Likelihood).\n",
    "- No. Observations: The number of data points used in the model (18,720).\n",
    "- No. Groups: The number of groups for the random effects (48, which likely corresponds to the number of Animal_IDs).\n",
    "- Scale: The estimated variance of the residuals from the model.\n",
    "- Log-Likelihood: The log-likelihood of the model, which is a measure of model fit (higher is generally better).\n",
    "- Converged: Indicates whether the optimization algorithm successfully converged (Yes in this case, despite the initial warning).\n",
    "\n",
    "**Coefficients Table:**\n",
    "- Coef.: The estimated effect sizes for each fixed effect.\n",
    "- Std.Err.: The standard error of each coefficient estimate.\n",
    "- z: The z-statistic for testing the null hypothesis that the coefficient is zero.\n",
    "- P>|z|: The p-value associated with the z-statistic.\n",
    "- [0.025 0.975]: The 95% confidence interval for the coefficient estimates.\n",
    "\n",
    "**Interpretation:**\n",
    "- Intercept: The baseline level of the dependent variable when all predictors are at their reference levels.\n",
    "- Time_Point and Group: The estimated effects of different time points and groups on the dependent variable.\n",
    "- Group Var: The estimated variance associated with the random effects (Animal_ID).\n",
    "\n",
    "**Important Points:**\n",
    "- The model has estimated effects for each time point within each group, as well as interactions between time points and groups.\n",
    "- Some coefficients do not have associated standard errors or p-values, which might be due to the singular covariance matrix or other issues with the data.\n",
    "- The p-values for some of the coefficients are significant (less than 0.05), suggesting that those time points or group interactions have a statistically significant effect on the dependent variable.\n",
    "  \n",
    "**Next Steps:**\n",
    "Given the warnings and some missing information in the output, may need to:\n",
    "- Check the data for any issues that could affect the model (e.g., outliers, missing data).\n",
    "- Consider simplifying the model if it's overparameterized.\n",
    "- Consult with a statistician to ensure the model is specified correctly and to interpret the results in the context of the study.\n",
    "- If convergence issues persist, you might need to adjust the optimization settings or consider alternative modeling approaches."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KruskalResult(statistic=24.827273570428428, pvalue=1.677920008189288e-05)\n"
     ]
    }
   ],
   "source": [
    "result = AlHf.run_kruskal_wallis_test(df_long, groups='Group', values='value')\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checking for normality after log normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Group the data by 'Time_Point', 'Group', and 'Motif' and apply the test\n",
    "normality_results_vsSham_Group = df_long_logNorm_sham.groupby(['Time_Point', 'Group', 'Motif']).apply(test_normality).reset_index()\n",
    "normality_results_vsBase_Group = df_long_logNorm_base.groupby(['Time_Point', 'Group', 'Motif']).apply(test_normality).reset_index()\n",
    "\n",
    "normality_results_vsSham_State = df_long_logNorm_sham.groupby(['Time_Point', 'Treatment_State', 'Motif']).apply(test_normality).reset_index()\n",
    "normality_results_vsBase_State = df_long_logNorm_base.groupby(['Time_Point', 'Treatment_State', 'Motif']).apply(test_normality).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ic| percent_normal_vsSham_Group: 89.57264957264958\n",
      "ic| percent_normal_vsBase_Group: 93.58974358974359\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "93.58974358974359"
      ]
     },
     "execution_count": 312,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normal_motifs_vsSham_Group = normality_results_vsSham_Group[normality_results_vsSham_Group['p'] > 0.05]\n",
    "non_normal_motifs_vsSham_Group = normality_results_vsSham_Group[normality_results_vsSham_Group['p'] <= 0.05]\n",
    "percent_normal_vsSham_Group = len(normal_motifs_vsSham_Group) / len(normality_results_vsSham_Group) * 100\n",
    "\n",
    "normal_motifs_vsBase_Group = normality_results_vsBase_Group[normality_results_vsBase_Group['p'] > 0.05]\n",
    "non_normal_motifs_vsBase_Group = normality_results_vsBase_Group[normality_results_vsBase_Group['p'] <= 0.05]\n",
    "percent_normal_vsBase_Group = len(normal_motifs_vsBase_Group) / len(normality_results_vsBase_Group) * 100\n",
    "\n",
    "ic(percent_normal_vsSham_Group)\n",
    "ic(percent_normal_vsBase_Group)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ic| percent_normal_vsSham_State: 88.38383838383838\n",
      "ic| percent_normal_vsBase_State: 92.82828282828282\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "92.82828282828282"
      ]
     },
     "execution_count": 313,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normal_motifs_vsSham_State = normality_results_vsSham_State[normality_results_vsSham_State['p'] > 0.05]\n",
    "non_normal_motifs_vsSham_State = normality_results_vsSham_State[normality_results_vsSham_State['p'] <= 0.05]\n",
    "percent_normal_vsSham_State = len(normal_motifs_vsSham_State) / len(normality_results_vsSham_State) * 100\n",
    "\n",
    "normal_motifs_vsBase_State = normality_results_vsBase_State[normality_results_vsBase_State['p'] > 0.05]\n",
    "non_normal_motifs_vsBase_State = normality_results_vsBase_State[normality_results_vsBase_State['p'] <= 0.05]\n",
    "percent_normal_vsBase_State = len(normal_motifs_vsBase_State) / len(normality_results_vsBase_State) * 100\n",
    "\n",
    "ic(percent_normal_vsSham_State)\n",
    "ic(percent_normal_vsBase_State)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Based on the Shapiro-Wilk test , it appears that the majority of the motif usages for each group (and state) at each time point do not significantly deviate from a normal distribution. This is indicated by the p-values being greater than the typical alpha level of 0.05.\n",
    "\n",
    "Given that the non-normal cases are relatively few compared to the normal ones, it does suggest that almost all of the motif usages are normal. However, for those motifs that are not normally distributed, will need use non-parametric statistical tests or consider data transformation techniques if parametric tests are required for analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time Point: Baseline_1, Group: ABX\n",
      "    Motif         W         p\n",
      "3       3  0.854541  0.041790\n",
      "12     12  0.736465  0.001937\n",
      "13     13  0.805416  0.010803\n",
      "\n",
      "\n",
      "Time Point: Baseline_1, Group: Injured\n",
      "    Motif         W         p\n",
      "35      5  0.847740  0.034432\n",
      "41     11  0.759385  0.003355\n",
      "42     12  0.831113  0.021627\n",
      "\n",
      "\n",
      "Time Point: Baseline_1, Group: Treated\n",
      "     Motif         W         p\n",
      "90       0  0.852460  0.039377\n",
      "99       9  0.828578  0.020169\n",
      "100     10  0.839822  0.027550\n",
      "102     12  0.850229  0.036953\n",
      "103     13  0.850584  0.037328\n",
      "104     14  0.855709  0.043212\n",
      "105     15  0.804713  0.010605\n",
      "108     18  0.843479  0.030528\n",
      "\n",
      "\n",
      "Time Point: Baseline_2, Group: ABX\n",
      "     Motif         W         p\n",
      "131     11  0.798395  0.008984\n",
      "132     12  0.763296  0.003692\n",
      "133     13  0.736121  0.001922\n",
      "135     15  0.831795  0.022038\n",
      "\n",
      "\n",
      "Time Point: Baseline_2, Group: Injured\n",
      "     Motif         W         p\n",
      "156      6  0.837713  0.025973\n",
      "165     15  0.735604  0.001898\n",
      "\n",
      "\n",
      "Time Point: Baseline_2, Group: Treated\n",
      "     Motif         W         p\n",
      "211      1  0.842461  0.029666\n",
      "225     15  0.668201  0.000423\n",
      "\n",
      "\n",
      "Time Point: Drug_Trt, Group: Injured\n",
      "     Motif         W         p\n",
      "252     12  0.848073  0.034758\n",
      "253     13  0.821337  0.016549\n",
      "256     16  0.781692  0.005845\n",
      "\n",
      "\n",
      "Time Point: Drug_Trt, Group: Treated\n",
      "     Motif         W         p\n",
      "309      9  0.842616  0.029796\n",
      "311     11  0.813531  0.013407\n",
      "317     17  0.837110  0.025540\n",
      "327     27  0.843167  0.030261\n",
      "\n",
      "\n",
      "Time Point: Week_02, Group: ABX\n",
      "     Motif         W         p\n",
      "333      3  0.841579  0.028940\n",
      "336      6  0.852989  0.039977\n",
      "341     11  0.839834  0.027559\n",
      "343     13  0.825570  0.018572\n",
      "354     24  0.852144  0.039024\n",
      "\n",
      "\n",
      "Time Point: Week_02, Group: Injured\n",
      "     Motif         W         p\n",
      "371     11  0.801300  0.009694\n",
      "372     12  0.853004  0.039993\n",
      "387     27  0.849302  0.035992\n",
      "\n",
      "\n",
      "Time Point: Week_02, Group: Treated\n",
      "     Motif         W         p\n",
      "445     25  0.858362  0.046633\n",
      "447     27  0.849254  0.035943\n",
      "\n",
      "\n",
      "Time Point: Week_04, Group: ABX\n",
      "     Motif         W         p\n",
      "460     10  0.855595  0.043072\n",
      "462     12  0.799957  0.009358\n",
      "463     13  0.816054  0.014347\n",
      "465     15  0.767005  0.004046\n",
      "\n",
      "\n",
      "Time Point: Week_04, Group: Injured\n",
      "     Motif         W         p\n",
      "493     13  0.765501  0.003898\n",
      "495     15  0.821660  0.016695\n",
      "504     24  0.815252  0.014041\n",
      "\n",
      "\n",
      "Time Point: Week_04, Group: Treated\n",
      "     Motif         W         p\n",
      "552     12  0.790945  0.007405\n",
      "553     13  0.832377  0.022396\n",
      "555     15  0.804208  0.010464\n",
      "565     25  0.820095  0.016001\n",
      "\n",
      "\n",
      "Time Point: Week_06, Group: ABX\n",
      "     Motif         W         p\n",
      "582     12  0.755318  0.003039\n",
      "583     13  0.780049  0.005607\n",
      "584     14  0.836303  0.024973\n",
      "587     17  0.810346  0.012313\n",
      "\n",
      "\n",
      "Time Point: Week_06, Group: Injured\n",
      "     Motif         W         p\n",
      "604      4  0.806278  0.011052\n",
      "612     12  0.840139  0.027796\n",
      "615     15  0.748606  0.002584\n",
      "627     27  0.840646  0.028193\n",
      "\n",
      "\n",
      "Time Point: Week_06, Group: Treated\n",
      "     Motif         W         p\n",
      "662      2  0.783679  0.006148\n",
      "667      7  0.859913  0.048765\n",
      "672     12  0.856741  0.044511\n",
      "680     20  0.837779  0.026021\n",
      "\n",
      "\n",
      "Time Point: Week_08, Group: ABX\n",
      "     Motif         W         p\n",
      "690      0  0.833785  0.023286\n",
      "702     12  0.812569  0.013066\n",
      "703     13  0.831216  0.021688\n",
      "\n",
      "\n",
      "Time Point: Week_08, Group: Injured\n",
      "     Motif         W         p\n",
      "723      3  0.808954  0.011865\n",
      "732     12  0.820900  0.016354\n",
      "733     13  0.849222  0.035910\n",
      "745     25  0.828117  0.019915\n",
      "\n",
      "\n",
      "Time Point: Week_08, Group: Treated\n",
      "     Motif         W         p\n",
      "780      0  0.846322  0.033077\n",
      "789      9  0.848173  0.034857\n",
      "792     12  0.807477  0.011409\n",
      "803     23  0.853651  0.040740\n",
      "804     24  0.634813  0.000213\n",
      "808     28  0.814352  0.013705\n",
      "809     29  0.830330  0.021165\n",
      "\n",
      "\n",
      "Time Point: Week_11, Group: ABX\n",
      "     Motif         W         p\n",
      "813      3  0.806135  0.011010\n",
      "822     12  0.854762  0.042056\n",
      "823     13  0.739776  0.002094\n",
      "825     15  0.828129  0.019921\n",
      "832     22  0.759834  0.003392\n",
      "837     27  0.851925  0.038781\n",
      "\n",
      "\n",
      "Time Point: Week_11, Group: Injured\n",
      "     Motif         W         p\n",
      "850     10  0.845125  0.031978\n",
      "851     11  0.816457  0.014503\n",
      "853     13  0.812089  0.012900\n",
      "860     20  0.821373  0.016565\n",
      "867     27  0.779922  0.005589\n",
      "\n",
      "\n",
      "Time Point: Week_11, Group: Treated\n",
      "     Motif         W         p\n",
      "902      2  0.783562  0.006129\n",
      "905      5  0.794628  0.008145\n",
      "911     11  0.827299  0.019473\n",
      "912     12  0.860768  0.049982\n",
      "915     15  0.804910  0.010660\n",
      "918     18  0.859478  0.048157\n",
      "920     20  0.859457  0.048127\n",
      "924     24  0.786709  0.006642\n",
      "927     27  0.860000  0.048886\n",
      "\n",
      "\n",
      "Time Point: Week_13, Group: ABX\n",
      "     Motif         W         p\n",
      "932      2  0.750866  0.002729\n",
      "935      5  0.847794  0.034484\n",
      "942     12  0.721529  0.001369\n",
      "945     15  0.819715  0.015837\n",
      "952     22  0.730150  0.001671\n",
      "\n",
      "\n",
      "Time Point: Week_13, Group: Injured\n",
      "     Motif         W         p\n",
      "965      5  0.643620  0.000254\n",
      "975     15  0.808169  0.011620\n",
      "987     27  0.824966  0.018268\n",
      "\n",
      "\n",
      "Time Point: Week_13, Group: Treated\n",
      "      Motif         W         p\n",
      "1032     12  0.847508  0.034206\n",
      "1033     13  0.759838  0.003392\n",
      "1045     25  0.860717  0.049908\n",
      "\n",
      "\n",
      "Time Point: Week_15, Group: ABX\n",
      "      Motif         W         p\n",
      "1051      1  0.800336  0.009452\n",
      "1057      7  0.838685  0.026688\n",
      "1063     13  0.827453  0.019555\n",
      "1065     15  0.825630  0.018603\n",
      "1068     18  0.742668  0.002243\n",
      "1069     19  0.860696  0.049878\n",
      "\n",
      "\n",
      "Time Point: Week_15, Group: Injured\n",
      "      Motif         W         p\n",
      "1092     12  0.855613  0.043093\n",
      "1093     13  0.791210  0.007456\n",
      "\n",
      "\n",
      "Time Point: Week_15, Group: Treated\n",
      "      Motif         W         p\n",
      "1143      3  0.753421  0.002902\n",
      "1144      4  0.745435  0.002396\n",
      "1149      9  0.775157  0.004957\n",
      "1152     12  0.813018  0.013224\n",
      "1153     13  0.837835  0.026062\n",
      "1154     14  0.810396  0.012330\n",
      "1155     15  0.800416  0.009471\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Examining the non-normal motifs more:\n",
    "# Group by 'Time_Point' and 'Group' to see which motifs are not normal in each case\n",
    "grouped_non_normal_motifs_Group = non_normal_motifs_vsSham_Group.groupby(['Time_Point', 'Group'])\n",
    "\n",
    "# Iterate over each group and print the motifs that are not normally distributed\n",
    "for (time_point, group), motifs in grouped_non_normal_motifs_Group:\n",
    "    print(f\"Time Point: {time_point}, Group: {group}\")\n",
    "    print(motifs[['Motif', 'W', 'p']])\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creating a column in the sham normalized data frame to denote when a combination of time point, group, and motif aren't normally distributed:\n",
    "# Create a dictionary to map the normality status\n",
    "normality_vsSham_dict_Group = {(row['Time_Point'], row['Group'], row['Motif']): row['p'] > 0.05 for index, row in normality_results_vsSham_Group.iterrows()}\n",
    "normality_vsBase_dict_Group = {(row['Time_Point'], row['Group'], row['Motif']): row['p'] > 0.05 for index, row in normality_results_vsBase_Group.iterrows()}\n",
    "\n",
    "normality_vsSham_dict_State = {(row['Time_Point'], row['Treatment_State'], row['Motif']): row['p'] > 0.05 for index, row in normality_results_vsSham_State.iterrows()}\n",
    "normality_vsBase_dict_State = {(row['Time_Point'], row['Treatment_State'], row['Motif']): row['p'] > 0.05 for index, row in normality_results_vsBase_State.iterrows()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to apply the normality status based on the dictionary for Group\n",
    "def apply_normality_status_Group(row, normality_dict):\n",
    "    return normality_dict.get((row['Time_Point'], row['Group'], row['Motif']), True)  # Default to True if not found\n",
    "\n",
    "# Apply the function to create a new column 'Is_Normal_Group'\n",
    "df_long_logNorm_sham['Is_Normal_Group'] = df_long_logNorm_sham.apply(apply_normality_status_Group, axis=1, normality_dict=normality_vsSham_dict_Group)\n",
    "df_long_logNorm_base['Is_Normal_Group'] = df_long_logNorm_base.apply(apply_normality_status_Group, axis=1, normality_dict=normality_vsBase_dict_Group)\n",
    "\n",
    "# Function to apply the normality status based on the dictionary for Treatment_State\n",
    "def apply_normality_status_State(row, normality_dict):\n",
    "    return normality_dict.get((row['Time_Point'], row['Treatment_State'], row['Motif']), True)  # Default to True if not found\n",
    "\n",
    "# Apply the function to create a new column 'Is_Normal_State'\n",
    "df_long_logNorm_sham['Is_Normal_State'] = df_long_logNorm_sham.apply(apply_normality_status_State, axis=1, normality_dict=normality_vsSham_dict_State)\n",
    "df_long_logNorm_base['Is_Normal_State'] = df_long_logNorm_base.apply(apply_normality_status_State, axis=1, normality_dict=normality_vsBase_dict_State)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Animal_ID</th>\n",
       "      <th>Time_Point</th>\n",
       "      <th>Group</th>\n",
       "      <th>Treatment_State</th>\n",
       "      <th>Motif</th>\n",
       "      <th>value</th>\n",
       "      <th>Cluster</th>\n",
       "      <th>Baseline_Mean</th>\n",
       "      <th>Normalized_Value</th>\n",
       "      <th>log_value</th>\n",
       "      <th>Sham_Log_Mean</th>\n",
       "      <th>Log_Normalized_Value</th>\n",
       "      <th>Is_Normal_Group</th>\n",
       "      <th>Is_Normal_State</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>424</td>\n",
       "      <td>5</td>\n",
       "      <td>436.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.052089</td>\n",
       "      <td>6.824923</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>1</td>\n",
       "      <td>105</td>\n",
       "      <td>1</td>\n",
       "      <td>73.5</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.663439</td>\n",
       "      <td>5.896740</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>2</td>\n",
       "      <td>764</td>\n",
       "      <td>3</td>\n",
       "      <td>1005.5</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.639876</td>\n",
       "      <td>6.600722</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>3</td>\n",
       "      <td>199</td>\n",
       "      <td>1</td>\n",
       "      <td>161.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.298317</td>\n",
       "      <td>5.864924</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>4</td>\n",
       "      <td>696</td>\n",
       "      <td>3</td>\n",
       "      <td>657.5</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.546785</td>\n",
       "      <td>6.420746</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14035</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>25</td>\n",
       "      <td>940</td>\n",
       "      <td>7</td>\n",
       "      <td>1972.5</td>\n",
       "      <td>0.476553</td>\n",
       "      <td>6.846943</td>\n",
       "      <td>7.143300</td>\n",
       "      <td>-0.296357</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14036</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>26</td>\n",
       "      <td>796</td>\n",
       "      <td>10</td>\n",
       "      <td>1700.0</td>\n",
       "      <td>0.468235</td>\n",
       "      <td>6.680855</td>\n",
       "      <td>6.565693</td>\n",
       "      <td>0.115162</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14037</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>27</td>\n",
       "      <td>134</td>\n",
       "      <td>3</td>\n",
       "      <td>26.0</td>\n",
       "      <td>5.153846</td>\n",
       "      <td>4.905275</td>\n",
       "      <td>3.710625</td>\n",
       "      <td>1.194649</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14038</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>28</td>\n",
       "      <td>738</td>\n",
       "      <td>9</td>\n",
       "      <td>916.5</td>\n",
       "      <td>0.805237</td>\n",
       "      <td>6.605298</td>\n",
       "      <td>6.536103</td>\n",
       "      <td>0.069195</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14039</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>1804</td>\n",
       "      <td>8</td>\n",
       "      <td>2268.5</td>\n",
       "      <td>0.795239</td>\n",
       "      <td>7.498316</td>\n",
       "      <td>7.276425</td>\n",
       "      <td>0.221891</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14040 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Animal_ID  Time_Point    Group Treatment_State  Motif  value  Cluster  \\\n",
       "0            A1  Baseline_1     Sham            Sham      0    424        5   \n",
       "1            A1  Baseline_1     Sham            Sham      1    105        1   \n",
       "2            A1  Baseline_1     Sham            Sham      2    764        3   \n",
       "3            A1  Baseline_1     Sham            Sham      3    199        1   \n",
       "4            A1  Baseline_1     Sham            Sham      4    696        3   \n",
       "...         ...         ...      ...             ...    ...    ...      ...   \n",
       "14035        Y2    Drug_Trt  Treated         Treated     25    940        7   \n",
       "14036        Y2    Drug_Trt  Treated         Treated     26    796       10   \n",
       "14037        Y2    Drug_Trt  Treated         Treated     27    134        3   \n",
       "14038        Y2    Drug_Trt  Treated         Treated     28    738        9   \n",
       "14039        Y2    Drug_Trt  Treated         Treated     29   1804        8   \n",
       "\n",
       "       Baseline_Mean  Normalized_Value  log_value  Sham_Log_Mean  \\\n",
       "0              436.0          1.000000   6.052089       6.824923   \n",
       "1               73.5          1.000000   4.663439       5.896740   \n",
       "2             1005.5          1.000000   6.639876       6.600722   \n",
       "3              161.0          1.000000   5.298317       5.864924   \n",
       "4              657.5          1.000000   6.546785       6.420746   \n",
       "...              ...               ...        ...            ...   \n",
       "14035         1972.5          0.476553   6.846943       7.143300   \n",
       "14036         1700.0          0.468235   6.680855       6.565693   \n",
       "14037           26.0          5.153846   4.905275       3.710625   \n",
       "14038          916.5          0.805237   6.605298       6.536103   \n",
       "14039         2268.5          0.795239   7.498316       7.276425   \n",
       "\n",
       "       Log_Normalized_Value  Is_Normal_Group  Is_Normal_State  \n",
       "0                  0.000000             True             True  \n",
       "1                  0.000000             True             True  \n",
       "2                  0.000000             True             True  \n",
       "3                  0.000000             True             True  \n",
       "4                  0.000000             True             True  \n",
       "...                     ...              ...              ...  \n",
       "14035             -0.296357             True             True  \n",
       "14036              0.115162             True             True  \n",
       "14037              1.194649            False            False  \n",
       "14038              0.069195             True             True  \n",
       "14039              0.221891             True             True  \n",
       "\n",
       "[14040 rows x 14 columns]"
      ]
     },
     "execution_count": 318,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_long_logNorm_sham"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Animal_ID</th>\n",
       "      <th>Time_Point</th>\n",
       "      <th>Group</th>\n",
       "      <th>Treatment_State</th>\n",
       "      <th>Motif</th>\n",
       "      <th>value</th>\n",
       "      <th>Cluster</th>\n",
       "      <th>Baseline_Mean</th>\n",
       "      <th>Normalized_Value</th>\n",
       "      <th>log_value</th>\n",
       "      <th>Baseline_Log_Mean</th>\n",
       "      <th>Log_Normalized_Value</th>\n",
       "      <th>Is_Normal_Group</th>\n",
       "      <th>Is_Normal_State</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>424</td>\n",
       "      <td>5</td>\n",
       "      <td>436.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.052089</td>\n",
       "      <td>6.079556</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>1</td>\n",
       "      <td>105</td>\n",
       "      <td>1</td>\n",
       "      <td>73.5</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.663439</td>\n",
       "      <td>4.212320</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>2</td>\n",
       "      <td>764</td>\n",
       "      <td>3</td>\n",
       "      <td>1005.5</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.639876</td>\n",
       "      <td>6.884587</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>3</td>\n",
       "      <td>199</td>\n",
       "      <td>1</td>\n",
       "      <td>161.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.298317</td>\n",
       "      <td>5.059299</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>4</td>\n",
       "      <td>696</td>\n",
       "      <td>3</td>\n",
       "      <td>657.5</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.546785</td>\n",
       "      <td>6.488252</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14035</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>25</td>\n",
       "      <td>940</td>\n",
       "      <td>7</td>\n",
       "      <td>1972.5</td>\n",
       "      <td>0.476553</td>\n",
       "      <td>6.846943</td>\n",
       "      <td>7.567698</td>\n",
       "      <td>-0.720755</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14036</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>26</td>\n",
       "      <td>796</td>\n",
       "      <td>10</td>\n",
       "      <td>1700.0</td>\n",
       "      <td>0.468235</td>\n",
       "      <td>6.680855</td>\n",
       "      <td>7.438958</td>\n",
       "      <td>-0.758103</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14037</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>27</td>\n",
       "      <td>134</td>\n",
       "      <td>3</td>\n",
       "      <td>26.0</td>\n",
       "      <td>5.153846</td>\n",
       "      <td>4.905275</td>\n",
       "      <td>3.001944</td>\n",
       "      <td>1.903331</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14038</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>28</td>\n",
       "      <td>738</td>\n",
       "      <td>9</td>\n",
       "      <td>916.5</td>\n",
       "      <td>0.805237</td>\n",
       "      <td>6.605298</td>\n",
       "      <td>6.821610</td>\n",
       "      <td>-0.216312</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14039</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>1804</td>\n",
       "      <td>8</td>\n",
       "      <td>2268.5</td>\n",
       "      <td>0.795239</td>\n",
       "      <td>7.498316</td>\n",
       "      <td>7.706572</td>\n",
       "      <td>-0.208256</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14040 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Animal_ID  Time_Point    Group Treatment_State  Motif  value  Cluster  \\\n",
       "0            A1  Baseline_1     Sham            Sham      0    424        5   \n",
       "1            A1  Baseline_1     Sham            Sham      1    105        1   \n",
       "2            A1  Baseline_1     Sham            Sham      2    764        3   \n",
       "3            A1  Baseline_1     Sham            Sham      3    199        1   \n",
       "4            A1  Baseline_1     Sham            Sham      4    696        3   \n",
       "...         ...         ...      ...             ...    ...    ...      ...   \n",
       "14035        Y2    Drug_Trt  Treated         Treated     25    940        7   \n",
       "14036        Y2    Drug_Trt  Treated         Treated     26    796       10   \n",
       "14037        Y2    Drug_Trt  Treated         Treated     27    134        3   \n",
       "14038        Y2    Drug_Trt  Treated         Treated     28    738        9   \n",
       "14039        Y2    Drug_Trt  Treated         Treated     29   1804        8   \n",
       "\n",
       "       Baseline_Mean  Normalized_Value  log_value  Baseline_Log_Mean  \\\n",
       "0              436.0          1.000000   6.052089           6.079556   \n",
       "1               73.5          1.000000   4.663439           4.212320   \n",
       "2             1005.5          1.000000   6.639876           6.884587   \n",
       "3              161.0          1.000000   5.298317           5.059299   \n",
       "4              657.5          1.000000   6.546785           6.488252   \n",
       "...              ...               ...        ...                ...   \n",
       "14035         1972.5          0.476553   6.846943           7.567698   \n",
       "14036         1700.0          0.468235   6.680855           7.438958   \n",
       "14037           26.0          5.153846   4.905275           3.001944   \n",
       "14038          916.5          0.805237   6.605298           6.821610   \n",
       "14039         2268.5          0.795239   7.498316           7.706572   \n",
       "\n",
       "       Log_Normalized_Value  Is_Normal_Group  Is_Normal_State  \n",
       "0                  0.000000             True             True  \n",
       "1                  0.000000             True             True  \n",
       "2                  0.000000             True             True  \n",
       "3                  0.000000             True             True  \n",
       "4                  0.000000             True             True  \n",
       "...                     ...              ...              ...  \n",
       "14035             -0.720755             True             True  \n",
       "14036             -0.758103             True             True  \n",
       "14037              1.903331             True             True  \n",
       "14038             -0.216312            False            False  \n",
       "14039             -0.208256             True             True  \n",
       "\n",
       "[14040 rows x 14 columns]"
      ]
     },
     "execution_count": 319,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_long_logNorm_base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {},
   "outputs": [],
   "source": [
    "reload(an)\n",
    "pVals_longNorm_sham_Group = an.calculate_p_values_vs_sham_Group(df_long_logNorm_sham, log_comp = True)\n",
    "pVals_longNorm_base_Group = an.calculate_p_values_vs_baseline_Group(df_long_logNorm_base, log_comp = True)\n",
    "pVals_longNorm_sham_State = an.calculate_p_values_vs_sham_State(df_long_logNorm_sham, log_comp = True)\n",
    "pVals_longNorm_base_State = an.calculate_p_values_vs_baseline_State(df_long_logNorm_base, log_comp = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     " skipping until confirmation"
    ]
   },
   "outputs": [],
   "source": [
    "from statsmodels.stats.multitest import multipletests\n",
    "\n",
    "# Adjusting p-values for the multiple tests:\n",
    "p_vals = pVals_longNorm_sham_Group['P_Value'].values\n",
    "adjusted_p_vals = multipletests(p_vals, alpha=0.05, method='bonferroni')\n",
    "\n",
    "\n",
    "adjusted_p_vals[1]\n",
    "# Add the adjusted p-values to the DataFrame\n",
    "#pVals_longNorm_sham_Group['Adjusted_P_Value'] = adjusted_p_vals[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "skipping until confirmation"
    ]
   },
   "outputs": [],
   "source": [
    "p_vals = pVals_longNorm_base_Group['P_Value'].values\n",
    "adjusted_p_vals = multipletests(p_vals, alpha=0.05, method='bonferroni')\n",
    "\n",
    "# Add the adjusted p-values to the DataFrame\n",
    "#pVals_longNorm_base_Group['Adjusted_P_Value'] = adjusted_p_vals[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "skipping until confirmation"
    ]
   },
   "outputs": [],
   "source": [
    "p_vals = pVals_longNorm_sham_State['P_Value'].values\n",
    "adjusted_p_vals = multipletests(p_vals, alpha=0.05, method='bonferroni')\n",
    "\n",
    "# Add the adjusted p-values to the DataFrame\n",
    "#pVals_longNorm_sham_State['Adjusted_P_Value'] = adjusted_p_vals[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": [
     "skipping until confirmation"
    ]
   },
   "outputs": [],
   "source": [
    "p_vals = pVals_longNorm_sham_State['P_Value'].values\n",
    "adjusted_p_vals = multipletests(p_vals, alpha=0.05, method='bonferroni')\n",
    "\n",
    "# Add the adjusted p-values to the DataFrame\n",
    "#pVals_longNorm_sham_State['Adjusted_P_Value'] = adjusted_p_vals[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Animal_ID</th>\n",
       "      <th>Time_Point</th>\n",
       "      <th>Group</th>\n",
       "      <th>Motif</th>\n",
       "      <th>P_Value</th>\n",
       "      <th>Stat_Value</th>\n",
       "      <th>Test_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>B1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>ABX</td>\n",
       "      <td>0</td>\n",
       "      <td>0.724564</td>\n",
       "      <td>-0.361513</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>ABX</td>\n",
       "      <td>1</td>\n",
       "      <td>0.658289</td>\n",
       "      <td>0.454526</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>B1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>ABX</td>\n",
       "      <td>2</td>\n",
       "      <td>0.679041</td>\n",
       "      <td>0.424986</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>B1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>ABX</td>\n",
       "      <td>3</td>\n",
       "      <td>0.028503</td>\n",
       "      <td>108.000000</td>\n",
       "      <td>Mann-Whitney</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>ABX</td>\n",
       "      <td>4</td>\n",
       "      <td>0.224695</td>\n",
       "      <td>1.286470</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>865</th>\n",
       "      <td>J1</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>25</td>\n",
       "      <td>0.523923</td>\n",
       "      <td>-0.658236</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>866</th>\n",
       "      <td>J1</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>26</td>\n",
       "      <td>0.598729</td>\n",
       "      <td>-0.541846</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>867</th>\n",
       "      <td>J1</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>27</td>\n",
       "      <td>0.028464</td>\n",
       "      <td>108.000000</td>\n",
       "      <td>Mann-Whitney</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>868</th>\n",
       "      <td>J1</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>28</td>\n",
       "      <td>0.093105</td>\n",
       "      <td>1.838587</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>869</th>\n",
       "      <td>J1</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>0.611405</td>\n",
       "      <td>-0.522913</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>870 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Animal_ID  Time_Point    Group  Motif   P_Value  Stat_Value     Test_Type\n",
       "0          B1  Baseline_1      ABX      0  0.724564   -0.361513        t-test\n",
       "1          B1  Baseline_1      ABX      1  0.658289    0.454526        t-test\n",
       "2          B1  Baseline_1      ABX      2  0.679041    0.424986        t-test\n",
       "3          B1  Baseline_1      ABX      3  0.028503  108.000000  Mann-Whitney\n",
       "4          B1  Baseline_1      ABX      4  0.224695    1.286470        t-test\n",
       "..        ...         ...      ...    ...       ...         ...           ...\n",
       "865        J1    Drug_Trt  Treated     25  0.523923   -0.658236        t-test\n",
       "866        J1    Drug_Trt  Treated     26  0.598729   -0.541846        t-test\n",
       "867        J1    Drug_Trt  Treated     27  0.028464  108.000000  Mann-Whitney\n",
       "868        J1    Drug_Trt  Treated     28  0.093105    1.838587        t-test\n",
       "869        J1    Drug_Trt  Treated     29  0.611405   -0.522913        t-test\n",
       "\n",
       "[870 rows x 7 columns]"
      ]
     },
     "execution_count": 374,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pVals_longNorm_sham_Group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Animal_ID</th>\n",
       "      <th>Time_Point</th>\n",
       "      <th>Group</th>\n",
       "      <th>Motif</th>\n",
       "      <th>P_Value</th>\n",
       "      <th>Stat_Value</th>\n",
       "      <th>Test_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A1</td>\n",
       "      <td>Week_02</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>0.168867</td>\n",
       "      <td>-1.472674</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1</td>\n",
       "      <td>Week_04</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>0.029836</td>\n",
       "      <td>-2.493752</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A1</td>\n",
       "      <td>Week_06</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000235</td>\n",
       "      <td>-5.347185</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A1</td>\n",
       "      <td>Week_08</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>0.001135</td>\n",
       "      <td>-4.360626</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A1</td>\n",
       "      <td>Week_11</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>Mann-Whitney</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>925</th>\n",
       "      <td>J1</td>\n",
       "      <td>Week_08</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>0.887676</td>\n",
       "      <td>-0.144557</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>926</th>\n",
       "      <td>J1</td>\n",
       "      <td>Week_11</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>0.134296</td>\n",
       "      <td>1.616421</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927</th>\n",
       "      <td>J1</td>\n",
       "      <td>Week_13</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>0.570718</td>\n",
       "      <td>0.584438</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>928</th>\n",
       "      <td>J1</td>\n",
       "      <td>Week_15</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>0.216773</td>\n",
       "      <td>1.310366</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>929</th>\n",
       "      <td>J1</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>0.319233</td>\n",
       "      <td>1.043217</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>930 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Animal_ID Time_Point    Group  Motif   P_Value  Stat_Value     Test_Type\n",
       "0          A1    Week_02     Sham      0  0.168867   -1.472674        t-test\n",
       "1          A1    Week_04     Sham      0  0.029836   -2.493752        t-test\n",
       "2          A1    Week_06     Sham      0  0.000235   -5.347185        t-test\n",
       "3          A1    Week_08     Sham      0  0.001135   -4.360626        t-test\n",
       "4          A1    Week_11     Sham      0  0.000002   24.000000  Mann-Whitney\n",
       "..        ...        ...      ...    ...       ...         ...           ...\n",
       "925        J1    Week_08  Treated     29  0.887676   -0.144557        t-test\n",
       "926        J1    Week_11  Treated     29  0.134296    1.616421        t-test\n",
       "927        J1    Week_13  Treated     29  0.570718    0.584438        t-test\n",
       "928        J1    Week_15  Treated     29  0.216773    1.310366        t-test\n",
       "929        J1   Drug_Trt  Treated     29  0.319233    1.043217        t-test\n",
       "\n",
       "[930 rows x 7 columns]"
      ]
     },
     "execution_count": 360,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pVals_longNorm_base_Group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Animal_ID</th>\n",
       "      <th>Time_Point</th>\n",
       "      <th>Treatment_State</th>\n",
       "      <th>Motif</th>\n",
       "      <th>P_Value</th>\n",
       "      <th>Stat_Value</th>\n",
       "      <th>Test_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>B1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>ABX</td>\n",
       "      <td>0</td>\n",
       "      <td>0.724564</td>\n",
       "      <td>-0.361513</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>B1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>ABX</td>\n",
       "      <td>1</td>\n",
       "      <td>0.658289</td>\n",
       "      <td>0.454526</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>B1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>ABX</td>\n",
       "      <td>2</td>\n",
       "      <td>0.679041</td>\n",
       "      <td>0.424986</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>B1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>ABX</td>\n",
       "      <td>3</td>\n",
       "      <td>0.028503</td>\n",
       "      <td>108.000000</td>\n",
       "      <td>Mann-Whitney</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>B1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>ABX</td>\n",
       "      <td>4</td>\n",
       "      <td>0.224695</td>\n",
       "      <td>1.286470</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>685</th>\n",
       "      <td>J1</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>25</td>\n",
       "      <td>0.523923</td>\n",
       "      <td>-0.658236</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>686</th>\n",
       "      <td>J1</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>26</td>\n",
       "      <td>0.598729</td>\n",
       "      <td>-0.541846</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>687</th>\n",
       "      <td>J1</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>27</td>\n",
       "      <td>0.028464</td>\n",
       "      <td>108.000000</td>\n",
       "      <td>Mann-Whitney</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>688</th>\n",
       "      <td>J1</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>28</td>\n",
       "      <td>0.093105</td>\n",
       "      <td>1.838587</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>689</th>\n",
       "      <td>J1</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>0.611405</td>\n",
       "      <td>-0.522913</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>690 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Animal_ID  Time_Point Treatment_State  Motif   P_Value  Stat_Value  \\\n",
       "0          B1  Baseline_1             ABX      0  0.724564   -0.361513   \n",
       "1          B1  Baseline_1             ABX      1  0.658289    0.454526   \n",
       "2          B1  Baseline_1             ABX      2  0.679041    0.424986   \n",
       "3          B1  Baseline_1             ABX      3  0.028503  108.000000   \n",
       "4          B1  Baseline_1             ABX      4  0.224695    1.286470   \n",
       "..        ...         ...             ...    ...       ...         ...   \n",
       "685        J1    Drug_Trt         Treated     25  0.523923   -0.658236   \n",
       "686        J1    Drug_Trt         Treated     26  0.598729   -0.541846   \n",
       "687        J1    Drug_Trt         Treated     27  0.028464  108.000000   \n",
       "688        J1    Drug_Trt         Treated     28  0.093105    1.838587   \n",
       "689        J1    Drug_Trt         Treated     29  0.611405   -0.522913   \n",
       "\n",
       "        Test_Type  \n",
       "0          t-test  \n",
       "1          t-test  \n",
       "2          t-test  \n",
       "3    Mann-Whitney  \n",
       "4          t-test  \n",
       "..            ...  \n",
       "685        t-test  \n",
       "686        t-test  \n",
       "687  Mann-Whitney  \n",
       "688        t-test  \n",
       "689        t-test  \n",
       "\n",
       "[690 rows x 7 columns]"
      ]
     },
     "execution_count": 361,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pVals_longNorm_sham_State"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Animal_ID</th>\n",
       "      <th>Time_Point</th>\n",
       "      <th>Treatment_State</th>\n",
       "      <th>Motif</th>\n",
       "      <th>P_Value</th>\n",
       "      <th>Stat_Value</th>\n",
       "      <th>Test_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A1</td>\n",
       "      <td>Week_02</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>1.688665e-01</td>\n",
       "      <td>-1.472674</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1</td>\n",
       "      <td>Week_04</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>2.983584e-02</td>\n",
       "      <td>-2.493752</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A1</td>\n",
       "      <td>Week_06</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>2.348192e-04</td>\n",
       "      <td>-5.347185</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A1</td>\n",
       "      <td>Week_08</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>1.135112e-03</td>\n",
       "      <td>-4.360626</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A1</td>\n",
       "      <td>Week_11</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>1.757720e-06</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>Mann-Whitney</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>805</th>\n",
       "      <td>J1</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>28</td>\n",
       "      <td>2.846117e-14</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Mann-Whitney</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>806</th>\n",
       "      <td>J1</td>\n",
       "      <td>Week_11</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>1.342959e-01</td>\n",
       "      <td>1.616421</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>807</th>\n",
       "      <td>J1</td>\n",
       "      <td>Week_13</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>5.707177e-01</td>\n",
       "      <td>0.584438</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>808</th>\n",
       "      <td>J1</td>\n",
       "      <td>Week_15</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>2.167733e-01</td>\n",
       "      <td>1.310366</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>809</th>\n",
       "      <td>J1</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>3.192328e-01</td>\n",
       "      <td>1.043217</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>810 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Animal_ID Time_Point Treatment_State  Motif       P_Value  Stat_Value  \\\n",
       "0          A1    Week_02            Sham      0  1.688665e-01   -1.472674   \n",
       "1          A1    Week_04            Sham      0  2.983584e-02   -2.493752   \n",
       "2          A1    Week_06            Sham      0  2.348192e-04   -5.347185   \n",
       "3          A1    Week_08            Sham      0  1.135112e-03   -4.360626   \n",
       "4          A1    Week_11            Sham      0  1.757720e-06   24.000000   \n",
       "..        ...        ...             ...    ...           ...         ...   \n",
       "805        J1   Drug_Trt         Treated     28  2.846117e-14    0.000000   \n",
       "806        J1    Week_11         Treated     29  1.342959e-01    1.616421   \n",
       "807        J1    Week_13         Treated     29  5.707177e-01    0.584438   \n",
       "808        J1    Week_15         Treated     29  2.167733e-01    1.310366   \n",
       "809        J1   Drug_Trt         Treated     29  3.192328e-01    1.043217   \n",
       "\n",
       "        Test_Type  \n",
       "0          t-test  \n",
       "1          t-test  \n",
       "2          t-test  \n",
       "3          t-test  \n",
       "4    Mann-Whitney  \n",
       "..            ...  \n",
       "805  Mann-Whitney  \n",
       "806        t-test  \n",
       "807        t-test  \n",
       "808        t-test  \n",
       "809        t-test  \n",
       "\n",
       "[810 rows x 7 columns]"
      ]
     },
     "execution_count": 362,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pVals_longNorm_base_State"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Animal_ID</th>\n",
       "      <th>Time_Point</th>\n",
       "      <th>Group</th>\n",
       "      <th>Treatment_State</th>\n",
       "      <th>Motif</th>\n",
       "      <th>value</th>\n",
       "      <th>Cluster</th>\n",
       "      <th>Baseline_Mean</th>\n",
       "      <th>Normalized_Value</th>\n",
       "      <th>log_value</th>\n",
       "      <th>Sham_Log_Mean</th>\n",
       "      <th>Log_Normalized_Value</th>\n",
       "      <th>Is_Normal_Group</th>\n",
       "      <th>Is_Normal_State</th>\n",
       "      <th>P_Value_Group</th>\n",
       "      <th>Stat_Value_Group</th>\n",
       "      <th>Test_Type_Group</th>\n",
       "      <th>P_Value_State</th>\n",
       "      <th>Stat_Value_State</th>\n",
       "      <th>Test_Type_State</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>424</td>\n",
       "      <td>5</td>\n",
       "      <td>436.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.052089</td>\n",
       "      <td>6.824923</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>1</td>\n",
       "      <td>105</td>\n",
       "      <td>1</td>\n",
       "      <td>73.5</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.663439</td>\n",
       "      <td>5.896740</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>2</td>\n",
       "      <td>764</td>\n",
       "      <td>3</td>\n",
       "      <td>1005.5</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.639876</td>\n",
       "      <td>6.600722</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>3</td>\n",
       "      <td>199</td>\n",
       "      <td>1</td>\n",
       "      <td>161.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.298317</td>\n",
       "      <td>5.864924</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>4</td>\n",
       "      <td>696</td>\n",
       "      <td>3</td>\n",
       "      <td>657.5</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.546785</td>\n",
       "      <td>6.420746</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14035</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>25</td>\n",
       "      <td>940</td>\n",
       "      <td>7</td>\n",
       "      <td>1972.5</td>\n",
       "      <td>0.476553</td>\n",
       "      <td>6.846943</td>\n",
       "      <td>7.143300</td>\n",
       "      <td>-0.296357</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.523923</td>\n",
       "      <td>-0.658236</td>\n",
       "      <td>t-test</td>\n",
       "      <td>0.523923</td>\n",
       "      <td>-0.658236</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14036</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>26</td>\n",
       "      <td>796</td>\n",
       "      <td>10</td>\n",
       "      <td>1700.0</td>\n",
       "      <td>0.468235</td>\n",
       "      <td>6.680855</td>\n",
       "      <td>6.565693</td>\n",
       "      <td>0.115162</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.598729</td>\n",
       "      <td>-0.541846</td>\n",
       "      <td>t-test</td>\n",
       "      <td>0.598729</td>\n",
       "      <td>-0.541846</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14037</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>27</td>\n",
       "      <td>134</td>\n",
       "      <td>3</td>\n",
       "      <td>26.0</td>\n",
       "      <td>5.153846</td>\n",
       "      <td>4.905275</td>\n",
       "      <td>3.710625</td>\n",
       "      <td>1.194649</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.028464</td>\n",
       "      <td>108.000000</td>\n",
       "      <td>Mann-Whitney</td>\n",
       "      <td>0.028464</td>\n",
       "      <td>108.000000</td>\n",
       "      <td>Mann-Whitney</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14038</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>28</td>\n",
       "      <td>738</td>\n",
       "      <td>9</td>\n",
       "      <td>916.5</td>\n",
       "      <td>0.805237</td>\n",
       "      <td>6.605298</td>\n",
       "      <td>6.536103</td>\n",
       "      <td>0.069195</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.093105</td>\n",
       "      <td>1.838587</td>\n",
       "      <td>t-test</td>\n",
       "      <td>0.093105</td>\n",
       "      <td>1.838587</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14039</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>1804</td>\n",
       "      <td>8</td>\n",
       "      <td>2268.5</td>\n",
       "      <td>0.795239</td>\n",
       "      <td>7.498316</td>\n",
       "      <td>7.276425</td>\n",
       "      <td>0.221891</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.611405</td>\n",
       "      <td>-0.522913</td>\n",
       "      <td>t-test</td>\n",
       "      <td>0.611405</td>\n",
       "      <td>-0.522913</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14040 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Animal_ID  Time_Point    Group Treatment_State  Motif  value  Cluster  \\\n",
       "0            A1  Baseline_1     Sham            Sham      0    424        5   \n",
       "1            A1  Baseline_1     Sham            Sham      1    105        1   \n",
       "2            A1  Baseline_1     Sham            Sham      2    764        3   \n",
       "3            A1  Baseline_1     Sham            Sham      3    199        1   \n",
       "4            A1  Baseline_1     Sham            Sham      4    696        3   \n",
       "...         ...         ...      ...             ...    ...    ...      ...   \n",
       "14035        Y2    Drug_Trt  Treated         Treated     25    940        7   \n",
       "14036        Y2    Drug_Trt  Treated         Treated     26    796       10   \n",
       "14037        Y2    Drug_Trt  Treated         Treated     27    134        3   \n",
       "14038        Y2    Drug_Trt  Treated         Treated     28    738        9   \n",
       "14039        Y2    Drug_Trt  Treated         Treated     29   1804        8   \n",
       "\n",
       "       Baseline_Mean  Normalized_Value  log_value  Sham_Log_Mean  \\\n",
       "0              436.0          1.000000   6.052089       6.824923   \n",
       "1               73.5          1.000000   4.663439       5.896740   \n",
       "2             1005.5          1.000000   6.639876       6.600722   \n",
       "3              161.0          1.000000   5.298317       5.864924   \n",
       "4              657.5          1.000000   6.546785       6.420746   \n",
       "...              ...               ...        ...            ...   \n",
       "14035         1972.5          0.476553   6.846943       7.143300   \n",
       "14036         1700.0          0.468235   6.680855       6.565693   \n",
       "14037           26.0          5.153846   4.905275       3.710625   \n",
       "14038          916.5          0.805237   6.605298       6.536103   \n",
       "14039         2268.5          0.795239   7.498316       7.276425   \n",
       "\n",
       "       Log_Normalized_Value  Is_Normal_Group  Is_Normal_State  P_Value_Group  \\\n",
       "0                  0.000000             True             True            NaN   \n",
       "1                  0.000000             True             True            NaN   \n",
       "2                  0.000000             True             True            NaN   \n",
       "3                  0.000000             True             True            NaN   \n",
       "4                  0.000000             True             True            NaN   \n",
       "...                     ...              ...              ...            ...   \n",
       "14035             -0.296357             True             True       0.523923   \n",
       "14036              0.115162             True             True       0.598729   \n",
       "14037              1.194649            False            False       0.028464   \n",
       "14038              0.069195             True             True       0.093105   \n",
       "14039              0.221891             True             True       0.611405   \n",
       "\n",
       "       Stat_Value_Group Test_Type_Group  P_Value_State  Stat_Value_State  \\\n",
       "0                   NaN             NaN            NaN               NaN   \n",
       "1                   NaN             NaN            NaN               NaN   \n",
       "2                   NaN             NaN            NaN               NaN   \n",
       "3                   NaN             NaN            NaN               NaN   \n",
       "4                   NaN             NaN            NaN               NaN   \n",
       "...                 ...             ...            ...               ...   \n",
       "14035         -0.658236          t-test       0.523923         -0.658236   \n",
       "14036         -0.541846          t-test       0.598729         -0.541846   \n",
       "14037        108.000000    Mann-Whitney       0.028464        108.000000   \n",
       "14038          1.838587          t-test       0.093105          1.838587   \n",
       "14039         -0.522913          t-test       0.611405         -0.522913   \n",
       "\n",
       "      Test_Type_State  \n",
       "0                 NaN  \n",
       "1                 NaN  \n",
       "2                 NaN  \n",
       "3                 NaN  \n",
       "4                 NaN  \n",
       "...               ...  \n",
       "14035          t-test  \n",
       "14036          t-test  \n",
       "14037    Mann-Whitney  \n",
       "14038          t-test  \n",
       "14039          t-test  \n",
       "\n",
       "[14040 rows x 20 columns]"
      ]
     },
     "execution_count": 364,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Merge the group p-values\n",
    "df_long_logNorm_sham = df_long_logNorm_sham.merge(\n",
    "    pVals_longNorm_sham_Group[['Time_Point', 'Group', 'Motif', 'P_Value', 'Stat_Value', 'Test_Type']],\n",
    "    on=['Time_Point', 'Group', 'Motif'],\n",
    "    how='left',\n",
    "    suffixes=('', '_group')\n",
    ")\n",
    "\n",
    "# Merge the state p-values\n",
    "df_long_logNorm_sham = df_long_logNorm_sham.merge(\n",
    "    pVals_longNorm_sham_State[['Time_Point', 'Treatment_State', 'Motif', 'P_Value', 'Stat_Value', 'Test_Type']],\n",
    "    on=['Time_Point', 'Treatment_State', 'Motif'],\n",
    "    how='left',\n",
    "    suffixes=('', '_state')\n",
    ")\n",
    "\n",
    "# Rename the columns to make them differentiable\n",
    "df_long_logNorm_sham.rename(columns={\n",
    "    'P_Value': 'P_Value_Group',\n",
    "    'Stat_Value': 'Stat_Value_Group',\n",
    "    'Test_Type': 'Test_Type_Group',\n",
    "    'P_Value_state': 'P_Value_State',\n",
    "    'Stat_Value_state': 'Stat_Value_State',\n",
    "    'Test_Type_state': 'Test_Type_State'\n",
    "}, inplace=True)\n",
    "\n",
    "df_long_logNorm_sham"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Animal_ID</th>\n",
       "      <th>Time_Point</th>\n",
       "      <th>Group</th>\n",
       "      <th>Treatment_State</th>\n",
       "      <th>Motif</th>\n",
       "      <th>value</th>\n",
       "      <th>Cluster</th>\n",
       "      <th>Baseline_Mean</th>\n",
       "      <th>Normalized_Value</th>\n",
       "      <th>log_value</th>\n",
       "      <th>Baseline_Log_Mean</th>\n",
       "      <th>Log_Normalized_Value</th>\n",
       "      <th>Is_Normal_Group</th>\n",
       "      <th>Is_Normal_State</th>\n",
       "      <th>P_Value_Group</th>\n",
       "      <th>Stat_Value_Group</th>\n",
       "      <th>Test_Type_Group</th>\n",
       "      <th>P_Value_State</th>\n",
       "      <th>Stat_Value_State</th>\n",
       "      <th>Test_Type_State</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>424</td>\n",
       "      <td>5</td>\n",
       "      <td>436.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.052089</td>\n",
       "      <td>6.079556</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>1</td>\n",
       "      <td>105</td>\n",
       "      <td>1</td>\n",
       "      <td>73.5</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.663439</td>\n",
       "      <td>4.212320</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>2</td>\n",
       "      <td>764</td>\n",
       "      <td>3</td>\n",
       "      <td>1005.5</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.639876</td>\n",
       "      <td>6.884587</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>3</td>\n",
       "      <td>199</td>\n",
       "      <td>1</td>\n",
       "      <td>161.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.298317</td>\n",
       "      <td>5.059299</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>4</td>\n",
       "      <td>696</td>\n",
       "      <td>3</td>\n",
       "      <td>657.5</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.546785</td>\n",
       "      <td>6.488252</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14035</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>25</td>\n",
       "      <td>940</td>\n",
       "      <td>7</td>\n",
       "      <td>1972.5</td>\n",
       "      <td>0.476553</td>\n",
       "      <td>6.846943</td>\n",
       "      <td>7.567698</td>\n",
       "      <td>-0.720755</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>7.478622e-01</td>\n",
       "      <td>0.329642</td>\n",
       "      <td>t-test</td>\n",
       "      <td>7.478622e-01</td>\n",
       "      <td>0.329642</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14036</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>26</td>\n",
       "      <td>796</td>\n",
       "      <td>10</td>\n",
       "      <td>1700.0</td>\n",
       "      <td>0.468235</td>\n",
       "      <td>6.680855</td>\n",
       "      <td>7.438958</td>\n",
       "      <td>-0.758103</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>7.269034e-04</td>\n",
       "      <td>-4.631269</td>\n",
       "      <td>t-test</td>\n",
       "      <td>7.269034e-04</td>\n",
       "      <td>-4.631269</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14037</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>27</td>\n",
       "      <td>134</td>\n",
       "      <td>3</td>\n",
       "      <td>26.0</td>\n",
       "      <td>5.153846</td>\n",
       "      <td>4.905275</td>\n",
       "      <td>3.001944</td>\n",
       "      <td>1.903331</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>2.200776e-01</td>\n",
       "      <td>-1.300313</td>\n",
       "      <td>t-test</td>\n",
       "      <td>2.200776e-01</td>\n",
       "      <td>-1.300313</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14038</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>28</td>\n",
       "      <td>738</td>\n",
       "      <td>9</td>\n",
       "      <td>916.5</td>\n",
       "      <td>0.805237</td>\n",
       "      <td>6.605298</td>\n",
       "      <td>6.821610</td>\n",
       "      <td>-0.216312</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>9.504392e-09</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Mann-Whitney</td>\n",
       "      <td>2.846117e-14</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>Mann-Whitney</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14039</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>1804</td>\n",
       "      <td>8</td>\n",
       "      <td>2268.5</td>\n",
       "      <td>0.795239</td>\n",
       "      <td>7.498316</td>\n",
       "      <td>7.706572</td>\n",
       "      <td>-0.208256</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>3.192328e-01</td>\n",
       "      <td>1.043217</td>\n",
       "      <td>t-test</td>\n",
       "      <td>3.192328e-01</td>\n",
       "      <td>1.043217</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14040 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Animal_ID  Time_Point    Group Treatment_State  Motif  value  Cluster  \\\n",
       "0            A1  Baseline_1     Sham            Sham      0    424        5   \n",
       "1            A1  Baseline_1     Sham            Sham      1    105        1   \n",
       "2            A1  Baseline_1     Sham            Sham      2    764        3   \n",
       "3            A1  Baseline_1     Sham            Sham      3    199        1   \n",
       "4            A1  Baseline_1     Sham            Sham      4    696        3   \n",
       "...         ...         ...      ...             ...    ...    ...      ...   \n",
       "14035        Y2    Drug_Trt  Treated         Treated     25    940        7   \n",
       "14036        Y2    Drug_Trt  Treated         Treated     26    796       10   \n",
       "14037        Y2    Drug_Trt  Treated         Treated     27    134        3   \n",
       "14038        Y2    Drug_Trt  Treated         Treated     28    738        9   \n",
       "14039        Y2    Drug_Trt  Treated         Treated     29   1804        8   \n",
       "\n",
       "       Baseline_Mean  Normalized_Value  log_value  Baseline_Log_Mean  \\\n",
       "0              436.0          1.000000   6.052089           6.079556   \n",
       "1               73.5          1.000000   4.663439           4.212320   \n",
       "2             1005.5          1.000000   6.639876           6.884587   \n",
       "3              161.0          1.000000   5.298317           5.059299   \n",
       "4              657.5          1.000000   6.546785           6.488252   \n",
       "...              ...               ...        ...                ...   \n",
       "14035         1972.5          0.476553   6.846943           7.567698   \n",
       "14036         1700.0          0.468235   6.680855           7.438958   \n",
       "14037           26.0          5.153846   4.905275           3.001944   \n",
       "14038          916.5          0.805237   6.605298           6.821610   \n",
       "14039         2268.5          0.795239   7.498316           7.706572   \n",
       "\n",
       "       Log_Normalized_Value  Is_Normal_Group  Is_Normal_State  P_Value_Group  \\\n",
       "0                  0.000000             True             True            NaN   \n",
       "1                  0.000000             True             True            NaN   \n",
       "2                  0.000000             True             True            NaN   \n",
       "3                  0.000000             True             True            NaN   \n",
       "4                  0.000000             True             True            NaN   \n",
       "...                     ...              ...              ...            ...   \n",
       "14035             -0.720755             True             True   7.478622e-01   \n",
       "14036             -0.758103             True             True   7.269034e-04   \n",
       "14037              1.903331             True             True   2.200776e-01   \n",
       "14038             -0.216312            False            False   9.504392e-09   \n",
       "14039             -0.208256             True             True   3.192328e-01   \n",
       "\n",
       "       Stat_Value_Group Test_Type_Group  P_Value_State  Stat_Value_State  \\\n",
       "0                   NaN             NaN            NaN               NaN   \n",
       "1                   NaN             NaN            NaN               NaN   \n",
       "2                   NaN             NaN            NaN               NaN   \n",
       "3                   NaN             NaN            NaN               NaN   \n",
       "4                   NaN             NaN            NaN               NaN   \n",
       "...                 ...             ...            ...               ...   \n",
       "14035          0.329642          t-test   7.478622e-01          0.329642   \n",
       "14036         -4.631269          t-test   7.269034e-04         -4.631269   \n",
       "14037         -1.300313          t-test   2.200776e-01         -1.300313   \n",
       "14038          0.000000    Mann-Whitney   2.846117e-14          0.000000   \n",
       "14039          1.043217          t-test   3.192328e-01          1.043217   \n",
       "\n",
       "      Test_Type_State  \n",
       "0                 NaN  \n",
       "1                 NaN  \n",
       "2                 NaN  \n",
       "3                 NaN  \n",
       "4                 NaN  \n",
       "...               ...  \n",
       "14035          t-test  \n",
       "14036          t-test  \n",
       "14037          t-test  \n",
       "14038    Mann-Whitney  \n",
       "14039          t-test  \n",
       "\n",
       "[14040 rows x 20 columns]"
      ]
     },
     "execution_count": 365,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Merge the group p-values\n",
    "df_long_logNorm_base = df_long_logNorm_base.merge(\n",
    "    pVals_longNorm_base_Group[['Time_Point', 'Group', 'Motif', 'P_Value', 'Stat_Value', 'Test_Type']],\n",
    "    on=['Time_Point', 'Group', 'Motif'],\n",
    "    how='left',\n",
    "    suffixes=('', '_group')\n",
    ")\n",
    "\n",
    "# Merge the state p-values\n",
    "df_long_logNorm_base = df_long_logNorm_base.merge(\n",
    "    pVals_longNorm_base_State[['Time_Point', 'Treatment_State', 'Motif', 'P_Value', 'Stat_Value', 'Test_Type']],\n",
    "    on=['Time_Point', 'Treatment_State', 'Motif'],\n",
    "    how='left',\n",
    "    suffixes=('', '_state')\n",
    ")\n",
    "\n",
    "# Rename the columns to make them differentiable\n",
    "df_long_logNorm_base.rename(columns={\n",
    "    'P_Value': 'P_Value_Group',\n",
    "    'Stat_Value': 'Stat_Value_Group',\n",
    "    'Test_Type': 'Test_Type_Group',\n",
    "    'P_Value_state': 'P_Value_State',\n",
    "    'Stat_Value_state': 'Stat_Value_State',\n",
    "    'Test_Type_state': 'Test_Type_State'\n",
    "}, inplace=True)\n",
    "\n",
    "df_long_logNorm_base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Count the number of significant motifs for each Group at each Time_Point when comparing to sham\n",
    "# and list the significant motifs\n",
    "significant_counts_vsSham_Group = pVals_longNorm_sham_Group[pVals_longNorm_sham_Group['P_Value'] <= 0.05].groupby(['Time_Point', 'Group'])['Motif'].apply(list).reset_index(name='Significant_Motifs_Group')\n",
    "significant_counts_vsSham_Group['Significant_Motif_Count_Group'] = significant_counts_vsSham_Group['Significant_Motifs_Group'].apply(len)\n",
    "\n",
    "significant_counts_vsBase_Group = pVals_longNorm_base_Group[pVals_longNorm_base_Group['P_Value'] <= 0.05].groupby(['Time_Point', 'Group'])['Motif'].apply(list).reset_index(name='Significant_Motifs_Group')\n",
    "significant_counts_vsBase_Group['Significant_Motif_Count_Group'] = significant_counts_vsBase_Group['Significant_Motifs_Group'].apply(len)\n",
    "\n",
    "significant_counts_vsSham_State = pVals_longNorm_sham_State[pVals_longNorm_sham_State['P_Value'] <= 0.05].groupby(['Time_Point', 'Treatment_State'])['Motif'].apply(list).reset_index(name='Significant_Motifs_State')\n",
    "significant_counts_vsSham_State['Significant_Motif_Count_State'] = significant_counts_vsSham_State['Significant_Motifs_State'].apply(len)\n",
    "\n",
    "significant_counts_vsBase_State = pVals_longNorm_base_State[pVals_longNorm_base_State['P_Value'] <= 0.05].groupby(['Time_Point', 'Treatment_State'])['Motif'].apply(list).reset_index(name='Significant_Motifs_State')\n",
    "significant_counts_vsBase_State['Significant_Motif_Count_State'] = significant_counts_vsBase_State['Significant_Motifs_State'].apply(len)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 500,
   "metadata": {},
   "outputs": [],
   "source": [
    "significant_counts_vsSham_Group\n",
    "significant_counts_vsSham_Group.to_csv('kmeans-30_significant_counts_vsSham_Group.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 501,
   "metadata": {},
   "outputs": [],
   "source": [
    "significant_counts_vsSham_State\n",
    "significant_counts_vsSham_State.to_csv('kmeans-30_significant_counts_vsSham_State.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 503,
   "metadata": {},
   "outputs": [],
   "source": [
    "significant_counts_vsBase_Group\n",
    "significant_counts_vsBase_Group.to_csv('kmeans-30_significant_counts_vsBase_Group.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 504,
   "metadata": {},
   "outputs": [],
   "source": [
    "significant_counts_vsBase_State\n",
    "significant_counts_vsBase_State.to_csv('kmeans-30_significant_counts_vsBase_State.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 485,
   "metadata": {},
   "outputs": [],
   "source": [
    "reload(AlPf)\n",
    "AlPf.plot_motifs_all_time_points(df_long_logNorm_sham, comparison = 'Group')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 506,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/Users/adanredwine/VAME/ALR_VAME.ipynb Cell 48\u001b[0m line \u001b[0;36m2\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/adanredwine/VAME/ALR_VAME.ipynb#Y141sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m reload(AlPf)\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/adanredwine/VAME/ALR_VAME.ipynb#Y141sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m AlPf\u001b[39m.\u001b[39;49mplot_motifs_all_time_points(df_long_logNorm_sham, comparison \u001b[39m=\u001b[39;49m \u001b[39m'\u001b[39;49m\u001b[39mTreatment_State\u001b[39;49m\u001b[39m'\u001b[39;49m)\n",
      "File \u001b[0;32m~/VAME/vame/custom/ALR_plottingFunctions.py:1084\u001b[0m, in \u001b[0;36mplot_motifs_all_time_points\u001b[0;34m(df, comparison, significance_level)\u001b[0m\n\u001b[1;32m   1073\u001b[0m plt\u001b[39m.\u001b[39mtight_layout()\n\u001b[1;32m   1075\u001b[0m plt\u001b[39m.\u001b[39msubplots_adjust(\n\u001b[1;32m   1076\u001b[0m      bottom\u001b[39m=\u001b[39m\u001b[39m0.035\u001b[39m, \u001b[39m# Increase the bottom padding to 0.2 (20% of the figure height)\u001b[39;00m\n\u001b[1;32m   1077\u001b[0m      top\u001b[39m=\u001b[39m\u001b[39m0.975\u001b[39m, \u001b[39m# Adjust the top padding\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1081\u001b[0m \u001b[39m#     wspace=0.0 # Adjust the horizontal space between subplots\u001b[39;00m\n\u001b[1;32m   1082\u001b[0m      )  \n\u001b[0;32m-> 1084\u001b[0m plt\u001b[39m.\u001b[39;49mshow()\n",
      "File \u001b[0;32m~/miniconda3/envs/VAME-2/lib/python3.11/site-packages/matplotlib/pyplot.py:527\u001b[0m, in \u001b[0;36mshow\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    483\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    484\u001b[0m \u001b[39mDisplay all open figures.\u001b[39;00m\n\u001b[1;32m    485\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    524\u001b[0m \u001b[39mexplicitly there.\u001b[39;00m\n\u001b[1;32m    525\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    526\u001b[0m _warn_if_gui_out_of_main_thread()\n\u001b[0;32m--> 527\u001b[0m \u001b[39mreturn\u001b[39;00m _get_backend_mod()\u001b[39m.\u001b[39;49mshow(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/miniconda3/envs/VAME-2/lib/python3.11/site-packages/matplotlib/backend_bases.py:3442\u001b[0m, in \u001b[0;36m_Backend.show\u001b[0;34m(cls, block)\u001b[0m\n\u001b[1;32m   3440\u001b[0m     block \u001b[39m=\u001b[39m \u001b[39mnot\u001b[39;00m ipython_pylab \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m is_interactive()\n\u001b[1;32m   3441\u001b[0m \u001b[39mif\u001b[39;00m block:\n\u001b[0;32m-> 3442\u001b[0m     \u001b[39mcls\u001b[39;49m\u001b[39m.\u001b[39;49mmainloop()\n",
      "File \u001b[0;32m~/miniconda3/envs/VAME-2/lib/python3.11/site-packages/matplotlib/backends/backend_qt.py:593\u001b[0m, in \u001b[0;36mFigureManagerQT.start_main_loop\u001b[0;34m(cls)\u001b[0m\n\u001b[1;32m    591\u001b[0m qapp \u001b[39m=\u001b[39m QtWidgets\u001b[39m.\u001b[39mQApplication\u001b[39m.\u001b[39minstance()\n\u001b[1;32m    592\u001b[0m \u001b[39mif\u001b[39;00m qapp:\n\u001b[0;32m--> 593\u001b[0m     \u001b[39mwith\u001b[39;49;00m _maybe_allow_interrupt(qapp):\n\u001b[1;32m    594\u001b[0m         qt_compat\u001b[39m.\u001b[39;49m_exec(qapp)\n",
      "File \u001b[0;32m~/miniconda3/envs/VAME-2/lib/python3.11/contextlib.py:144\u001b[0m, in \u001b[0;36m_GeneratorContextManager.__exit__\u001b[0;34m(self, typ, value, traceback)\u001b[0m\n\u001b[1;32m    142\u001b[0m \u001b[39mif\u001b[39;00m typ \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m    143\u001b[0m     \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 144\u001b[0m         \u001b[39mnext\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mgen)\n\u001b[1;32m    145\u001b[0m     \u001b[39mexcept\u001b[39;00m \u001b[39mStopIteration\u001b[39;00m:\n\u001b[1;32m    146\u001b[0m         \u001b[39mreturn\u001b[39;00m \u001b[39mFalse\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/VAME-2/lib/python3.11/site-packages/matplotlib/backends/qt_compat.py:230\u001b[0m, in \u001b[0;36m_maybe_allow_interrupt\u001b[0;34m(qapp)\u001b[0m\n\u001b[1;32m    228\u001b[0m signal\u001b[39m.\u001b[39msignal(signal\u001b[39m.\u001b[39mSIGINT, old_sigint_handler)\n\u001b[1;32m    229\u001b[0m \u001b[39mif\u001b[39;00m handler_args \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m--> 230\u001b[0m     old_sigint_handler(\u001b[39m*\u001b[39;49mhandler_args)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "reload(AlPf)\n",
    "AlPf.plot_motifs_all_time_points(df_long_logNorm_sham, comparison = 'Treatment_State')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 401,
   "metadata": {},
   "outputs": [],
   "source": [
    "reload(AlPf)\n",
    "AlPf.plot_motifs4(df_long_logNorm_sham, comparison = 'Group')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "metadata": {},
   "outputs": [],
   "source": [
    "reload(AlPf)\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Baseline_1\", comparison = 'Group')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Baseline_2\", comparison = 'Group')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Week_02\", comparison = 'Group')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Week_04\", comparison = 'Group')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Week_06\", comparison = 'Group')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Week_08\", comparison = 'Group')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Week_11\", comparison = 'Group')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Week_13\", comparison = 'Group')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Week_15\", comparison = 'Group')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Drug_Trt\", comparison = 'Group')\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Baseline_1\", comparison = 'Treatment_State')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Baseline_2\", comparison = 'Treatment_State')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Week_02\", comparison = 'Treatment_State')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Week_04\", comparison = 'Treatment_State')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Week_06\", comparison = 'Treatment_State')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Week_08\", comparison = 'Treatment_State')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Week_11\", comparison = 'Treatment_State')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Week_13\", comparison = 'Treatment_State')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Week_15\", comparison = 'Treatment_State')\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Drug_Trt\", comparison = 'Treatment_State')\n",
    "\n",
    "\n",
    "\n",
    "AlPf.plot_motifs3(df_long_logNorm_sham, time_point=\"Week_08\", comparison = 'Group')\n",
    "\n",
    "AlPf.plot_motifs3(df_long_logNorm_base, time_point=\"Week_08\", comparison = 'Group')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "metadata": {},
   "outputs": [],
   "source": [
    "AlPf.plot_motifs2(df_long_logNorm_base, time_point=\"Week_15\", comparison = 'Group')\n",
    "AlPf.plot_motifs2(df_long_logNorm_base, time_point=\"Week_15\", comparison = 'Treatment_State')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Animal_ID</th>\n",
       "      <th>Time_Point</th>\n",
       "      <th>Group</th>\n",
       "      <th>Treatment_State</th>\n",
       "      <th>Motif</th>\n",
       "      <th>value</th>\n",
       "      <th>Cluster</th>\n",
       "      <th>Baseline_Mean</th>\n",
       "      <th>Normalized_Value</th>\n",
       "      <th>log_value</th>\n",
       "      <th>Sham_Log_Mean</th>\n",
       "      <th>Log_Normalized_Value</th>\n",
       "      <th>Is_Normal_Group</th>\n",
       "      <th>Is_Normal_State</th>\n",
       "      <th>P_Value_Group</th>\n",
       "      <th>Stat_Value_Group</th>\n",
       "      <th>Test_Type_Group</th>\n",
       "      <th>P_Value_State</th>\n",
       "      <th>Stat_Value_State</th>\n",
       "      <th>Test_Type_State</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>424</td>\n",
       "      <td>5</td>\n",
       "      <td>436.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.052089</td>\n",
       "      <td>6.824923</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>1</td>\n",
       "      <td>105</td>\n",
       "      <td>1</td>\n",
       "      <td>73.5</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.663439</td>\n",
       "      <td>5.896740</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>2</td>\n",
       "      <td>764</td>\n",
       "      <td>3</td>\n",
       "      <td>1005.5</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.639876</td>\n",
       "      <td>6.600722</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>3</td>\n",
       "      <td>199</td>\n",
       "      <td>1</td>\n",
       "      <td>161.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>5.298317</td>\n",
       "      <td>5.864924</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A1</td>\n",
       "      <td>Baseline_1</td>\n",
       "      <td>Sham</td>\n",
       "      <td>Sham</td>\n",
       "      <td>4</td>\n",
       "      <td>696</td>\n",
       "      <td>3</td>\n",
       "      <td>657.5</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.546785</td>\n",
       "      <td>6.420746</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14035</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>25</td>\n",
       "      <td>940</td>\n",
       "      <td>7</td>\n",
       "      <td>1972.5</td>\n",
       "      <td>0.476553</td>\n",
       "      <td>6.846943</td>\n",
       "      <td>7.143300</td>\n",
       "      <td>-0.296357</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.523923</td>\n",
       "      <td>-0.658236</td>\n",
       "      <td>t-test</td>\n",
       "      <td>0.523923</td>\n",
       "      <td>-0.658236</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14036</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>26</td>\n",
       "      <td>796</td>\n",
       "      <td>10</td>\n",
       "      <td>1700.0</td>\n",
       "      <td>0.468235</td>\n",
       "      <td>6.680855</td>\n",
       "      <td>6.565693</td>\n",
       "      <td>0.115162</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.598729</td>\n",
       "      <td>-0.541846</td>\n",
       "      <td>t-test</td>\n",
       "      <td>0.598729</td>\n",
       "      <td>-0.541846</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14037</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>27</td>\n",
       "      <td>134</td>\n",
       "      <td>3</td>\n",
       "      <td>26.0</td>\n",
       "      <td>5.153846</td>\n",
       "      <td>4.905275</td>\n",
       "      <td>3.710625</td>\n",
       "      <td>1.194649</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.028464</td>\n",
       "      <td>108.000000</td>\n",
       "      <td>Mann-Whitney</td>\n",
       "      <td>0.028464</td>\n",
       "      <td>108.000000</td>\n",
       "      <td>Mann-Whitney</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14038</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>28</td>\n",
       "      <td>738</td>\n",
       "      <td>9</td>\n",
       "      <td>916.5</td>\n",
       "      <td>0.805237</td>\n",
       "      <td>6.605298</td>\n",
       "      <td>6.536103</td>\n",
       "      <td>0.069195</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.093105</td>\n",
       "      <td>1.838587</td>\n",
       "      <td>t-test</td>\n",
       "      <td>0.093105</td>\n",
       "      <td>1.838587</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14039</th>\n",
       "      <td>Y2</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>1804</td>\n",
       "      <td>8</td>\n",
       "      <td>2268.5</td>\n",
       "      <td>0.795239</td>\n",
       "      <td>7.498316</td>\n",
       "      <td>7.276425</td>\n",
       "      <td>0.221891</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.611405</td>\n",
       "      <td>-0.522913</td>\n",
       "      <td>t-test</td>\n",
       "      <td>0.611405</td>\n",
       "      <td>-0.522913</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>14040 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Animal_ID  Time_Point    Group Treatment_State  Motif  value  Cluster  \\\n",
       "0            A1  Baseline_1     Sham            Sham      0    424        5   \n",
       "1            A1  Baseline_1     Sham            Sham      1    105        1   \n",
       "2            A1  Baseline_1     Sham            Sham      2    764        3   \n",
       "3            A1  Baseline_1     Sham            Sham      3    199        1   \n",
       "4            A1  Baseline_1     Sham            Sham      4    696        3   \n",
       "...         ...         ...      ...             ...    ...    ...      ...   \n",
       "14035        Y2    Drug_Trt  Treated         Treated     25    940        7   \n",
       "14036        Y2    Drug_Trt  Treated         Treated     26    796       10   \n",
       "14037        Y2    Drug_Trt  Treated         Treated     27    134        3   \n",
       "14038        Y2    Drug_Trt  Treated         Treated     28    738        9   \n",
       "14039        Y2    Drug_Trt  Treated         Treated     29   1804        8   \n",
       "\n",
       "       Baseline_Mean  Normalized_Value  log_value  Sham_Log_Mean  \\\n",
       "0              436.0          1.000000   6.052089       6.824923   \n",
       "1               73.5          1.000000   4.663439       5.896740   \n",
       "2             1005.5          1.000000   6.639876       6.600722   \n",
       "3              161.0          1.000000   5.298317       5.864924   \n",
       "4              657.5          1.000000   6.546785       6.420746   \n",
       "...              ...               ...        ...            ...   \n",
       "14035         1972.5          0.476553   6.846943       7.143300   \n",
       "14036         1700.0          0.468235   6.680855       6.565693   \n",
       "14037           26.0          5.153846   4.905275       3.710625   \n",
       "14038          916.5          0.805237   6.605298       6.536103   \n",
       "14039         2268.5          0.795239   7.498316       7.276425   \n",
       "\n",
       "       Log_Normalized_Value  Is_Normal_Group  Is_Normal_State  P_Value_Group  \\\n",
       "0                  0.000000             True             True            NaN   \n",
       "1                  0.000000             True             True            NaN   \n",
       "2                  0.000000             True             True            NaN   \n",
       "3                  0.000000             True             True            NaN   \n",
       "4                  0.000000             True             True            NaN   \n",
       "...                     ...              ...              ...            ...   \n",
       "14035             -0.296357             True             True       0.523923   \n",
       "14036              0.115162             True             True       0.598729   \n",
       "14037              1.194649            False            False       0.028464   \n",
       "14038              0.069195             True             True       0.093105   \n",
       "14039              0.221891             True             True       0.611405   \n",
       "\n",
       "       Stat_Value_Group Test_Type_Group  P_Value_State  Stat_Value_State  \\\n",
       "0                   NaN             NaN            NaN               NaN   \n",
       "1                   NaN             NaN            NaN               NaN   \n",
       "2                   NaN             NaN            NaN               NaN   \n",
       "3                   NaN             NaN            NaN               NaN   \n",
       "4                   NaN             NaN            NaN               NaN   \n",
       "...                 ...             ...            ...               ...   \n",
       "14035         -0.658236          t-test       0.523923         -0.658236   \n",
       "14036         -0.541846          t-test       0.598729         -0.541846   \n",
       "14037        108.000000    Mann-Whitney       0.028464        108.000000   \n",
       "14038          1.838587          t-test       0.093105          1.838587   \n",
       "14039         -0.522913          t-test       0.611405         -0.522913   \n",
       "\n",
       "      Test_Type_State  \n",
       "0                 NaN  \n",
       "1                 NaN  \n",
       "2                 NaN  \n",
       "3                 NaN  \n",
       "4                 NaN  \n",
       "...               ...  \n",
       "14035          t-test  \n",
       "14036          t-test  \n",
       "14037    Mann-Whitney  \n",
       "14038          t-test  \n",
       "14039          t-test  \n",
       "\n",
       "[14040 rows x 20 columns]"
      ]
     },
     "execution_count": 375,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_long_logNorm_sham"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "07:16:34 PM INFO:Using categorical units to plot a list of strings that are all parsable as floats or dates. If these strings should be plotted as numbers, cast to the appropriate data type before plotting.\n",
      "07:16:34 PM INFO:Using categorical units to plot a list of strings that are all parsable as floats or dates. If these strings should be plotted as numbers, cast to the appropriate data type before plotting.\n"
     ]
    }
   ],
   "source": [
    "reload(AlPf)\n",
    "reload(an)\n",
    "\n",
    "AlPf.plot_boxplot_by_group_and_timepoint_2(df_long_logNorm_sham, significance_level=0.05, specific_time_point=\"Baseline_1\", log_norm=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       Time_Point    Group  Motif  Significant_Motif_Count  Cluster\n",
      "0        Drug_Trt  Injured      0                        1        5\n",
      "468      Drug_Trt  Injured      2                        1        3\n",
      "936      Drug_Trt  Injured      3                        1        1\n",
      "1404     Drug_Trt  Injured      4                        1        3\n",
      "1872     Drug_Trt  Injured      5                        1        7\n",
      "...           ...      ...    ...                      ...      ...\n",
      "157248    Week_15  Treated     21                        1        5\n",
      "157716    Week_15  Treated     23                        1        3\n",
      "158184    Week_15  Treated     24                        1        3\n",
      "158652    Week_15  Treated     26                        1       10\n",
      "159120    Week_15  Treated     28                        1        9\n",
      "\n",
      "[341 rows x 5 columns]\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "reload(AlPf)\n",
    "reload(an)\n",
    "pVals_longNorm_base = an.calculate_p_values_vs_sham(df_long_logNorm_base, test = 'u_stat', log_comp = True)\n",
    "AlPf.plot_normalized_values_by_group_and_timepoint_3(pVals_longNorm_base, df_long_logNorm_base, significance_level=0.05, specific_time_point=None, log_norm=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 502,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Animal_ID</th>\n",
       "      <th>Time_Point</th>\n",
       "      <th>Group</th>\n",
       "      <th>Motif</th>\n",
       "      <th>P_Value</th>\n",
       "      <th>Stat_Value</th>\n",
       "      <th>Test_Type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A1</td>\n",
       "      <td>Week_02</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>0.168867</td>\n",
       "      <td>-1.472674</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1</td>\n",
       "      <td>Week_04</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>0.029836</td>\n",
       "      <td>-2.493752</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A1</td>\n",
       "      <td>Week_06</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000235</td>\n",
       "      <td>-5.347185</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A1</td>\n",
       "      <td>Week_08</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>0.001135</td>\n",
       "      <td>-4.360626</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A1</td>\n",
       "      <td>Week_11</td>\n",
       "      <td>Sham</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>Mann-Whitney</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>925</th>\n",
       "      <td>J1</td>\n",
       "      <td>Week_08</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>0.887676</td>\n",
       "      <td>-0.144557</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>926</th>\n",
       "      <td>J1</td>\n",
       "      <td>Week_11</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>0.134296</td>\n",
       "      <td>1.616421</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927</th>\n",
       "      <td>J1</td>\n",
       "      <td>Week_13</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>0.570718</td>\n",
       "      <td>0.584438</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>928</th>\n",
       "      <td>J1</td>\n",
       "      <td>Week_15</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>0.216773</td>\n",
       "      <td>1.310366</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>929</th>\n",
       "      <td>J1</td>\n",
       "      <td>Drug_Trt</td>\n",
       "      <td>Treated</td>\n",
       "      <td>29</td>\n",
       "      <td>0.319233</td>\n",
       "      <td>1.043217</td>\n",
       "      <td>t-test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>930 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Animal_ID Time_Point    Group  Motif   P_Value  Stat_Value     Test_Type\n",
       "0          A1    Week_02     Sham      0  0.168867   -1.472674        t-test\n",
       "1          A1    Week_04     Sham      0  0.029836   -2.493752        t-test\n",
       "2          A1    Week_06     Sham      0  0.000235   -5.347185        t-test\n",
       "3          A1    Week_08     Sham      0  0.001135   -4.360626        t-test\n",
       "4          A1    Week_11     Sham      0  0.000002   24.000000  Mann-Whitney\n",
       "..        ...        ...      ...    ...       ...         ...           ...\n",
       "925        J1    Week_08  Treated     29  0.887676   -0.144557        t-test\n",
       "926        J1    Week_11  Treated     29  0.134296    1.616421        t-test\n",
       "927        J1    Week_13  Treated     29  0.570718    0.584438        t-test\n",
       "928        J1    Week_15  Treated     29  0.216773    1.310366        t-test\n",
       "929        J1   Drug_Trt  Treated     29  0.319233    1.043217        t-test\n",
       "\n",
       "[930 rows x 7 columns]"
      ]
     },
     "execution_count": 502,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pVals_longNorm_base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_df = pd.read_csv(\"/Users/adanredwine/Desktop/ALR_1-VAME-Project-Feb3-2023/all_sequences_kmeans-30.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "07:16:14 PM INFO:Using categorical units to plot a list of strings that are all parsable as floats or dates. If these strings should be plotted as numbers, cast to the appropriate data type before plotting.\n",
      "07:16:14 PM INFO:Using categorical units to plot a list of strings that are all parsable as floats or dates. If these strings should be plotted as numbers, cast to the appropriate data type before plotting.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       group  motif  frequency\n",
      "0        ABX      0      10080\n",
      "1        ABX      1       6120\n",
      "2        ABX      2       7326\n",
      "3        ABX      3       5676\n",
      "4        ABX      4      10926\n",
      "..       ...    ...        ...\n",
      "115  Treated     25      15642\n",
      "116  Treated     26       9386\n",
      "117  Treated     27       2673\n",
      "118  Treated     28       9447\n",
      "119  Treated     29      18106\n",
      "\n",
      "[120 rows x 3 columns]\n"
     ]
    }
   ],
   "source": [
    "reload(AlPf)\n",
    "time_point = \"Week_15\"\n",
    "AlPf.plot_motif_frequency(master_df, time_point, motifs=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kinematics Analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Total Distance Traveled\n",
    "### The Plan:\n",
    "Step 1: Data Preparation\n",
    "    - 1.1. Load the dataset into a Polars DataFrame if not already loaded.\n",
    "    - 1.2. Verify the integrity of the 'distance' column (no missing or aberrant values).\n",
    "    - 1.3. Group the data by 'group' and 'time_point' to prepare for aggregation.\n",
    "  \n",
    "Step 2: Baseline Calculation\n",
    "    - 2.1. Filter the dataset for 'Baseline_1' and 'Baseline_2' time points.\n",
    "    - 2.2. Calculate the average distance and standard deviation for these two baseline time points for each 'group'.\n",
    "    - 2.3. Store these baseline averages in a separate DataFrame or dictionary for later use.\n",
    "  \n",
    "Step 3: Normality Checks\n",
    "    - 3.1. For each 'group' and 'time_point', perform normality tests (e.g., Shapiro-Wilk) on the 'distance' data.\n",
    "    - 3.2. Use Q-Q plots and histograms for visual inspection of normality.\n",
    "    - 3.3. Document the results to determine the appropriate statistical tests.\n",
    "\n",
    "Step 4: Total Distance Calculation\n",
    "    - 4.1. Aggregate the total distance traveled by summing the 'distance' column for each 'group' and 'time_point'.\n",
    "    - 4.2. Store these totals in a new DataFrame with 'group', 'time_point', and 'total_distance' columns.\n",
    "\n",
    "Step 5: Statistical Comparisons (Pre-Normalization)\n",
    "    - 5.1. For each 'group' and 'time_point', compare the total distance to the group's baseline average using the appropriate statistical test based on normality results.\n",
    "    - 5.2. Adjust for multiple comparisons if necessary.\n",
    "    - 5.3. Document the results, including p-values and effect sizes.\n",
    "\n",
    "Step 6: Normalization\n",
    "    - 6.1. Normalize the total distance for each 'group' and 'time_point' to their respective baseline averages.\n",
    "    - 6.2. Normalize the total distance for each 'group' and 'time_point' to the 'Sham' group's values at the same time points.\n",
    "    - 6.3. Handle any zero or near-zero baseline values carefully to avoid division by zero.\n",
    "\n",
    "Step 7: Normality Checks (Post-Normalization)\n",
    "    - 7.1. Re-check the normalized data for normality using the same methods as in Step 3.\n",
    "    - 7.2. Document any changes in the distribution due to normalization.\n",
    "\n",
    "Step 8: Statistical Comparisons (Post-Normalization)\n",
    "    - 8.1. Repeat the statistical comparisons from Step 5 using the normalized data.\n",
    "    - 8.2. Adjust for multiple comparisons if necessary.\n",
    "    - 8.3. Document the results, including p-values and effect sizes.\n",
    "\n",
    "Step 9: Interpretation and Reporting\n",
    "    - 9.1. Interpret the results from both pre- and post-normalization analyses.\n",
    "    - 9.2. Discuss any significant findings and their implications.\n",
    "    - 9.3. Prepare a report or presentation summarizing the methodology, analyses, and conclusions.\n",
    "\n",
    "Step 10: Review and Validation\n",
    "    - 10.1. Review all steps and results for accuracy and completeness.\n",
    "    - 10.2. Validate the findings by cross-checking with other data or analyses if available.\n",
    "    - 10.3. Ensure that all assumptions for statistical tests are met and documented.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}