EDA_Epidemic.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploratory Data Analysis - Epidemic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import datetime\n",
    "from datetime import date, timedelta\n",
    "import math\n",
    "import pickle\n",
    "\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading epidemic datasets\n",
    "\n",
    "## Cases and testing\n",
    "cases_malaysia = pd.read_csv('dataset/epidemic/cases_malaysia.csv')\n",
    "cases_state = pd.read_csv('dataset/epidemic/cases_state.csv')\n",
    "tests_malaysia = pd.read_csv('dataset/epidemic/tests_malaysia.csv') \n",
    "tests_state = pd.read_csv('dataset/epidemic/tests_state.csv')\n",
    "\n",
    "## Deaths\n",
    "deaths_malaysia = pd.read_csv('dataset/epidemic/deaths_malaysia.csv')\n",
    "deaths_state = pd.read_csv('dataset/epidemic/deaths_state.csv')\n",
    "\n",
    "## Clustering\n",
    "clusters = pd.read_csv('dataset/epidemic/clusters.csv')\n",
    "\n",
    "## Healthcare\n",
    "hospital = pd.read_csv('dataset/epidemic/hospital.csv')\n",
    "icu = pd.read_csv('dataset/epidemic/icu.csv')\n",
    "pkrc = pd.read_csv('dataset/epidemic/pkrc.csv') # PUSAT KUARANTINE DAN RAWATAN COVID, QUARANTINE AND TREATMENT CENTERS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Planning\n",
    "\n",
    "Overall data wrangling pipeline\n",
    "\n",
    "1. Handle duplicate\n",
    "    - checkDuplicate\n",
    "2. Handle missing value\n",
    "    - checkNaN\n",
    "    - checkStateQuantity\n",
    "3. Handle outlier\n",
    "    - checkOutlier\n",
    "4. Handle dates (and segment date into smaller pieces)\n",
    "\t- dateReformat (date format checks)\n",
    "\t- Segment date into year>month>week>day\n",
    "99. Lastly, custom checking\n",
    "\n",
    "Overall EDA\n",
    "1. Merge?\n",
    "2. Agg?\n",
    "99. Plot useful graphs\n",
    "    - multidimensional views"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic info\n",
    "def basicInfo(df):\n",
    "    print(\"Shape: \" + str(df.shape))\n",
    "    return df.tail(3)\n",
    "    \n",
    "# Handle duplicate\n",
    "def checkDuplicate(df):\n",
    "    df[df.duplicated()]\n",
    "    \n",
    "# Handle missing value\n",
    "def checkNaN(df):\n",
    "    null_rows = df.isna().any(axis=1)\n",
    "    rows = null_rows.shape[0]  \n",
    "    # print(rows, \"rows with missing values\")\n",
    "    \n",
    "    if rows > 0: \n",
    "        print(df.isna().sum(), \"\\n\")\n",
    "        return null_rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Handle outlier\n",
    "def check_outlier(df):\n",
    "    numeric_columns = df.describe().columns.copy()\n",
    "    num_of_columns = len(numeric_columns)\n",
    "    \n",
    "    for i in range(math.ceil(num_of_columns/2)):\n",
    "        c = 2*i\n",
    "        \n",
    "        plt.figure(figsize=(20,2))\n",
    "        plt.subplot(1, 2, 1)\n",
    "        plot_boxplot(df[numeric_columns[c]], xlabel=numeric_columns[c])\n",
    "        plt.subplot(1, 2, 2)\n",
    "        try:\n",
    "            plot_boxplot(df[numeric_columns[c+1]], xlabel=numeric_columns[c+1])\n",
    "        except IndexError:\n",
    "            plt.xticks([]), plt.yticks([])\n",
    "            plt.axis(\"off\")\n",
    "        plt.show()\n",
    "    \n",
    "def plot_boxplot(series, title='', xlabel=''):\n",
    "    bp = sns.boxplot(x=series)\n",
    "    bp.set(title=title,\n",
    "           xlabel=xlabel)\n",
    "    return bp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Handle dates\n",
    "\n",
    "# # wrangle relevant date columns\n",
    "# df.date = pd.to_datetime(df.date, errors='coerce').dt.date\n",
    "# df.date_positive = pd.to_datetime(df.date_positive, errors='coerce').dt.date\n",
    "# assert len(df[df.date.isnull()]) == len(df[df.date_positive.isnull()]) == 0\n",
    "# df.date_dose2 = pd.to_datetime(df.date_dose2, errors='coerce').dt.date"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cases and Testing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cases_malaysia.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape: (620, 19)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>cases_new</th>\n",
       "      <th>cases_import</th>\n",
       "      <th>cases_recovered</th>\n",
       "      <th>cases_active</th>\n",
       "      <th>cases_cluster</th>\n",
       "      <th>cases_pvax</th>\n",
       "      <th>cases_fvax</th>\n",
       "      <th>cases_child</th>\n",
       "      <th>cases_adolescent</th>\n",
       "      <th>cases_adult</th>\n",
       "      <th>cases_elderly</th>\n",
       "      <th>cluster_import</th>\n",
       "      <th>cluster_religious</th>\n",
       "      <th>cluster_community</th>\n",
       "      <th>cluster_highRisk</th>\n",
       "      <th>cluster_education</th>\n",
       "      <th>cluster_detentionCentre</th>\n",
       "      <th>cluster_workplace</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>617</th>\n",
       "      <td>2021-10-03</td>\n",
       "      <td>9066</td>\n",
       "      <td>12</td>\n",
       "      <td>14454</td>\n",
       "      <td>150146</td>\n",
       "      <td>388</td>\n",
       "      <td>1119</td>\n",
       "      <td>4941</td>\n",
       "      <td>1700</td>\n",
       "      <td>764</td>\n",
       "      <td>5348</td>\n",
       "      <td>1087</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>224.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>140.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>618</th>\n",
       "      <td>2021-10-04</td>\n",
       "      <td>8075</td>\n",
       "      <td>6</td>\n",
       "      <td>15456</td>\n",
       "      <td>142746</td>\n",
       "      <td>731</td>\n",
       "      <td>1093</td>\n",
       "      <td>4282</td>\n",
       "      <td>1353</td>\n",
       "      <td>641</td>\n",
       "      <td>5050</td>\n",
       "      <td>948</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>613.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>619</th>\n",
       "      <td>2021-10-05</td>\n",
       "      <td>8817</td>\n",
       "      <td>14</td>\n",
       "      <td>15615</td>\n",
       "      <td>135945</td>\n",
       "      <td>619</td>\n",
       "      <td>1060</td>\n",
       "      <td>5105</td>\n",
       "      <td>1386</td>\n",
       "      <td>686</td>\n",
       "      <td>5624</td>\n",
       "      <td>1007</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>199.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>377.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           date  cases_new  cases_import  cases_recovered  cases_active  \\\n",
       "617  2021-10-03       9066            12            14454        150146   \n",
       "618  2021-10-04       8075             6            15456        142746   \n",
       "619  2021-10-05       8817            14            15615        135945   \n",
       "\n",
       "     cases_cluster  cases_pvax  cases_fvax  cases_child  cases_adolescent  \\\n",
       "617            388        1119        4941         1700               764   \n",
       "618            731        1093        4282         1353               641   \n",
       "619            619        1060        5105         1386               686   \n",
       "\n",
       "     cases_adult  cases_elderly  cluster_import  cluster_religious  \\\n",
       "617         5348           1087             0.0                1.0   \n",
       "618         5050            948             0.0                5.0   \n",
       "619         5624           1007             0.0                0.0   \n",
       "\n",
       "     cluster_community  cluster_highRisk  cluster_education  \\\n",
       "617              224.0              17.0                1.0   \n",
       "618               73.0              25.0                8.0   \n",
       "619              199.0              30.0               10.0   \n",
       "\n",
       "     cluster_detentionCentre  cluster_workplace  \n",
       "617                      5.0              140.0  \n",
       "618                      7.0              613.0  \n",
       "619                      3.0              377.0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "basicInfo(cases_malaysia)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>cases_new</th>\n",
       "      <th>cases_import</th>\n",
       "      <th>cases_recovered</th>\n",
       "      <th>cases_active</th>\n",
       "      <th>cases_cluster</th>\n",
       "      <th>cases_pvax</th>\n",
       "      <th>cases_fvax</th>\n",
       "      <th>cases_child</th>\n",
       "      <th>cases_adolescent</th>\n",
       "      <th>cases_adult</th>\n",
       "      <th>cases_elderly</th>\n",
       "      <th>cluster_import</th>\n",
       "      <th>cluster_religious</th>\n",
       "      <th>cluster_community</th>\n",
       "      <th>cluster_highRisk</th>\n",
       "      <th>cluster_education</th>\n",
       "      <th>cluster_detentionCentre</th>\n",
       "      <th>cluster_workplace</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [date, cases_new, cases_import, cases_recovered, cases_active, cases_cluster, cases_pvax, cases_fvax, cases_child, cases_adolescent, cases_adult, cases_elderly, cluster_import, cluster_religious, cluster_community, cluster_highRisk, cluster_education, cluster_detentionCentre, cluster_workplace]\n",
       "Index: []"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cases_malaysia[cases_malaysia.duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "date                         0\n",
      "cases_new                    0\n",
      "cases_import                 0\n",
      "cases_recovered              0\n",
      "cases_active                 0\n",
      "cases_cluster                0\n",
      "cases_pvax                   0\n",
      "cases_fvax                   0\n",
      "cases_child                  0\n",
      "cases_adolescent             0\n",
      "cases_adult                  0\n",
      "cases_elderly                0\n",
      "cluster_import             342\n",
      "cluster_religious          342\n",
      "cluster_community          342\n",
      "cluster_highRisk           342\n",
      "cluster_education          342\n",
      "cluster_detentionCentre    342\n",
      "cluster_workplace          342\n",
      "dtype: int64 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>cases_new</th>\n",
       "      <th>cases_import</th>\n",
       "      <th>cases_recovered</th>\n",
       "      <th>cases_active</th>\n",
       "      <th>cases_cluster</th>\n",
       "      <th>cases_pvax</th>\n",
       "      <th>cases_fvax</th>\n",
       "      <th>cases_child</th>\n",
       "      <th>cases_adolescent</th>\n",
       "      <th>cases_adult</th>\n",
       "      <th>cases_elderly</th>\n",
       "      <th>cluster_import</th>\n",
       "      <th>cluster_religious</th>\n",
       "      <th>cluster_community</th>\n",
       "      <th>cluster_highRisk</th>\n",
       "      <th>cluster_education</th>\n",
       "      <th>cluster_detentionCentre</th>\n",
       "      <th>cluster_workplace</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-01-25</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-01-26</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-01-27</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-01-28</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-01-29</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>337</th>\n",
       "      <td>2020-12-27</td>\n",
       "      <td>1196</td>\n",
       "      <td>5</td>\n",
       "      <td>997</td>\n",
       "      <td>20198</td>\n",
       "      <td>508</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>112</td>\n",
       "      <td>45</td>\n",
       "      <td>876</td>\n",
       "      <td>91</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>338</th>\n",
       "      <td>2020-12-28</td>\n",
       "      <td>1594</td>\n",
       "      <td>3</td>\n",
       "      <td>1181</td>\n",
       "      <td>20603</td>\n",
       "      <td>743</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>129</td>\n",
       "      <td>51</td>\n",
       "      <td>1300</td>\n",
       "      <td>88</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>339</th>\n",
       "      <td>2020-12-29</td>\n",
       "      <td>1925</td>\n",
       "      <td>10</td>\n",
       "      <td>1123</td>\n",
       "      <td>21402</td>\n",
       "      <td>1098</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>127</td>\n",
       "      <td>42</td>\n",
       "      <td>1544</td>\n",
       "      <td>79</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>340</th>\n",
       "      <td>2020-12-30</td>\n",
       "      <td>1870</td>\n",
       "      <td>2</td>\n",
       "      <td>745</td>\n",
       "      <td>22519</td>\n",
       "      <td>920</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>141</td>\n",
       "      <td>62</td>\n",
       "      <td>1304</td>\n",
       "      <td>115</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>341</th>\n",
       "      <td>2020-12-31</td>\n",
       "      <td>2525</td>\n",
       "      <td>13</td>\n",
       "      <td>1481</td>\n",
       "      <td>23556</td>\n",
       "      <td>1505</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>142</td>\n",
       "      <td>62</td>\n",
       "      <td>2152</td>\n",
       "      <td>88</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>342 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           date  cases_new  cases_import  cases_recovered  cases_active  \\\n",
       "0    2020-01-25          4             4                0             4   \n",
       "1    2020-01-26          0             0                0             4   \n",
       "2    2020-01-27          0             0                0             4   \n",
       "3    2020-01-28          0             0                0             4   \n",
       "4    2020-01-29          3             3                0             7   \n",
       "..          ...        ...           ...              ...           ...   \n",
       "337  2020-12-27       1196             5              997         20198   \n",
       "338  2020-12-28       1594             3             1181         20603   \n",
       "339  2020-12-29       1925            10             1123         21402   \n",
       "340  2020-12-30       1870             2              745         22519   \n",
       "341  2020-12-31       2525            13             1481         23556   \n",
       "\n",
       "     cases_cluster  cases_pvax  cases_fvax  cases_child  cases_adolescent  \\\n",
       "0                0           0           0            0                 0   \n",
       "1                0           0           0            0                 0   \n",
       "2                0           0           0            0                 0   \n",
       "3                0           0           0            0                 0   \n",
       "4                0           0           0            1                 0   \n",
       "..             ...         ...         ...          ...               ...   \n",
       "337            508           0           0          112                45   \n",
       "338            743           0           0          129                51   \n",
       "339           1098           0           0          127                42   \n",
       "340            920           0           0          141                62   \n",
       "341           1505           0           0          142                62   \n",
       "\n",
       "     cases_adult  cases_elderly  cluster_import  cluster_religious  \\\n",
       "0              1              0             NaN                NaN   \n",
       "1              0              0             NaN                NaN   \n",
       "2              0              0             NaN                NaN   \n",
       "3              0              0             NaN                NaN   \n",
       "4              2              0             NaN                NaN   \n",
       "..           ...            ...             ...                ...   \n",
       "337          876             91             NaN                NaN   \n",
       "338         1300             88             NaN                NaN   \n",
       "339         1544             79             NaN                NaN   \n",
       "340         1304            115             NaN                NaN   \n",
       "341         2152             88             NaN                NaN   \n",
       "\n",
       "     cluster_community  cluster_highRisk  cluster_education  \\\n",
       "0                  NaN               NaN                NaN   \n",
       "1                  NaN               NaN                NaN   \n",
       "2                  NaN               NaN                NaN   \n",
       "3                  NaN               NaN                NaN   \n",
       "4                  NaN               NaN                NaN   \n",
       "..                 ...               ...                ...   \n",
       "337                NaN               NaN                NaN   \n",
       "338                NaN               NaN                NaN   \n",
       "339                NaN               NaN                NaN   \n",
       "340                NaN               NaN                NaN   \n",
       "341                NaN               NaN                NaN   \n",
       "\n",
       "     cluster_detentionCentre  cluster_workplace  \n",
       "0                        NaN                NaN  \n",
       "1                        NaN                NaN  \n",
       "2                        NaN                NaN  \n",
       "3                        NaN                NaN  \n",
       "4                        NaN                NaN  \n",
       "..                       ...                ...  \n",
       "337                      NaN                NaN  \n",
       "338                      NaN                NaN  \n",
       "339                      NaN                NaN  \n",
       "340                      NaN                NaN  \n",
       "341                      NaN                NaN  \n",
       "\n",
       "[342 rows x 19 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cases_malaysia[checkNaN(cases_malaysia)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Outlier not applicable since it's not human errors\n",
    "# check_outlier(cases_malaysia)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Observation\n",
    "\n",
    "1. There are total of 342 rows with NaN value, all on the cluster data and starting from 2020-01-25 to 2020-12-31. \n",
    "2. Therefore, suggesting spliting dataset into 2\n",
    "    - First dataset, drop all null cluster rows, retain other fields\n",
    "    - Second datset, drop all clusters fields, retain other fields\n",
    "3. Besides, the cluster here is not the same as clusters.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape: (278, 19)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>cases_new</th>\n",
       "      <th>cases_import</th>\n",
       "      <th>cases_recovered</th>\n",
       "      <th>cases_active</th>\n",
       "      <th>cases_cluster</th>\n",
       "      <th>cases_pvax</th>\n",
       "      <th>cases_fvax</th>\n",
       "      <th>cases_child</th>\n",
       "      <th>cases_adolescent</th>\n",
       "      <th>cases_adult</th>\n",
       "      <th>cases_elderly</th>\n",
       "      <th>cluster_import</th>\n",
       "      <th>cluster_religious</th>\n",
       "      <th>cluster_community</th>\n",
       "      <th>cluster_highRisk</th>\n",
       "      <th>cluster_education</th>\n",
       "      <th>cluster_detentionCentre</th>\n",
       "      <th>cluster_workplace</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>617</th>\n",
       "      <td>2021-10-03</td>\n",
       "      <td>9066</td>\n",
       "      <td>12</td>\n",
       "      <td>14454</td>\n",
       "      <td>150146</td>\n",
       "      <td>388</td>\n",
       "      <td>1119</td>\n",
       "      <td>4941</td>\n",
       "      <td>1700</td>\n",
       "      <td>764</td>\n",
       "      <td>5348</td>\n",
       "      <td>1087</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>224.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>140.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>618</th>\n",
       "      <td>2021-10-04</td>\n",
       "      <td>8075</td>\n",
       "      <td>6</td>\n",
       "      <td>15456</td>\n",
       "      <td>142746</td>\n",
       "      <td>731</td>\n",
       "      <td>1093</td>\n",
       "      <td>4282</td>\n",
       "      <td>1353</td>\n",
       "      <td>641</td>\n",
       "      <td>5050</td>\n",
       "      <td>948</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>613.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>619</th>\n",
       "      <td>2021-10-05</td>\n",
       "      <td>8817</td>\n",
       "      <td>14</td>\n",
       "      <td>15615</td>\n",
       "      <td>135945</td>\n",
       "      <td>619</td>\n",
       "      <td>1060</td>\n",
       "      <td>5105</td>\n",
       "      <td>1386</td>\n",
       "      <td>686</td>\n",
       "      <td>5624</td>\n",
       "      <td>1007</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>199.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>377.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           date  cases_new  cases_import  cases_recovered  cases_active  \\\n",
       "617  2021-10-03       9066            12            14454        150146   \n",
       "618  2021-10-04       8075             6            15456        142746   \n",
       "619  2021-10-05       8817            14            15615        135945   \n",
       "\n",
       "     cases_cluster  cases_pvax  cases_fvax  cases_child  cases_adolescent  \\\n",
       "617            388        1119        4941         1700               764   \n",
       "618            731        1093        4282         1353               641   \n",
       "619            619        1060        5105         1386               686   \n",
       "\n",
       "     cases_adult  cases_elderly  cluster_import  cluster_religious  \\\n",
       "617         5348           1087             0.0                1.0   \n",
       "618         5050            948             0.0                5.0   \n",
       "619         5624           1007             0.0                0.0   \n",
       "\n",
       "     cluster_community  cluster_highRisk  cluster_education  \\\n",
       "617              224.0              17.0                1.0   \n",
       "618               73.0              25.0                8.0   \n",
       "619              199.0              30.0               10.0   \n",
       "\n",
       "     cluster_detentionCentre  cluster_workplace  \n",
       "617                      5.0              140.0  \n",
       "618                      7.0              613.0  \n",
       "619                      3.0              377.0  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# First dataset\n",
    "cases_malaysia_cluster = cases_malaysia.dropna()\n",
    "basicInfo(cases_malaysia_cluster)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape: (620, 12)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>cases_new</th>\n",
       "      <th>cases_import</th>\n",
       "      <th>cases_recovered</th>\n",
       "      <th>cases_active</th>\n",
       "      <th>cases_cluster</th>\n",
       "      <th>cases_pvax</th>\n",
       "      <th>cases_fvax</th>\n",
       "      <th>cases_child</th>\n",
       "      <th>cases_adolescent</th>\n",
       "      <th>cases_adult</th>\n",
       "      <th>cases_elderly</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>617</th>\n",
       "      <td>2021-10-03</td>\n",
       "      <td>9066</td>\n",
       "      <td>12</td>\n",
       "      <td>14454</td>\n",
       "      <td>150146</td>\n",
       "      <td>388</td>\n",
       "      <td>1119</td>\n",
       "      <td>4941</td>\n",
       "      <td>1700</td>\n",
       "      <td>764</td>\n",
       "      <td>5348</td>\n",
       "      <td>1087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>618</th>\n",
       "      <td>2021-10-04</td>\n",
       "      <td>8075</td>\n",
       "      <td>6</td>\n",
       "      <td>15456</td>\n",
       "      <td>142746</td>\n",
       "      <td>731</td>\n",
       "      <td>1093</td>\n",
       "      <td>4282</td>\n",
       "      <td>1353</td>\n",
       "      <td>641</td>\n",
       "      <td>5050</td>\n",
       "      <td>948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>619</th>\n",
       "      <td>2021-10-05</td>\n",
       "      <td>8817</td>\n",
       "      <td>14</td>\n",
       "      <td>15615</td>\n",
       "      <td>135945</td>\n",
       "      <td>619</td>\n",
       "      <td>1060</td>\n",
       "      <td>5105</td>\n",
       "      <td>1386</td>\n",
       "      <td>686</td>\n",
       "      <td>5624</td>\n",
       "      <td>1007</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           date  cases_new  cases_import  cases_recovered  cases_active  \\\n",
       "617  2021-10-03       9066            12            14454        150146   \n",
       "618  2021-10-04       8075             6            15456        142746   \n",
       "619  2021-10-05       8817            14            15615        135945   \n",
       "\n",
       "     cases_cluster  cases_pvax  cases_fvax  cases_child  cases_adolescent  \\\n",
       "617            388        1119        4941         1700               764   \n",
       "618            731        1093        4282         1353               641   \n",
       "619            619        1060        5105         1386               686   \n",
       "\n",
       "     cases_adult  cases_elderly  \n",
       "617         5348           1087  \n",
       "618         5050            948  \n",
       "619         5624           1007  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cases_malaysia = cases_malaysia[[\"date\",\"cases_new\",\"cases_import\",\\\n",
    "                                 \"cases_recovered\",\"cases_active\",\"cases_cluster\",\\\n",
    "                                 \"cases_pvax\",\"cases_fvax\",\"cases_child\",\\\n",
    "                                 \"cases_adolescent\",\"cases_adult\",\"cases_elderly\"]]\n",
    "basicInfo(cases_malaysia)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cases_state.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape: (9920, 13)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>state</th>\n",
       "      <th>cases_new</th>\n",
       "      <th>cases_import</th>\n",
       "      <th>cases_recovered</th>\n",
       "      <th>cases_active</th>\n",
       "      <th>cases_cluster</th>\n",
       "      <th>cases_pvax</th>\n",
       "      <th>cases_fvax</th>\n",
       "      <th>cases_child</th>\n",
       "      <th>cases_adolescent</th>\n",
       "      <th>cases_adult</th>\n",
       "      <th>cases_elderly</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9917</th>\n",
       "      <td>2021-10-05</td>\n",
       "      <td>W.P. Kuala Lumpur</td>\n",
       "      <td>217</td>\n",
       "      <td>0</td>\n",
       "      <td>305</td>\n",
       "      <td>3471</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>146</td>\n",
       "      <td>39</td>\n",
       "      <td>13</td>\n",
       "      <td>144</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9918</th>\n",
       "      <td>2021-10-05</td>\n",
       "      <td>W.P. Labuan</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>35</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9919</th>\n",
       "      <td>2021-10-05</td>\n",
       "      <td>W.P. Putrajaya</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "      <td>413</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>16</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            date              state  cases_new  cases_import  cases_recovered  \\\n",
       "9917  2021-10-05  W.P. Kuala Lumpur        217             0              305   \n",
       "9918  2021-10-05        W.P. Labuan          3             0                9   \n",
       "9919  2021-10-05     W.P. Putrajaya         23             0               33   \n",
       "\n",
       "      cases_active  cases_cluster  cases_pvax  cases_fvax  cases_child  \\\n",
       "9917          3471              0          15         146           39   \n",
       "9918            35              1           0           3            0   \n",
       "9919           413              0           3          16            4   \n",
       "\n",
       "      cases_adolescent  cases_adult  cases_elderly  \n",
       "9917                13          144             21  \n",
       "9918                 0            3              0  \n",
       "9919                 2           17              0  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "basicInfo(cases_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>state</th>\n",
       "      <th>cases_new</th>\n",
       "      <th>cases_import</th>\n",
       "      <th>cases_recovered</th>\n",
       "      <th>cases_active</th>\n",
       "      <th>cases_cluster</th>\n",
       "      <th>cases_pvax</th>\n",
       "      <th>cases_fvax</th>\n",
       "      <th>cases_child</th>\n",
       "      <th>cases_adolescent</th>\n",
       "      <th>cases_adult</th>\n",
       "      <th>cases_elderly</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [date, state, cases_new, cases_import, cases_recovered, cases_active, cases_cluster, cases_pvax, cases_fvax, cases_child, cases_adolescent, cases_adult, cases_elderly]\n",
       "Index: []"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cases_state[cases_state.duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "date                0\n",
      "state               0\n",
      "cases_new           0\n",
      "cases_import        0\n",
      "cases_recovered     0\n",
      "cases_active        0\n",
      "cases_cluster       0\n",
      "cases_pvax          0\n",
      "cases_fvax          0\n",
      "cases_child         0\n",
      "cases_adolescent    0\n",
      "cases_adult         0\n",
      "cases_elderly       0\n",
      "dtype: int64 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>state</th>\n",
       "      <th>cases_new</th>\n",
       "      <th>cases_import</th>\n",
       "      <th>cases_recovered</th>\n",
       "      <th>cases_active</th>\n",
       "      <th>cases_cluster</th>\n",
       "      <th>cases_pvax</th>\n",
       "      <th>cases_fvax</th>\n",
       "      <th>cases_child</th>\n",
       "      <th>cases_adolescent</th>\n",
       "      <th>cases_adult</th>\n",
       "      <th>cases_elderly</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [date, state, cases_new, cases_import, cases_recovered, cases_active, cases_cluster, cases_pvax, cases_fvax, cases_child, cases_adolescent, cases_adult, cases_elderly]\n",
       "Index: []"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cases_state[checkNaN(cases_state)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "W.P. Putrajaya       620\n",
       "Pahang               620\n",
       "Pulau Pinang         620\n",
       "Sarawak              620\n",
       "Negeri Sembilan      620\n",
       "Johor                620\n",
       "W.P. Labuan          620\n",
       "Perak                620\n",
       "Selangor             620\n",
       "Melaka               620\n",
       "Kedah                620\n",
       "W.P. Kuala Lumpur    620\n",
       "Kelantan             620\n",
       "Sabah                620\n",
       "Perlis               620\n",
       "Terengganu           620\n",
       "Name: state, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the quantity counts of each state (all of them have to tally)\n",
    "cases_state.state.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Outlier not applicable since it's not human errors\n",
    "# check_outlier(cases_state)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Possible customChecking for cases_malaysia and cases_state\n",
    "- ageCat check categories (some of the cases have unverifiable age)\n",
    "- ageCat is divided into cases_child, cases_adolescent, cases_adult, cases_elderly, and the rest is unverified"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tests_malaysia.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape: (618, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>rtk-ag</th>\n",
       "      <th>pcr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>615</th>\n",
       "      <td>2021-09-30</td>\n",
       "      <td>96673</td>\n",
       "      <td>43082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>616</th>\n",
       "      <td>2021-10-01</td>\n",
       "      <td>100825</td>\n",
       "      <td>37382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>617</th>\n",
       "      <td>2021-10-02</td>\n",
       "      <td>67827</td>\n",
       "      <td>34369</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           date  rtk-ag    pcr\n",
       "615  2021-09-30   96673  43082\n",
       "616  2021-10-01  100825  37382\n",
       "617  2021-10-02   67827  34369"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "basicInfo(tests_malaysia)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>rtk-ag</th>\n",
       "      <th>pcr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [date, rtk-ag, pcr]\n",
       "Index: []"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tests_malaysia[tests_malaysia.duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "date      0\n",
      "rtk-ag    0\n",
      "pcr       0\n",
      "dtype: int64 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>rtk-ag</th>\n",
       "      <th>pcr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [date, rtk-ag, pcr]\n",
       "Index: []"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tests_malaysia[checkNaN(tests_malaysia)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tests_state.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape: (1504, 4)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>state</th>\n",
       "      <th>rtk-ag</th>\n",
       "      <th>pcr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1501</th>\n",
       "      <td>2021-10-02</td>\n",
       "      <td>W.P. Kuala Lumpur</td>\n",
       "      <td>5805</td>\n",
       "      <td>3133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1502</th>\n",
       "      <td>2021-10-02</td>\n",
       "      <td>W.P. Labuan</td>\n",
       "      <td>344</td>\n",
       "      <td>174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1503</th>\n",
       "      <td>2021-10-02</td>\n",
       "      <td>W.P. Putrajaya</td>\n",
       "      <td>141</td>\n",
       "      <td>305</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            date              state  rtk-ag   pcr\n",
       "1501  2021-10-02  W.P. Kuala Lumpur    5805  3133\n",
       "1502  2021-10-02        W.P. Labuan     344   174\n",
       "1503  2021-10-02     W.P. Putrajaya     141   305"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "basicInfo(tests_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>state</th>\n",
       "      <th>rtk-ag</th>\n",
       "      <th>pcr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [date, state, rtk-ag, pcr]\n",
       "Index: []"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tests_state[tests_state.duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "date      0\n",
      "state     0\n",
      "rtk-ag    0\n",
      "pcr       0\n",
      "dtype: int64 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>state</th>\n",
       "      <th>rtk-ag</th>\n",
       "      <th>pcr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [date, state, rtk-ag, pcr]\n",
       "Index: []"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tests_state[checkNaN(tests_state)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## deaths_malaysia.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape: (568, 8)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>deaths_new</th>\n",
       "      <th>deaths_bid</th>\n",
       "      <th>deaths_new_dod</th>\n",
       "      <th>deaths_bid_dod</th>\n",
       "      <th>deaths_pvax</th>\n",
       "      <th>deaths_fvax</th>\n",
       "      <th>deaths_tat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>565</th>\n",
       "      <td>2021-10-03</td>\n",
       "      <td>118</td>\n",
       "      <td>27</td>\n",
       "      <td>44</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>566</th>\n",
       "      <td>2021-10-04</td>\n",
       "      <td>76</td>\n",
       "      <td>15</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>567</th>\n",
       "      <td>2021-10-05</td>\n",
       "      <td>117</td>\n",
       "      <td>16</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           date  deaths_new  deaths_bid  deaths_new_dod  deaths_bid_dod  \\\n",
       "565  2021-10-03         118          27              44               3   \n",
       "566  2021-10-04          76          15              19               0   \n",
       "567  2021-10-05         117          16               3               0   \n",
       "\n",
       "     deaths_pvax  deaths_fvax  deaths_tat  \n",
       "565            6           22           3  \n",
       "566            5            6           2  \n",
       "567            0            2           3  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "basicInfo(deaths_malaysia)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>deaths_new</th>\n",
       "      <th>deaths_bid</th>\n",
       "      <th>deaths_new_dod</th>\n",
       "      <th>deaths_bid_dod</th>\n",
       "      <th>deaths_pvax</th>\n",
       "      <th>deaths_fvax</th>\n",
       "      <th>deaths_tat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [date, deaths_new, deaths_bid, deaths_new_dod, deaths_bid_dod, deaths_pvax, deaths_fvax, deaths_tat]\n",
       "Index: []"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "deaths_malaysia[deaths_malaysia.duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "date              0\n",
      "deaths_new        0\n",
      "deaths_bid        0\n",
      "deaths_new_dod    0\n",
      "deaths_bid_dod    0\n",
      "deaths_pvax       0\n",
      "deaths_fvax       0\n",
      "deaths_tat        0\n",
      "dtype: int64 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>deaths_new</th>\n",
       "      <th>deaths_bid</th>\n",
       "      <th>deaths_new_dod</th>\n",
       "      <th>deaths_bid_dod</th>\n",
       "      <th>deaths_pvax</th>\n",
       "      <th>deaths_fvax</th>\n",
       "      <th>deaths_tat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [date, deaths_new, deaths_bid, deaths_new_dod, deaths_bid_dod, deaths_pvax, deaths_fvax, deaths_tat]\n",
       "Index: []"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "deaths_malaysia[checkNaN(deaths_malaysia)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## deaths_state.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape: (9088, 9)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>state</th>\n",
       "      <th>deaths_new</th>\n",
       "      <th>deaths_bid</th>\n",
       "      <th>deaths_new_dod</th>\n",
       "      <th>deaths_bid_dod</th>\n",
       "      <th>deaths_pvax</th>\n",
       "      <th>deaths_fvax</th>\n",
       "      <th>deaths_tat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9085</th>\n",
       "      <td>2021-10-05</td>\n",
       "      <td>W.P. Kuala Lumpur</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9086</th>\n",
       "      <td>2021-10-05</td>\n",
       "      <td>W.P. Labuan</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9087</th>\n",
       "      <td>2021-10-05</td>\n",
       "      <td>W.P. Putrajaya</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            date              state  deaths_new  deaths_bid  deaths_new_dod  \\\n",
       "9085  2021-10-05  W.P. Kuala Lumpur           4           1               0   \n",
       "9086  2021-10-05        W.P. Labuan           0           0               0   \n",
       "9087  2021-10-05     W.P. Putrajaya           0           0               0   \n",
       "\n",
       "      deaths_bid_dod  deaths_pvax  deaths_fvax  deaths_tat  \n",
       "9085               0            0            0           2  \n",
       "9086               0            0            0           0  \n",
       "9087               0            0            0           0  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "basicInfo(deaths_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>state</th>\n",
       "      <th>deaths_new</th>\n",
       "      <th>deaths_bid</th>\n",
       "      <th>deaths_new_dod</th>\n",
       "      <th>deaths_bid_dod</th>\n",
       "      <th>deaths_pvax</th>\n",
       "      <th>deaths_fvax</th>\n",
       "      <th>deaths_tat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [date, state, deaths_new, deaths_bid, deaths_new_dod, deaths_bid_dod, deaths_pvax, deaths_fvax, deaths_tat]\n",
       "Index: []"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "deaths_state[deaths_state.duplicated()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "date              0\n",
      "state             0\n",
      "deaths_new        0\n",
      "deaths_bid        0\n",
      "deaths_new_dod    0\n",
      "deaths_bid_dod    0\n",
      "deaths_pvax       0\n",
      "deaths_fvax       0\n",
      "deaths_tat        0\n",
      "dtype: int64 \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>state</th>\n",
       "      <th>deaths_new</th>\n",
       "      <th>deaths_bid</th>\n",
       "      <th>deaths_new_dod</th>\n",
       "      <th>deaths_bid_dod</th>\n",
       "      <th>deaths_pvax</th>\n",
       "      <th>deaths_fvax</th>\n",
       "      <th>deaths_tat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [date, state, deaths_new, deaths_bid, deaths_new_dod, deaths_bid_dod, deaths_pvax, deaths_fvax, deaths_tat]\n",
       "Index: []"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "deaths_state[checkNaN(deaths_state)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <span style=\"color:blue\">Multidimensional Data Views</span>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Datetime manipulation\n",
    "https://www.w3schools.com/python/python_datetime.asp\n",
    "or\n",
    "https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add a 'month' column by using 'date'\n",
    "def add_month(df):\n",
    "    df['month'] = df['date'].str[:-3]\n",
    "    return df\n",
    "\n",
    "# group data by months and count\n",
    "def count_by_months(df):\n",
    "    count_df = df.groupby(['month']).count()\n",
    "    count_df.reset_index(inplace = True)\n",
    "    return count_df\n",
    "\n",
    "# group data by months and get mean\n",
    "def avg_by_months(df):\n",
    "    avg_df = df.groupby(['month']).mean()\n",
    "    avg_df.reset_index(inplace = True)\n",
    "    return avg_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pickle the Datasets\n",
    "\n",
    "Pickle all the cleaned data frames for later use.\n",
    "\n",
    "The cleaned dataset has the information below.\n",
    "\n",
    "| Dataset         | Start date | End date   | Number of rows |\n",
    "| --------------- | ---------- | ---------- | -------------- |\n",
    "| cases_malaysia  | 2020-01-25 | 2021-10-05 | 620            |\n",
    "| cases_malaysia_cluster | 2021-01-01 | 2021-10-05 | 278     |\n",
    "| cases_state     | 2020-01-25 | 2021-10-05 | 9920           |\n",
    "| clusters        | 2020-03-01 | 2021-10-05 | 5507           |\n",
    "| tests_malaysia  | 2020-01-24 | 2021-10-02 | 618            |\n",
    "| tests_state     | 2021-07-01 | 2021-10-02 | 1504           |\n",
    "| deaths_malaysia | 2020-03-17 | 2021-10-05 | 568            |\n",
    "| deaths_state    | 2020-03-17 | 2021-10-05 | 9088           |\n",
    "| hospital        | 2020-03-24 | 2021-09-11 |            |\n",
    "| icu             | 2020-03-24 | 2021-09-11 |            |\n",
    "| pkrc            | 2020-03-28 | 2021-09-11 |            |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickle.dump(datasetName, open('pickle_files/datasetName.pkl', 'wb'))\n",
    "pickle.dump(cases_malaysia, open('pickle_files/cases_malaysia.pkl', 'wb'))\n",
    "pickle.dump(cases_malaysia_cluster, open('pickle_files/cases_malaysia_cluster.pkl', 'wb'))\n",
    "pickle.dump(cases_state, open('pickle_files/cases_state.pkl', 'wb'))\n",
    "pickle.dump(tests_malaysia, open('pickle_files/tests_malaysia.pkl', 'wb'))\n",
    "pickle.dump(tests_state, open('pickle_files/tests_state.pkl', 'wb'))\n",
    "pickle.dump(deaths_malaysia, open('pickle_files/deaths_malaysia.pkl', 'wb'))\n",
    "pickle.dump(deaths_state, open('pickle_files/deaths_state.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Documentation for epidemic datasets\n",
    "\n",
    "## File naming convention\n",
    "\n",
    "| Filename | Naming convention | Update frequency |\n",
    "| :--- | :---: | :---: |\n",
    "| cases_malaysia.csv | Static name | Daily by 2359 (for T-0) |\n",
    "| cases_state.csv | Static name | Daily by 2359 (for T-0) |\n",
    "| deaths_malaysia.csv | Static name | Daily by 2359 (for T-0) |\n",
    "| deaths_state.csv | Static name | Daily by 2359 (for T-0) |\n",
    "| clusters.csv | Static name | Daily by 2359 (for T-1) |\n",
    "| pkrc.csv| Static name |  Daily by 2359 (for T-0) |\n",
    "| hospital.csv | Static name |  Daily by 2359 (for T-0) |\n",
    "| icu.csv | Static name |  Daily by 2359 (for T-0) |\n",
    "| tests_malaysia.csv | Static name | At least twice weekly |\n",
    "| tests_state.csv | Static name | At least twice weekly |\n",
    "\n",
    "## Variables and Methodology\n",
    "\n",
    "### Cases and Testing\n",
    "\n",
    "1) `date`: yyyy-mm-dd format; data correct as of 1200hrs on that date<br>\n",
    "2) `state`: name of state (present in state file, but not country file)<br>\n",
    "3) `cases_new`: cases reported in the 24h since the last report<br>\n",
    "4) `cases_import`: imported cases reported in the 24h since the last report<br>\n",
    "5) `cases_active`: Covid+ individuals who have not recovered or died<br>\n",
    "6) `cases_recovered` recovered cases reported in the 24h since the last report<br>\n",
    "7) `cases_cluster`: number of cases attributable to clusters; the difference between `cases_new` and the sum of cases attributable to clusters is the number of sporadic cases<br>\n",
    "8) `cluster_x`: cases attributable to clusters under category `x`; possible values for `x` are import, religious, community, highRisk, education, detentionCentre, and workplace<br>\n",
    "9) `cases_agecat`: cases falling into one of 4 age categories, i.e. child (0-11), adolescent (12-17), adult (18-59), elderly (60+); note that the sum of cases by age may not equal the total cases for that day, as some cases are registered without ages or with unverifiable age data<br> \n",
    "10) `cases_pvax`: number of partially-vaccinated individuals who tested positive for Covid (perfect subset of `cases_new`), where \"partially vaccinated\" is defined as receiving at least 1 dose of a 2-dose vaccine at least 1 day prior to testing positive, or receiving the Cansino vaccine between 1-27 days before testing positive<br>\n",
    "11) `cases_fvax`: number of fully-vaccinated who tested positive for Covid (perfect subset of `cases_new`), where \"fully vaccinated\" is defined as receiving the 2nd dose of a 2-dose vaccine at least 14 days prior to testing positive, or receiving the Cansino vaccine at least 28 days before testing positive<br>\n",
    "12) `rtk-ag`: number of tests done using Antigen Rapid Test Kits (RTK-Ag)<br>\n",
    "13) `pcr`: number of tests done using Real-time Reverse Transcription Polymerase Chain Reaction (RT-PCR) technology<br>\n",
    "\n",
    "### Deaths\n",
    "\n",
    "1) `date`: yyyy-mm-dd format; data correct as of 1200hrs on that date<br>\n",
    "2) `state`: name of state (present in state file, but not country file)<br>\n",
    "3) `deaths_new`: deaths due to COVID-19 based on **date reported to public**<br>\n",
    "4) `deaths_bid`: deaths due to COVID-19 which were brought-in dead based on **date reported to public** (perfect subset of `deaths_new`)<br>\n",
    "5) `deaths_new_dod`: deaths due to COVID-19 based on **date of death**<br>\n",
    "6) `deaths_bid_dod`: deaths due to COVID-19 which were brought-in dead based on **date of death** (perfect subset of `deaths_new_dod`)<br>\n",
    "7) `deaths_pvax`: number of partially-vaccinated individuals who died due to COVID-19 based on **date of death** (perfect subset of `deaths_new_dod`), where \"partially vaccinated\" is defined as receiving at least 1 dose of a 2-dose vaccine at least 1 day prior to testing positive, or receiving the Cansino vaccine between 1-27 days before testing positive.<br>\n",
    "8) `deaths_fvax`: number of fully-vaccinated who died due to COVID-19 based on **date of death** (perfect subset of `deaths_new_dod`), where \"fully vaccinated\" is defined as receiving the 2nd dose of a 2-dose vaccine at least 14 days prior to testing positive, or receiving the Cansino vaccine at least 28 days before testing positive.<br>\n",
    "9) `deaths_tat`: median days between date of death and date of report for all deaths reported on the day<br>\n",
    "\n",
    "### Cluster analysis\n",
    "\n",
    "1) `cluster`: unique textual identifier of cluster; nomenclature does not necessarily signify address<br>\n",
    "2) `state` and `district`: geographical epicentre of cluster, if localised; inter-district and inter-state clusters are possible and present in the dataset<br>\n",
    "3) `date_announced`: date of declaration as cluster<br>\n",
    "4) `date_last_onset`: most recent date of onset of symptoms for individuals within the cluster. note that this is distinct from the date on which said individual was tested, and the date on which their test result was received; consequently, today's date may not necessarily be present in this column.<br>\n",
    "5) `category`: classification as per variable `cluster_x` above<br>\n",
    "6) `status`: active or ended<br>\n",
    "7) `cases_new`: number of new cases detected within cluster in the 24h since the last report<br>\n",
    "8) `cases_total`: total number of cases traced to cluster<br>\n",
    "9) `cases_active`: active cases within cluster<br>\n",
    "10) `tests`: number of tests carried out on individuals within the cluster; denominator for computing a cluster's current positivity rate<br>\n",
    "11) `icu`: number of individuals within the cluster currently under intensive care<br>\n",
    "12) `deaths`: number of individuals within the cluster who passed away due to COVID-19<br>\n",
    "13) `recovered`: number of individuals within the cluster who tested positive for and subsequently recovered from COVID-19<br>\n",
    "\n",
    "\n",
    "### Healthcare \n",
    "\n",
    "_The datasets below have been constructed to provide 3 kinds of insight. First, the inflow and outflow of patients from quarantine centres, hospitals, and intensive care is, without any further scaling or context, critical to monitor - especially when clear divergences between infections and healthcare outcomes start to be observed (e.g. due to vaccination). Second, comparing against available capacity (number of beds, intensive care units, ventilators) allows for understanding of the strain exerted by the epidemic on the healthcare system. Third, the inclusion of datapoints on non-Covid patients demonstrates the interactions between the epidemic and broader health outcomes._\n",
    "\n",
    "### PKRC (COVID-19 Quarantine and Treatment Centre)\n",
    "\n",
    "1) `date`: yyyy-mm-dd format; data correct as of 2359hrs on that date<br>\n",
    "2) `state`: name of state; note that (unlike with other datasets), it is not necessary that there be an observation for every state on every date. for instance, there are no PKRCs in W.P. Kuala Lumpur and W.P Putrajaya.<br>\n",
    "3) `beds`: total PKRC beds (with related medical infrastructure)<br>\n",
    "4) `admitted_x`: number of individuals in category `x` admitted to PKRCs, where `x` can be suspected/probable, COVID-19 positive, or non-COVID<br>\n",
    "5) `discharged_x`: number of individuals in category `x` discharged from PKRCs<br>\n",
    "6) `pkrc_x`: total number of individuals in category `x` in PKRCs; this is a stock variable altered by flows from admissions and discharges<br>\n",
    "\n",
    "\n",
    "### Hospital\n",
    "\n",
    "1) `date`: yyyy-mm-dd format; data correct as of 2359hrs on that date<br>\n",
    "2) `state`: name of state, with similar qualification on exhaustiveness of date-state combos as PKRC data<br>\n",
    "3) `beds`: total hospital beds (with related medical infrastructure)<br>\n",
    "3) `beds_covid`: total beds dedicated for COVID-19<br>\n",
    "4) `beds_noncrit`: total hospital beds for non-critical care<br>\n",
    "5) `admitted_x`: number of individuals in category `x` admitted to hospitals, where `x` can be suspected/probable, COVID-19 positive, or non-COVID<br>\n",
    "6) `discharged_x`: number of individuals in category `x` discharged from hospitals<br>\n",
    "7) `hosp_x`: total number of individuals in category `x` in hospitals; this is a stock variable altered by flows from admissions and discharges<br>\n",
    "\n",
    "\n",
    "### ICU\n",
    "\n",
    "1) `date`: yyyy-mm-dd format; data correct as of 2359hrs on that date<br>\n",
    "2) `state`: name of state, with similar qualification on exhaustiveness of date-state combos as PKRC data<br>\n",
    "3) `beds_icu`: total gazetted ICU beds<br>\n",
    "4) `beds_icu_rep`: total beds aside from (3) which are temporarily or permanently designated to be under the care of Anaesthesiology & Critical Care departments<br>\n",
    "5) `beds_icu_total`: total critical care beds available (with related medical infrastructure)<br>\n",
    "6) `beds_icu_covid`: total critical care beds dedicated for COVID-19<br>\n",
    "7) `vent`: total available ventilators<br>\n",
    "8) `vent_port`: total available portable ventilators<br>\n",
    "9) `icu_x`: total number of individuals in category `x` under intensive care, where `x` can be  suspected/probable, COVID-19 positive, or non-COVID; this is a stock variable<br>\n",
    "10) `vent_x`: total number of individuals in category `x` on mechanical ventilation, where `x` can be suspected/probable, COVID-19 positive, or non-COVID; this is a stock variable<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>state</th>\n",
       "      <th>beds</th>\n",
       "      <th>beds_covid</th>\n",
       "      <th>beds_noncrit</th>\n",
       "      <th>admitted_pui</th>\n",
       "      <th>admitted_covid</th>\n",
       "      <th>admitted_total</th>\n",
       "      <th>discharged_pui</th>\n",
       "      <th>discharged_covid</th>\n",
       "      <th>discharged_total</th>\n",
       "      <th>hosp_covid</th>\n",
       "      <th>hosp_pui</th>\n",
       "      <th>hosp_noncovid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Johor</td>\n",
       "      <td>1440</td>\n",
       "      <td>434</td>\n",
       "      <td>1408</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Kedah</td>\n",
       "      <td>1218</td>\n",
       "      <td>143</td>\n",
       "      <td>1183</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>46</td>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Kelantan</td>\n",
       "      <td>1463</td>\n",
       "      <td>280</td>\n",
       "      <td>1424</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>78</td>\n",
       "      <td>58</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Melaka</td>\n",
       "      <td>1091</td>\n",
       "      <td>82</td>\n",
       "      <td>1065</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Negeri Sembilan</td>\n",
       "      <td>1223</td>\n",
       "      <td>253</td>\n",
       "      <td>1205</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>90</td>\n",
       "      <td>13</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Pahang</td>\n",
       "      <td>851</td>\n",
       "      <td>87</td>\n",
       "      <td>830</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Perak</td>\n",
       "      <td>1706</td>\n",
       "      <td>211</td>\n",
       "      <td>1667</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>78</td>\n",
       "      <td>40</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Perlis</td>\n",
       "      <td>508</td>\n",
       "      <td>12</td>\n",
       "      <td>495</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Pulau Pinang</td>\n",
       "      <td>1158</td>\n",
       "      <td>86</td>\n",
       "      <td>1132</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>68</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Sabah</td>\n",
       "      <td>2598</td>\n",
       "      <td>605</td>\n",
       "      <td>2528</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>161</td>\n",
       "      <td>46</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Sarawak</td>\n",
       "      <td>2390</td>\n",
       "      <td>418</td>\n",
       "      <td>2327</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>73</td>\n",
       "      <td>85</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Selangor</td>\n",
       "      <td>887</td>\n",
       "      <td>504</td>\n",
       "      <td>783</td>\n",
       "      <td>5</td>\n",
       "      <td>16</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>261</td>\n",
       "      <td>39</td>\n",
       "      <td>184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Terengganu</td>\n",
       "      <td>1232</td>\n",
       "      <td>215</td>\n",
       "      <td>1209</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>W.P. Kuala Lumpur</td>\n",
       "      <td>1570</td>\n",
       "      <td>248</td>\n",
       "      <td>1537</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>70</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>W.P. Labuan</td>\n",
       "      <td>122</td>\n",
       "      <td>9</td>\n",
       "      <td>115</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2020-03-25</td>\n",
       "      <td>Johor</td>\n",
       "      <td>1440</td>\n",
       "      <td>511</td>\n",
       "      <td>1408</td>\n",
       "      <td>20</td>\n",
       "      <td>14</td>\n",
       "      <td>34</td>\n",
       "      <td>26</td>\n",
       "      <td>0</td>\n",
       "      <td>26</td>\n",
       "      <td>134</td>\n",
       "      <td>120</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          date              state  beds  beds_covid  beds_noncrit  \\\n",
       "0   2020-03-24              Johor  1440         434          1408   \n",
       "1   2020-03-24              Kedah  1218         143          1183   \n",
       "2   2020-03-24           Kelantan  1463         280          1424   \n",
       "3   2020-03-24             Melaka  1091          82          1065   \n",
       "4   2020-03-24    Negeri Sembilan  1223         253          1205   \n",
       "5   2020-03-24             Pahang   851          87           830   \n",
       "6   2020-03-24              Perak  1706         211          1667   \n",
       "7   2020-03-24             Perlis   508          12           495   \n",
       "8   2020-03-24       Pulau Pinang  1158          86          1132   \n",
       "9   2020-03-24              Sabah  2598         605          2528   \n",
       "10  2020-03-24            Sarawak  2390         418          2327   \n",
       "11  2020-03-24           Selangor   887         504           783   \n",
       "12  2020-03-24         Terengganu  1232         215          1209   \n",
       "13  2020-03-24  W.P. Kuala Lumpur  1570         248          1537   \n",
       "14  2020-03-24        W.P. Labuan   122           9           115   \n",
       "15  2020-03-25              Johor  1440         511          1408   \n",
       "\n",
       "    admitted_pui  admitted_covid  admitted_total  discharged_pui  \\\n",
       "0              0               0               0               0   \n",
       "1              0               0               0               0   \n",
       "2              9               6              15               1   \n",
       "3              0               0               0               0   \n",
       "4              0               0               0               0   \n",
       "5              0               0               0               0   \n",
       "6              0               0               0               0   \n",
       "7              0               0               0               0   \n",
       "8              0               0               0               0   \n",
       "9              0               0               0               0   \n",
       "10             0               0               0               0   \n",
       "11             5              16              21               0   \n",
       "12             0               0               0               0   \n",
       "13             0               0               0               0   \n",
       "14             0               0               0               0   \n",
       "15            20              14              34              26   \n",
       "\n",
       "    discharged_covid  discharged_total  hosp_covid  hosp_pui  hosp_noncovid  \n",
       "0                  0                 0          18         9              1  \n",
       "1                  0                 0          46         5             12  \n",
       "2                  0                 1          78        58             23  \n",
       "3                  0                 0          19        10             12  \n",
       "4                  0                 0          90        13              8  \n",
       "5                  0                 0          36        14             14  \n",
       "6                  1                 1          78        40              6  \n",
       "7                  0                 0           7         8              8  \n",
       "8                  0                 0          68         2              8  \n",
       "9                  0                 0         161        46             35  \n",
       "10                 0                 0          73        85             29  \n",
       "11                24                24         261        39            184  \n",
       "12                 0                 0           9         6             14  \n",
       "13                 0                 0          70        15              1  \n",
       "14                 0                 0           5         3              2  \n",
       "15                 0                26         134       120              1  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hospital.head(16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>state</th>\n",
       "      <th>beds_icu</th>\n",
       "      <th>beds_icu_rep</th>\n",
       "      <th>beds_icu_total</th>\n",
       "      <th>beds_icu_covid</th>\n",
       "      <th>vent</th>\n",
       "      <th>vent_port</th>\n",
       "      <th>icu_covid</th>\n",
       "      <th>icu_pui</th>\n",
       "      <th>icu_noncovid</th>\n",
       "      <th>vent_covid</th>\n",
       "      <th>vent_pui</th>\n",
       "      <th>vent_noncovid</th>\n",
       "      <th>vent_used</th>\n",
       "      <th>vent_port_used</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Johor</td>\n",
       "      <td>10</td>\n",
       "      <td>22</td>\n",
       "      <td>32</td>\n",
       "      <td>10</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Kedah</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>16</td>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Kelantan</td>\n",
       "      <td>39</td>\n",
       "      <td>0</td>\n",
       "      <td>39</td>\n",
       "      <td>20</td>\n",
       "      <td>42</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>26</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Melaka</td>\n",
       "      <td>26</td>\n",
       "      <td>0</td>\n",
       "      <td>26</td>\n",
       "      <td>12</td>\n",
       "      <td>23</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-03-24</td>\n",
       "      <td>Negeri Sembilan</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>10</td>\n",
       "      <td>14</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date            state  beds_icu  beds_icu_rep  beds_icu_total  \\\n",
       "0  2020-03-24            Johor        10            22              32   \n",
       "1  2020-03-24            Kedah        35             0              35   \n",
       "2  2020-03-24         Kelantan        39             0              39   \n",
       "3  2020-03-24           Melaka        26             0              26   \n",
       "4  2020-03-24  Negeri Sembilan        18             0              18   \n",
       "\n",
       "   beds_icu_covid  vent  vent_port  icu_covid  icu_pui  icu_noncovid  \\\n",
       "0              10    11          2          0        0             1   \n",
       "1              16    34          0          2        1            12   \n",
       "2              20    42          4          1        0            20   \n",
       "3              12    23          4          0        2            12   \n",
       "4              10    14          9          0        0             8   \n",
       "\n",
       "   vent_covid  vent_pui  vent_noncovid  vent_used  vent_port_used  \n",
       "0           0         0              0          8               0  \n",
       "1           1         1              0         19               0  \n",
       "2           1         0              0         26               0  \n",
       "3           0         1              0         13               5  \n",
       "4           0         0              0          1               6  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "icu.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>state</th>\n",
       "      <th>beds</th>\n",
       "      <th>admitted_pui</th>\n",
       "      <th>admitted_covid</th>\n",
       "      <th>admitted_total</th>\n",
       "      <th>discharge_pui</th>\n",
       "      <th>discharge_covid</th>\n",
       "      <th>discharge_total</th>\n",
       "      <th>pkrc_covid</th>\n",
       "      <th>pkrc_pui</th>\n",
       "      <th>pkrc_noncovid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-03-28</td>\n",
       "      <td>Kelantan</td>\n",
       "      <td>147</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>53</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-03-28</td>\n",
       "      <td>Perlis</td>\n",
       "      <td>54</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-03-28</td>\n",
       "      <td>Pulau Pinang</td>\n",
       "      <td>58</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-03-28</td>\n",
       "      <td>Sarawak</td>\n",
       "      <td>74</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-03-28</td>\n",
       "      <td>Selangor</td>\n",
       "      <td>1452</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>145</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date         state  beds  admitted_pui  admitted_covid  \\\n",
       "0  2020-03-28      Kelantan   147             0               0   \n",
       "1  2020-03-28        Perlis    54             0               0   \n",
       "2  2020-03-28  Pulau Pinang    58             0               0   \n",
       "3  2020-03-28       Sarawak    74             0               0   \n",
       "4  2020-03-28      Selangor  1452             0              12   \n",
       "\n",
       "   admitted_total  discharge_pui  discharge_covid  discharge_total  \\\n",
       "0               0              0                0                0   \n",
       "1               0              0                0                0   \n",
       "2               0              0                0                0   \n",
       "3               0              0                0                0   \n",
       "4              12              0                5                5   \n",
       "\n",
       "   pkrc_covid  pkrc_pui  pkrc_noncovid  \n",
       "0           0        53              0  \n",
       "1           8         3              0  \n",
       "2          13         0              0  \n",
       "3          22         0              0  \n",
       "4         145         0              8  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pkrc.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}