population_dashboard_article_version_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Author: Shai Sussman\n",
    "\n",
    "Date: 26 May 2023\n",
    "\n",
    "Description: Script to perform 'last-minute' preprocessing to the reulsts files\n",
    "\n",
    "Use batyam_dashboard venv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import geopandas as gpd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load settings file:\n",
    "response = requests.get('https://raw.githubusercontent.com/Shai2u/demographic_estimation_dashboard_article/main/assets/settings.json')\n",
    "# Check if the request was successful (status code 200)\n",
    "if response.status_code == 200:\n",
    "    # Load the JSON data from the response content\n",
    "    settings = json.loads(response.content)\n",
    "else:\n",
    "    print(f\"Error: {response.status_code}\")\n",
    "    \n",
    "income_dict = settings['income_dict']\n",
    "\n",
    "# Enrich agent track status\n",
    "agents_track_status = pd.read_csv('https://raw.githubusercontent.com/Shai2u/demographic_estimation_dashboard_article/main/dashboard/data/agents_track_status.csv', index_col=0)\n",
    "\n",
    "# Add 1 to renters and 0 to owners for pivoting\n",
    "agents_track_status['income_cat'] = pd.cut(agents_track_status['income'], [0, income_dict[\"Low\"], income_dict[\"Medium\"], income_dict[\"High\"]],right=True, labels=['Low','Medium','High'],ordered=True)\n",
    "agents_track_status.loc[agents_track_status['rent']==1,'rent_own']='Rent'\n",
    "agents_track_status.loc[agents_track_status['rent']==0,'rent_own']='Own'\n",
    "\n",
    "# Enrich agent_stat_summary_by_year\n",
    "agents_stat_summary_by_year = pd.read_csv('https://raw.githubusercontent.com/Shai2u/demographic_estimation_dashboard_article/main/dashboard/data/yearly_stats_for_dashboard.csv', index_col=0)\n",
    "\n",
    "agents_stat_summary_by_year['New Comers'] = agents_stat_summary_by_year['New Comers_income_low'] + agents_stat_summary_by_year['New Comers_income_medium'] +  + agents_stat_summary_by_year['New Comers_income_high']\n",
    "agents_stat_summary_by_year['stay'] = agents_stat_summary_by_year['stay_income_low'] + agents_stat_summary_by_year['stay_income_medium'] +  + agents_stat_summary_by_year['stay_income_high']\n",
    "agents_stat_summary_by_year['total_pop'] = agents_stat_summary_by_year['New Comers'] + agents_stat_summary_by_year['stay']\n",
    "agents_stat_summary_by_year['New Comers_income_low_ratio'] = agents_stat_summary_by_year['New Comers_income_low']/agents_stat_summary_by_year['total_pop']\n",
    "agents_stat_summary_by_year['New Comers_income_medium_ratio'] = agents_stat_summary_by_year['New Comers_income_medium']/agents_stat_summary_by_year['total_pop']\n",
    "agents_stat_summary_by_year['New Comers_income_high_ratio'] = agents_stat_summary_by_year['New Comers_income_high']/agents_stat_summary_by_year['total_pop']\n",
    "agents_stat_summary_by_year['stay_income_low_ratio'] = agents_stat_summary_by_year['stay_income_low']/agents_stat_summary_by_year['total_pop']\n",
    "agents_stat_summary_by_year['stay_income_medium_ratio'] = agents_stat_summary_by_year['stay_income_medium']/agents_stat_summary_by_year['total_pop']\n",
    "agents_stat_summary_by_year['stay_income_high_ratio'] = agents_stat_summary_by_year['stay_income_high']/agents_stat_summary_by_year['total_pop']\n",
    "agents_stat_summary_by_year.reset_index(drop = True, inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Export if needed (do not export automaticly)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# agents_track_status.to_csv('dashboard/data/agents_track_status.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# agents_stat_summary_by_year.to_csv('dashboard/data/yearly_stats_for_dashboard.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "batyam_dashboard",
   "language": "python",
   "name": "batyam_dashboard"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}