oc.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 2,
 "metadata": {
  "language_info": {
   "name": "python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "version": "3.7.5-final"
  },
  "orig_nbformat": 2,
  "file_extension": ".py",
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3,
  "kernelspec": {
   "name": "python37564bitpy37conda77ac5beb731349fc9914c857e6b71b0f",
   "display_name": "Python 3.7.5 64-bit ('py37': conda)"
  }
 },
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Based on https://gist.github.com/AO8/63b9a5acb9fb238cbed13a0269d14137\n",
    "# Collects and parses data posted by OC Health\n",
    "import csv\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "import datetime\n",
    "from datetime import timedelta\n",
    "import json\n",
    "\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "import matplotlib.dates as mdates\n",
    "import matplotlib.ticker as ticker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://occovid19.ochealthinfo.com/coronavirus-in-oc\"\n",
    "out_filename = \"oc_data.json\"\n",
    "\n",
    "def get_todays_info(the_url):\n",
    "    html = urlopen(url)\n",
    "    soup = BeautifulSoup(html, \"html.parser\")\n",
    "    tables = soup.findAll(\"table\")\n",
    "    tests_table = tables[0]\n",
    "    city_table = tables[1]\n",
    "    demo_table = tables[2]\n",
    "    return (tests_table, city_table, demo_table, soup)\n",
    "\n",
    "def html_table_to_list(table):\n",
    "    list_table = []\n",
    "    for row in table.findAll(\"tr\"):\n",
    "        row_table = []\n",
    "        for cell in row.findAll([\"td\", \"th\"]):\n",
    "            row_table.append(cell.get_text())\n",
    "        list_table.append(row_table)\n",
    "    return list_table\n",
    "\n",
    "def get_todays_lists(the_url):\n",
    "    tables = get_todays_info(the_url)\n",
    "    the_lists = [html_table_to_list(table) for table in tables]\n",
    "    todays_date_string = str(tables[-1].text.split('Posted Date:')[-1].split('\\n')[0].replace(' ',''))\n",
    "    return the_lists, todays_date_string\n",
    "\n",
    "def get_todays_json(the_url):\n",
    "    # Get top-line stats\n",
    "    total_cases = int(get_todays_info(url)[-1].findAll('h1')[-4].text.strip())\n",
    "    deaths = int(get_todays_info(url)[-1].findAll('h1')[-3].text.strip())\n",
    "    # Get lists from today's tables\n",
    "    todays_lists, todays_date_string = get_todays_lists(the_url)\n",
    "    \n",
    "    todays_json = {}\n",
    "    todays_json[\"date\"] = todays_date_string\n",
    "    todays_json[\"cumulative_cases\"] = total_cases\n",
    "    todays_json[\"cumulative_deaths\"] = deaths\n",
    "    todays_json[\"tests\"] = {}\n",
    "    todays_json[\"populations\"] = {}\n",
    "    todays_json[\"city_cases\"] = {}\n",
    "    for row in todays_lists[1][1:]:\n",
    "        if row[0] != 'Total Population':\n",
    "            todays_json[\"city_cases\"][row[0]] = int(row[-1].replace(',', '').replace('***', '-1'))\n",
    "    for row in todays_lists[1][1:-5]:\n",
    "        todays_json[\"populations\"][row[0]] = int(row[1].replace(',', ''))\n",
    "    todays_json[\"tests\"][\"people_tested\"] = int(todays_lists[0][0][-1].replace(',', ''))\n",
    "    todays_json[\"tests\"][\"kits_available\"] = int(todays_lists[0][1][-1].replace(',', '').replace('specimens',''))\n",
    "\n",
    "    todays_json[\"Stats\"] = {}\n",
    "    rows = [\"TotalCases\", \"Deaths\", \"TravelRelated\", \"PersonToPerson\", \"CommunityAcquired\", \"UnderInvestigation\"]\n",
    "    cols = [\"Total\", \"Male\", \"Female\", \"OtherGender\", \"Under18\", \"18to49\", \"50to64\", \"65andUp\", \"UnknownAge\"]\n",
    "    for i,row in enumerate(todays_lists[2][3:-1]):\n",
    "        todays_json[\"Stats\"][rows[i]] = {}\n",
    "        for j,col in enumerate(cols):\n",
    "            todays_json[\"Stats\"][rows[i]][cols[j]] = int(row[j+1].replace('\\n','').replace(',',''))\n",
    "    print(todays_date_string)\n",
    "    return todays_date_string, todays_json\n",
    "\n",
    "def append_todays_json(the_out_filename, the_url):\n",
    "    date_string, todays_json = get_todays_json(url)\n",
    "    with open(the_out_filename, 'r') as saved_json_file:\n",
    "        saved_json = json.load(saved_json_file)\n",
    "    if date_string not in saved_json:\n",
    "        out_json = saved_json\n",
    "        out_json[date_string] = todays_json\n",
    "        with open(the_out_filename, 'w') as out_json_file:\n",
    "            json.dump(out_json, out_json_file)\n",
    "\n",
    "def load_oc_data(the_out_filename):\n",
    "    with open(the_out_filename, 'r') as saved_json_file:\n",
    "        saved_json = json.load(saved_json_file)\n",
    "        return saved_json\n",
    "\n",
    "def get_city_time_series(the_json):\n",
    "    date_strings = the_json.keys()\n",
    "    deaths = []\n",
    "    dates = []\n",
    "    total_cases = []\n",
    "    for date_string in date_strings:\n",
    "        date_list = date_string.split('/')\n",
    "        dates.append(datetime.datetime(int(date_list[-1]), \n",
    "                                       int(date_list[0]), int(date_list[1])))\n",
    "        deaths.append(the_json[date_string]['cumulative_deaths'])\n",
    "        total_cases.append(the_json[date_string]['cumulative_cases'])\n",
    "\n",
    "    cities = []\n",
    "    city_case_series = {}\n",
    "    for city in the_json[list(date_strings)[0]]['city_cases']:\n",
    "        cities.append(city)\n",
    "        city_case_series[city] = []\n",
    "\n",
    "    for date_string in date_strings:\n",
    "        for city in cities:\n",
    "            city_case_series[city].append(the_json[date_string]['city_cases'][city])\n",
    "\n",
    "    return dates, deaths, total_cases, city_case_series, the_json\n",
    ""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First time, just output a base file\n",
    "# todays_json = get_todays_json(url)\n",
    "# json_base = {}\n",
    "# json_base[\"March 26, 2020\"] = todays_json\n",
    "# with open(out_filename, 'w') as file:\n",
    "#     json.dump(json_base, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Append to the base file if today's data is different\n",
    "append_todays_json(out_filename, url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot city data on a log scale\n",
    "oc_data = load_oc_data(out_filename)\n",
    "\n",
    "dates, deaths, total_cases, city_series, my_json = get_city_time_series(oc_data)\n",
    "\n",
    "plt.clf()\n",
    "ax = plt.gca()\n",
    "ax.xaxis.set_major_locator(mdates.DayLocator())\n",
    "ax.xaxis.set_major_formatter(mdates.DateFormatter('%b %d'))\n",
    "plt.plot_date(dates, deaths, \"-k\", label = 'Deaths')\n",
    "plt.plot_date(dates, total_cases, \"-m\", label = 'OC Total')\n",
    "for city in ['Fullerton', 'Anaheim', \"Placentia\", 'Buena Park']:\n",
    "    plt.plot_date(dates, city_series[city], \"-\", label=city)\n",
    "plt.legend(loc='best')\n",
    "plt.xlim(dates[0]-datetime.timedelta(days=3), dates[-1]+timedelta(days=1))\n",
    "plt.yscale('log')\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}