Version update 0.0.2: Python 3.11 support, testing module, minor bugfixes, added utilities

README.md

@@ -2,7 +2,7 @@
 Web scraping utilties for DWR, USACE, USGS, CNRFC, CVO and SacALERT data repositories.
 
 ## Setup instructions
-### Create a virtual environment, specifying Python version 3.8
+### Create a virtual environment, specifying Python version 3.11
 
 #### With pure Python (3+)
 Create a virtual environment with Python 3's built-in `venv` library.  

collect/alert/alert.py

@@ -10,6 +10,8 @@
 
 from bs4 import BeautifulSoup, SoupStrainer
 import pandas as pd
+import requests
+
 from collect import utils
 
 
@@ -33,7 +35,7 @@
 
 def _ustrip(x):
     """
-    strips whitespace represented by unicode non-breaking space in additon to default white
+    strips whitespace represented by unicode non-breaking space in addition to default white
     space stripping by python's str.strip() method
     Arguments:
         x (str): string containing an encoded whitespace
@@ -64,8 +66,9 @@ def get_sites(as_dataframe=True, datatype='stream'):
     group_type_id = {'rain': 14, 'stream': 19, 'temperature': 30}.get(datatype)
 
     url = f'https://www.sacflood.org/{measure}?&view_id=1&group_type_id={group_type_id}'
-    soup = BeautifulSoup(utils.get_session_response(url).text, 'lxml')
-    df = pd.read_html(str(soup.find('table')))[0]
+    soup = BeautifulSoup(requests.get(url).text, 'html.parser')
+    with io.StringIO(str(soup.find('table'))) as text:
+        df = pd.read_html(text)[0]
 
     # strip whitespace from columns
     df.columns = [_ustrip(x) for x in df.columns]
@@ -87,7 +90,7 @@ def get_sites_from_list(as_dataframe=True, sensor_class=None):
     url = 'https://www.sacflood.org/list/'
     if sensor_class:
         url += '?&sensor_class={}'.format(sensor_class)
-    soup = BeautifulSoup(utils.get_session_response(url).text, 'lxml')
+    soup = BeautifulSoup(requests.get(url).text, 'html.parser')
 
     entries = []
     for x in soup.find_all('a', {'class': None, 'target': None}, 
@@ -114,7 +117,7 @@ def get_site_notes(site_id):
     """
     url = f'https://www.sacflood.org/site/?site_id={site_id}'
     strainer = SoupStrainer('div', {'class': 'card-body'})
-    soup = BeautifulSoup(utils.get_session_response(url).text, 'lxml', parse_only=strainer)
+    soup = BeautifulSoup(requests.get(url).text, 'html.parser', parse_only=strainer)
     for card in soup.find_all('div', {'class': 'card-body'}):
         if 'Notes' in card.find('h3', {'class': 'card-title'}).text:
             notes_block = card.find('p', {'class': 'list-group-item-text'})
@@ -137,7 +140,7 @@ def get_site_location(site_id):
     """
     url = f'https://www.sacflood.org/site/?site_id={site_id}'
     result = {'site_id': site_id, 'url': url}
-    soup = BeautifulSoup(utils.get_session_response(url).text, 'lxml')
+    soup = BeautifulSoup(requests.get(url).text, 'html.parser')
     cards = soup.find_all('div', {'class': 'card-body'})
     for card in cards:
         if 'Map' in card.find('h3', {'class': 'card-title'}).text:
@@ -156,7 +159,7 @@ def get_site_sensors(site_id):
     """
     url = f'https://www.sacflood.org/site/?site_id={site_id}'
     result = {'site_id': site_id, 'url': url, 'sensors': []}
-    soup = BeautifulSoup(utils.get_session_response(url).text, 'lxml')
+    soup = BeautifulSoup(requests.get(url).text, 'html.parser')
     cards = soup.find_all('div', {'class': 'card-body'})
     for card in cards:
         if 'Sensors' in card.find('h3', {'class': 'card-title'}).text:
@@ -206,7 +209,7 @@ def get_query_url(site_id, device_id, start, end):
 
 def get_device_series(site_id, device_id, start, end, ascending=True):
     url = get_query_url(site_id, device_id, start, end)
-    response = io.StringIO(utils.get_session_response(url).text)
+    response = io.StringIO(requests.get(url).text)
     df = pd.read_csv(response)
     return df.sort_values(by='Reading', ascending=ascending)