Skip to content

Commit

Permalink
basic catalog functionality
Browse files Browse the repository at this point in the history
  • Loading branch information
@jmunroe
jmunroe committed Oct 14, 2020
1 parent 9d9ec83 commit 29a3461
Show file tree
Hide file tree
Showing 5 changed files with 367 additions and 7 deletions.
359 changes: 359 additions & 0 deletions examples/intake-erddap.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,359 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Intake-ERDDAP"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from intake_erddap.erddap_cat import ERDDAPCatalog\n",
"import intake\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"server = 'https://cioosatlantic.ca/erddap'\n",
"cat = ERDDAPCatalog(server)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['coastal_action_7471_9460_c025',\n",
" 'SMA_bay_of_exploits',\n",
" 'SMA_bay_of_islands',\n",
" 'bb1_bop_level1',\n",
" 'SMA_bonavista',\n",
" 'SMA_port_aux_basques',\n",
" 'SMA_port_aux_basqes_wharf',\n",
" 'coast-of-bays-hydrographic-2009-2013',\n",
" 'SMA_corner_brook_wharf',\n",
" 'SMA_saint_john_cruise_terminal',\n",
" 'SMA_Fortune_Bay_Buoy',\n",
" 'SMA_halifax',\n",
" 'SMA_halifax_fairview',\n",
" 'SMA_halifax_anemometer1',\n",
" 'SMA_halifax_pier9c',\n",
" 'SMA_head_of_placentia_bay-come_by_chance_point',\n",
" 'SMA_Holyrood_Buoy2',\n",
" 'SMA_holyrood_wharf',\n",
" 'SMA_bay_of_exploits_wharf',\n",
" 'SMA_manolis_buoy',\n",
" 'SMA_MouthofPlacentiaBayBuoy',\n",
" 'sma_negl_black_tickle_nlqu0003',\n",
" 'sma_negl_cartwright_junction_nlqu0004',\n",
" 'sma_negl_north_west_river_nlqu0007',\n",
" 'sma_negl_postville_nlqu0001',\n",
" 'sma_negl_red_bay_nlqu0005',\n",
" 'sma_negl_rigolet_nlqu0002',\n",
" 'SMA_red_island_shoal',\n",
" 'DFO_Sutron_NHARB',\n",
" 'DFO_Sutron_KLUMI',\n",
" 'SMA_saint_john_wharf',\n",
" 'SMA_saint_john',\n",
" 'SMA_st_johns',\n",
" 'SMA_st_johns_wharf',\n",
" 'SmartAtlantic_XEOS_hk4_buoy']"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"list(cat)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'name': 'SMA_bay_of_exploits',\n",
" 'container': 'dataframe',\n",
" 'plugin': ['erddap'],\n",
" 'description': 'ERDDAP dataset_id SMA_bay_of_exploits from https://cioosatlantic.ca/erddap',\n",
" 'direct_access': True,\n",
" 'user_parameters': [],\n",
" 'metadata': {},\n",
" 'args': {'server': 'https://cioosatlantic.ca/erddap',\n",
" 'dataset_id': 'SMA_bay_of_exploits',\n",
" 'protocol': 'tabledap'}}"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cat.SMA_bay_of_exploits.describe()"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"df = cat.SMA_manolis_buoy.read()"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>station_name</th>\n",
" <th>time (UTC)</th>\n",
" <th>longitude (degrees_east)</th>\n",
" <th>latitude (degrees_north)</th>\n",
" <th>wind_spd_avg</th>\n",
" <th>wind_spd_max</th>\n",
" <th>wind_dir_avg</th>\n",
" <th>air_temp_avg</th>\n",
" <th>air_pressure_avg</th>\n",
" <th>air_humidity_avg</th>\n",
" <th>air_dewpoint_avg</th>\n",
" <th>surface_temp_avg</th>\n",
" <th>wave_ht_max</th>\n",
" <th>wave_ht_sig</th>\n",
" <th>wave_period_max</th>\n",
" <th>wave_dir_avg</th>\n",
" <th>wave_spread_avg</th>\n",
" <th>curr_dir_avg</th>\n",
" <th>curr_spd_avg</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>smb_manolis</td>\n",
" <td>2018-07-10T00:23:41Z</td>\n",
" <td>-54.507837</td>\n",
" <td>49.675302</td>\n",
" <td>5.9</td>\n",
" <td>7.2</td>\n",
" <td>222.0</td>\n",
" <td>18.9</td>\n",
" <td>1008.1</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>10.9</td>\n",
" <td>0.8</td>\n",
" <td>0.5</td>\n",
" <td>8.7</td>\n",
" <td>26.0</td>\n",
" <td>46.0</td>\n",
" <td>274.1</td>\n",
" <td>104.9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>smb_manolis</td>\n",
" <td>2018-07-10T00:53:41Z</td>\n",
" <td>-54.507828</td>\n",
" <td>49.675285</td>\n",
" <td>4.8</td>\n",
" <td>6.1</td>\n",
" <td>218.0</td>\n",
" <td>18.4</td>\n",
" <td>1008.5</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>10.9</td>\n",
" <td>0.8</td>\n",
" <td>0.5</td>\n",
" <td>8.7</td>\n",
" <td>26.0</td>\n",
" <td>46.0</td>\n",
" <td>271.5</td>\n",
" <td>85.4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>smb_manolis</td>\n",
" <td>2018-07-10T01:23:41Z</td>\n",
" <td>-54.507797</td>\n",
" <td>49.675293</td>\n",
" <td>7.8</td>\n",
" <td>9.9</td>\n",
" <td>235.0</td>\n",
" <td>19.8</td>\n",
" <td>1008.3</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>10.9</td>\n",
" <td>0.8</td>\n",
" <td>0.5</td>\n",
" <td>8.7</td>\n",
" <td>26.0</td>\n",
" <td>46.0</td>\n",
" <td>301.8</td>\n",
" <td>60.2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>smb_manolis</td>\n",
" <td>2018-07-10T01:53:41Z</td>\n",
" <td>-54.507805</td>\n",
" <td>49.675285</td>\n",
" <td>7.8</td>\n",
" <td>9.7</td>\n",
" <td>237.0</td>\n",
" <td>19.1</td>\n",
" <td>1008.3</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>10.9</td>\n",
" <td>0.8</td>\n",
" <td>0.5</td>\n",
" <td>8.7</td>\n",
" <td>26.0</td>\n",
" <td>46.0</td>\n",
" <td>326.3</td>\n",
" <td>64.3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>smb_manolis</td>\n",
" <td>2018-07-10T02:23:41Z</td>\n",
" <td>-54.507788</td>\n",
" <td>49.675310</td>\n",
" <td>6.9</td>\n",
" <td>7.6</td>\n",
" <td>233.0</td>\n",
" <td>17.3</td>\n",
" <td>1008.3</td>\n",
" <td>NaN</td>\n",
" <td>NaN</td>\n",
" <td>10.8</td>\n",
" <td>0.8</td>\n",
" <td>0.5</td>\n",
" <td>8.7</td>\n",
" <td>26.0</td>\n",
" <td>46.0</td>\n",
" <td>337.1</td>\n",
" <td>57.5</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" station_name time (UTC) longitude (degrees_east) \\\n",
"0 smb_manolis 2018-07-10T00:23:41Z -54.507837 \n",
"1 smb_manolis 2018-07-10T00:53:41Z -54.507828 \n",
"2 smb_manolis 2018-07-10T01:23:41Z -54.507797 \n",
"3 smb_manolis 2018-07-10T01:53:41Z -54.507805 \n",
"4 smb_manolis 2018-07-10T02:23:41Z -54.507788 \n",
"\n",
" latitude (degrees_north) wind_spd_avg wind_spd_max wind_dir_avg \\\n",
"0 49.675302 5.9 7.2 222.0 \n",
"1 49.675285 4.8 6.1 218.0 \n",
"2 49.675293 7.8 9.9 235.0 \n",
"3 49.675285 7.8 9.7 237.0 \n",
"4 49.675310 6.9 7.6 233.0 \n",
"\n",
" air_temp_avg air_pressure_avg air_humidity_avg air_dewpoint_avg \\\n",
"0 18.9 1008.1 NaN NaN \n",
"1 18.4 1008.5 NaN NaN \n",
"2 19.8 1008.3 NaN NaN \n",
"3 19.1 1008.3 NaN NaN \n",
"4 17.3 1008.3 NaN NaN \n",
"\n",
" surface_temp_avg wave_ht_max wave_ht_sig wave_period_max wave_dir_avg \\\n",
"0 10.9 0.8 0.5 8.7 26.0 \n",
"1 10.9 0.8 0.5 8.7 26.0 \n",
"2 10.9 0.8 0.5 8.7 26.0 \n",
"3 10.9 0.8 0.5 8.7 26.0 \n",
"4 10.8 0.8 0.5 8.7 26.0 \n",
"\n",
" wave_spread_avg curr_dir_avg curr_spd_avg \n",
"0 46.0 274.1 104.9 \n",
"1 46.0 271.5 85.4 \n",
"2 46.0 301.8 60.2 \n",
"3 46.0 326.3 64.3 \n",
"4 46.0 337.1 57.5 "
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:intake-erddap]",
"language": "python",
"name": "conda-env-intake-erddap-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.6"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
3 changes: 2 additions & 1 deletion intake_erddap/erddap_cat.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ def _load(self):
continue

description = 'ERDDAP dataset_id %s from %s' % (dataset_id, self.server)
args = {'server': self.server,
args = {'server': self.server,
'dataset_id': dataset_id,
'protocol': 'tabledap',
}

Expand Down
1 change: 1 addition & 0 deletions requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
intake
pandas
erddapy
panel
7 changes: 3 additions & 4 deletions tests/cat.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
sources:
tables:
args:
uri: "{{URI}}"
server: "{{URI}}"
parameters: # User defined parameters
URI:
description: sqlite
type: str
default: "env(TEST_SQLITE_URI)"
description: "points to sqlite database"
driver: sql_cat
description: "points to ERDDAP server"
driver: erddap_cat
direct_access: forbid
metadata: {}
Loading

0 comments on commit 29a3461

Please sign in to comment.