Merge pull request #51 from mullenkamp/dev

updated to version 2

conda/meta.yaml

@@ -1,5 +1,5 @@
 {% set name = "hilltop-py" %}
-{% set version = "1.5.6" %}
+{% set version = "2.0.0" %}
 # {% set sha256 = "72a156e328247c91cb7f5440ffa98069c0090892f8d9d07fd57e36c0611a0403" %}
 
 # sha256 is the prefered checksum -- you can get it for a file with:
@@ -37,6 +37,8 @@ requirements:
   run:
     - python
     - pandas
+    - pydantic
+    - orjson
 
 test:
   imports:

hilltoppy/tests/test_web_service.py

@@ -48,7 +48,6 @@
     measurement = 'Total Phosphorus',
     from_date = '2012-01-22 10:50',
     to_date = '2018-04-13 14:05',
-    dtl_method = 'trend'
     )
 
 ### Tests
@@ -77,12 +76,6 @@ def test_site_list_with_collection(data):
     assert len(sites) > 40
 
 
-@pytest.mark.parametrize('data', [test_data1])
-def test_wq_sample_parameter_list(data):
-    mtype_df2 = wq_sample_parameter_list(data['base_url'], data['hts'], data['site'])
-    assert len(mtype_df2) > 6
-
-
 @pytest.mark.parametrize('data', [test_data1])
 def test_collection_list(data):
     cl = collection_list(data['base_url'], data['hts'])
@@ -93,21 +86,3 @@ def test_collection_list(data):
 def test_get_data1(data):
     tsdata1 = get_data(data['base_url'], data['hts'], data['site'], data['measurement'], from_date=data['from_date'], to_date=data['to_date'])
     assert len(tsdata1) > 70
-
-
-@pytest.mark.parametrize('data', [test_data1])
-def test_get_data2(data):
-    tsdata2, extra2 = get_data(data['base_url'], data['hts'], data['site'], data['measurement'], from_date=data['from_date'], to_date=data['to_date'], parameters=True)
-    assert (len(tsdata2) > 70) & (len(extra2) > 300)
-
-
-@pytest.mark.parametrize('data', [test_data1])
-def test_get_data3(data):
-    tsdata3 = get_data(data['base_url'], data['hts'], data['site'], 'WQ Sample', from_date=data['from_date'], to_date=data['to_date'])
-    assert len(tsdata3) > 300
-
-
-@pytest.mark.parametrize('data', [test_data1])
-def test_get_data4(data):
-    tsdata4, extra4 = get_data(data['base_url'], data['hts'], data['site'], data['measurement'], from_date=data['from_date'], to_date=data['to_date'], parameters=True, dtl_method=data['dtl_method'])
-    assert (len(tsdata4) > 70) & (len(extra4) > 300) & ('float' in tsdata4.Value.dtype.name)

hilltoppy/utils.py

@@ -10,77 +10,124 @@
     from configparser import ConfigParser
 except ImportError:
     from ConfigParser import SafeConfigParser as ConfigParser
-# import orjson
-# from typing import List, Optional, Dict, Union, Literal
-# from pydantic import BaseModel, Field, HttpUrl, conint, confloat
-# from enum import Enum
+import orjson
+from typing import List, Optional, Dict, Union, Literal
+from pydantic import BaseModel, Field, HttpUrl, conint, confloat
+from enum import Enum
+import urllib
+import urllib.request
+import xml.etree.ElementTree as ET
+from time import sleep
 
 ##############################################
 ### Data models
 
 
-# class TSType(str, Enum):
-#     """
-#     The time series type according to Hilltop.
-#     """
-#     std_series = 'StdSeries'
-#     std_qual_series = 'StdQualSeries'
-#     check_series = 'CheckSeries'
-#
-#
-# class DataType(str, Enum):
-#     """
-#     The data type according to Hilltop.
-#     """
-#     simple_ts = 'SimpleTimeSeries'
-#     hyd_section = 'HydSection'
-#     hyd_facecard = 'HydFacecard'
-#     gauging_results = 'GaugingResults'
-#     wq_data = 'WQData'
-#     wq_sample = 'WQSample'
-#
-#
-# class Interpolation(str, Enum):
-#     """
-#     The method of Measurement and subsequently the kind of interpolation that should be applied to the Measurements.
-#     """
-#     discrete = 'Discrete'
-#     instant = 'Instant'
-#     incremental = 'Incremental'
-#     event = 'Event'
-#
-#
-# class Measurement(BaseModel):
-#     """
-#
-#     """
-#     MeasurementName: str = Field(..., description='The measurement name associated with the DataSource. The DataSourceName has been appended to the MeasurementName (in the form of MeasurementName [DataSourceName]), because this is the requirement for requests to the Hilltop web server.')
-#     # Item: int = Field(..., description='The Measurement item position in the Data when NumItems in the DataSource > 1.')
-#     Units: str
-#     Precision: int = Field(..., description='The precision of the data as the number of decimal places.')
-#     MeasurementGroup: str = Field(..., description="I've only seen Virtual Measurements so far...")
-#     VMStart: datetime = Field(..., description="The start time of the virtual measurement.")
-#     VMFinish: datetime = Field(..., description="The end time of the virtual measurement.")
-#
-#
-# class DataSource(BaseModel):
-#     """
-#
-#     """
-#     DataSourceName: str
-#     # NumItems: int = Field(..., description='The Number of Measurements grouped per GetData request. If this is greater than 1, then any GetData request to a Measurement will return more than one Measurement Data.')
-#     TSType: TSType
-#     DataType: DataType
-#     Interpolation: Interpolation
-#     From: datetime
-#     To: datetime
-#     Measurements: List[Measurement]
+def orjson_dumps(v, *, default):
+    return orjson.dumps(v, default=default, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_OMIT_MICROSECONDS | orjson.OPT_SERIALIZE_NUMPY).decode()
+
+
+
+class TSType(str, Enum):
+    """
+    The time series type according to Hilltop.
+    """
+    std_series = 'StdSeries'
+    std_qual_series = 'StdQualSeries'
+    check_series = 'CheckSeries'
+
+
+class DataType(str, Enum):
+    """
+    The data type according to Hilltop.
+    """
+    simple_ts = 'SimpleTimeSeries'
+    hyd_section = 'HydSection'
+    hyd_facecard = 'HydFacecard'
+    gauging_results = 'GaugingResults'
+    wq_data = 'WQData'
+    wq_sample = 'WQSample'
+    meter_reading = 'MeterReading'
+
+
+class Interpolation(str, Enum):
+    """
+    The method of Measurement and subsequently the kind of interpolation that should be applied to the Measurements.
+    """
+    discrete = 'Discrete'
+    instant = 'Instant'
+    incremental = 'Incremental'
+    event = 'Event'
+
+
+class Measurement(BaseModel):
+    """
+
+    """
+    MeasurementName: str = Field(..., description='The measurement name associated with the DataSource. The DataSourceName has been appended to the MeasurementName (in the form of MeasurementName [DataSourceName]), because this is the requirement for requests to the Hilltop web server.')
+    # Item: int = Field(..., description='The Measurement item position in the Data when NumItems in the DataSource > 1.')
+    Units: str = Field(None, description="The units of the data.")
+    Precision: int = Field(..., description='The precision of the data as the number of decimal places.')
+    Divisor: int = Field(None, description="Divide the data by the divisor to get the appropriate Units.")
+    Precision: int = Field(..., description='The precision of the data as the number of decimal places.')
+    MeasurementGroup: str = Field(None, description="I've only seen Virtual Measurements so far...")
+    VMStart: datetime = Field(None, description="The start time of the virtual measurement.")
+    VMFinish: datetime = Field(None, description="The end time of the virtual measurement.")
+
+    class Config:
+        json_loads = orjson.loads
+        json_dumps = orjson_dumps
+
+
+class DataSource(BaseModel):
+    """
+
+    """
+    SiteName: str
+    DataSourceName: str
+    # NumItems: int = Field(..., description='The Number of Measurements grouped per GetData request. If this is greater than 1, then any GetData request to a Measurement will return more than one Measurement Data.')
+    TSType: TSType
+    DataType: DataType
+    Interpolation: Interpolation
+    From: datetime = None
+    To: datetime = None
+    Measurements: List[Measurement] = None
+
+    class Config:
+        json_loads = orjson.loads
+        json_dumps = orjson_dumps
 
 
 ##############################################
 ### Functions
 
 
+def get_hilltop_xml(url, timeout=60):
+    """
+
+    """
+    counter = [10, 20, 30, None]
+    for c in counter:
+        try:
+            with urllib.request.urlopen(url, timeout=timeout) as req:
+                tree1 = ET.parse(req)
+            break
+        except ET.ParseError:
+            raise ET.ParseError('Could not parse xml. Check to make sure the URL is correct.')
+        except urllib.error.URLError:
+            raise urllib.error.URLError('Could not read the URL. Check to make sure the URL is correct.')
+        except Exception as err:
+            print(str(err))
+
+            if c is None:
+                raise urllib.error.URLError('The Hilltop request tried too many times...the server is probably down')
+
+            print('Trying again in ' + str(c) + ' seconds.')
+            sleep(c)
+
+    return tree1
+
+
 def convert_value(text):
     """
 
@@ -310,8 +357,3 @@ def proc_ht_use_data(ht_data):
 
     return df3
 
-
-# def orjson_dumps(v, *, default):
-#     # orjson.dumps returns bytes, to match standard json.dumps we need to decode
-#     # return orjson.dumps(v, default=default, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_OMIT_MICROSECONDS | orjson.OPT_SERIALIZE_NUMPY | orjson.OPT_INDENT_2).decode()
-#     return orjson.dumps(v, default=default, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_OMIT_MICROSECONDS | orjson.OPT_SERIALIZE_NUMPY).decode()