From 19b49aa83c52a381a65c64fc0715e6865945962e Mon Sep 17 00:00:00 2001 From: Amber-Rigg Date: Tue, 19 Dec 2023 14:14:55 +0000 Subject: [PATCH 01/16] Simplify Table Name Signed-off-by: Amber-Rigg --- .../queries/weather/_weather_query_builder.py | 34 ++--------- .../rtdip_sdk/queries/weather/latest.py | 10 +--- .../python/rtdip_sdk/queries/weather/raw.py | 10 +--- .../queries/weather/weather_query_builder.py | 60 ++++--------------- .../rtdip_sdk/queries/weather/test_latest.py | 11 +--- .../rtdip_sdk/queries/weather/test_raw.py | 16 +---- .../weather/test_weather_query_builder.py | 20 ++----- 7 files changed, 30 insertions(+), 131 deletions(-) diff --git a/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py index 495baf099..36ca09279 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py @@ -34,10 +34,7 @@ def _build_parameters( ) -> dict: if area_type == "grid": raw_parameters = { - "forecast": parameters_dict.get("forecast", None), - "region": parameters_dict.get("region"), - "data_security_level": parameters_dict.get("data_security_level"), - "data_type": parameters_dict.get("data_type"), + "table_name": parameters_dict.get["table_name"], "start_date": parameters_dict["start_date"], "end_date": parameters_dict["end_date"], "max_lat": parameters_dict["max_lat"], @@ -58,10 +55,7 @@ def _build_parameters( if area_type == "point": raw_parameters = { - "forecast": parameters_dict.get("forecast", None), - "region": parameters_dict.get("region"), - "data_security_level": parameters_dict.get("data_security_level"), - "data_type": parameters_dict.get("data_type"), + "table_name": parameters_dict.get["table_name"], "start_date": parameters_dict["start_date"], "end_date": parameters_dict["end_date"], "lat": parameters_dict["lat"], @@ -84,11 +78,7 @@ def _build_parameters( def _raw_query_grid(parameters_dict: dict) -> str: raw_query_grid = ( "SELECT * FROM " - "{% if source is defined and source is not none %}" - "`{{ source|lower }}` " - "{% else %}" - "`{{ forecast|lower }}`.`weather`.`{{ region|lower }}_weather_{{ data_security_level|lower }}_events_{{ data_type|lower }}` " - "{% endif %}" + "`{{table_name|lower }}`" 'WHERE `{{ timestamp_column }}` BETWEEN to_timestamp("{{ start_date }}") AND to_timestamp("{{ end_date }}")' "AND `{{ latitude_column }}` > '{{ min_lat}}' " "AND `{{ latitude_column }}` < '{{ max_lat}}' " @@ -112,11 +102,7 @@ def _raw_query_grid(parameters_dict: dict) -> str: def _raw_query_point(parameters_dict: dict) -> str: raw_query_point = ( "SELECT * FROM " - "{% if source is defined and source is not none %}" - "`{{ source|lower }}` " - "{% else %}" - "`{{ forecast|lower }}`.`weather`.`{{ region|lower }}_weather_{{ data_security_level|lower }}_events_{{ data_type|lower }}` " - "{% endif %}" + "`{{table_name|lower }}`" 'WHERE `{{ timestamp_column }}` BETWEEN to_timestamp("{{ start_date }}") AND to_timestamp("{{ end_date }}")' "AND `{{ latitude_column }}` > '{{lat}}' " "AND `{{ longitude_column }}` > '{{lon}}' " @@ -138,11 +124,7 @@ def _raw_query_point(parameters_dict: dict) -> str: def _latest_query_grid(parameters_dict: dict) -> str: latest_query_grid = ( "SELECT * FROM " - "{% if source is defined and source is not none %}" - "`{{ source|lower }}` " - "{% else %}" - "`{{ forecast|lower }}`.`weather`.`{{ region|lower }}_weather_{{ data_security_level|lower }}_events_{{ data_type|lower }}` " - "{% endif %}" + "`{{table_name|lower }}`" "WHERE `{{ latitude_column }}` > '{{ min_lat}}' " "AND `{{ latitude_column }}` < '{{ max_lat}}' " "AND `{{ longitude_column }}` > '{{ min_lon}}' " @@ -165,11 +147,7 @@ def _latest_query_grid(parameters_dict: dict) -> str: def _latest_query_point(parameters_dict: dict) -> str: latest_query_point = ( "SELECT * FROM " - "{% if source is defined and source is not none %}" - "`{{ source|lower }}` " - "{% else %}" - "`{{ forecast|lower }}`.`weather`.`{{ region|lower }}_weather_{{ data_security_level|lower }}_events_{{ data_type|lower }}` " - "{% endif %}" + "`{{table_name|lower }}`" "WHERE `{{ latitude_column }}` == '{{lat}}' " "AND `{{ longitude_column }}` == '{{lon}}' " "{% if source is defined and source is not none %}" diff --git a/src/sdk/python/rtdip_sdk/queries/weather/latest.py b/src/sdk/python/rtdip_sdk/queries/weather/latest.py index 611760b43..6ac6e6ce2 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/latest.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/latest.py @@ -34,10 +34,7 @@ def get_grid(connection: object, parameters_dict: dict) -> pd.DataFrame: parameters_dict: A dictionary of parameters (see Attributes table below) Attributes: - forecast (str): Business unit - region (str): Region - data_security_level (str): Level of data security - data_type (str): Type of the data (float, integer, double, string) + table_name (str): Name of the table max_lat (float): Maximum latitude max_lon (float): Maximum longitude min_lat (float): Minimum latitude @@ -84,10 +81,7 @@ def get_point(connection: object, parameters_dict: dict) -> pd.DataFrame: parameters_dict: A dictionary of parameters (see Attributes table below) Attributes: - forecast (str): Business unit - region (str): Region - data_security_level (str): Level of data security - data_type (str): Type of the data (float, integer, double, string) + table_name (str): Name of the table lat (float): latitude lon (float): longitude source (optional str): Source of the data ie ECMWF diff --git a/src/sdk/python/rtdip_sdk/queries/weather/raw.py b/src/sdk/python/rtdip_sdk/queries/weather/raw.py index 9941f2789..78d90a366 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/raw.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/raw.py @@ -34,10 +34,7 @@ def get_grid(connection: object, parameters_dict: dict) -> pd.DataFrame: parameters_dict: A dictionary of parameters (see Attributes table below) Attributes: - forecast (str): Business unit - region (str): Region - data_security_level (str): Level of data security - data_type (str): Type of the data (float, integer, double, string) + table_name (str): Name of the table start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) max_lat (float): Maximum latitude @@ -89,10 +86,7 @@ def get_point(connection: object, parameters_dict: dict) -> pd.DataFrame: parameters_dict: A dictionary of parameters (see Attributes table below) Attributes: - forecast (str): Business unit - region (str): Region - data_security_level (str): Level of data security - data_type (str): Type of the data (float, integer, double, string) + table_name (str): Name of the table start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) lat (float): latitude diff --git a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py index eb78fde10..e69640e24 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py @@ -66,10 +66,7 @@ def source( def raw_point( self, - forecast: str, - region: str, - data_security_level: str, - data_type: str, + table_name: str, start_date: str, end_date: str, lat: float, @@ -83,10 +80,7 @@ def raw_point( A function to return back raw data for a point. Args: - forecast (str): Business unit - region (str): Region - data_security_level (str): Level of data security - data_type (str): Type of the data (float, integer, double, string) + table_name (str): Table name start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) lat (float): latitude @@ -100,10 +94,7 @@ def raw_point( DataFrame: A dataframe of raw timeseries data. """ raw_parameters = { - "forecast": forecast, - "region": region, - "data_security_level": data_security_level, - "data_type": data_type, + "table_name": table_name, "start_date": start_date, "end_date": end_date, "lat": lat, @@ -119,10 +110,7 @@ def raw_point( def latest_point( self, - forecast: str, - region: str, - data_security_level: str, - data_type: str, + table_name: str, lat: float, lon: float, source: str = None, @@ -132,10 +120,7 @@ def latest_point( A function to return back the latest data for a point. Args: - forecast (str): Business unit - region (str): Region - data_security_level (str): Level of data security - data_type (str): Type of the data (float, integer, double, string) + table_name (str): Table name lat (float): latitude lon (float): longitude source (optional str): Source of the data ie ECMWF @@ -145,10 +130,7 @@ def latest_point( DataFrame: A dataframe of raw timeseries data. """ raw_parameters = { - "forecast": forecast, - "region": region, - "data_security_level": data_security_level, - "data_type": data_type, + "table_name": table_name, "lat": lat, "lon": lon, "source": source, @@ -160,10 +142,7 @@ def latest_point( def raw_grid( # NOSONAR self, # NOSONAR - forecast: str, - region: str, - data_security_level: str, - data_type: str, + table_name: str, start_date: str, end_date: str, min_lat: float, @@ -179,10 +158,7 @@ def raw_grid( # NOSONAR A function to return back raw data for a point. Args: - forecast (str): Business unit - region (str): Region - data_security_level (str): Level of data security - data_type (str): Type of the data (float, integer, double, string) + table_name (str): Table name start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) min_lat (float): Min latitude @@ -198,10 +174,7 @@ def raw_grid( # NOSONAR DataFrame: A dataframe of raw timeseries data. """ raw_parameters = { - "forecast": forecast, - "region": region, - "data_security_level": data_security_level, - "data_type": data_type, + "table_name": table_name, "start_date": start_date, "end_date": end_date, "min_lat": min_lat, @@ -219,10 +192,7 @@ def raw_grid( # NOSONAR def latest_grid( self, - forecast: str, - region: str, - data_security_level: str, - data_type: str, + table_name: str, min_lat: float, min_lon: float, max_lat: float, @@ -234,10 +204,7 @@ def latest_grid( A function to return back the latest data for a point. Args: - forecast (str): Business unit - region (str): Region - data_security_level (str): Level of data security - data_type (str): Type of the data (float, integer, double, string) + table_name (str): Table name min_lat (float): Min latitude min_lon (float): Min longitude max_lat (float): Max latitude @@ -249,10 +216,7 @@ def latest_grid( DataFrame: A dataframe of raw timeseries data. """ raw_parameters = { - "forecast": forecast, - "region": region, - "data_security_level": data_security_level, - "data_type": data_type, + "table_name": table_name, "min_lat": min_lat, "min_lon": min_lon, "max_lat": max_lat, diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py index 8fdb641df..1130d4aa0 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py @@ -42,11 +42,7 @@ MOCKED_QUERY_OFFSET_LIMIT = "LIMIT 10 OFFSET 10 " MOCKED_PARAMETER_DICT_GRID = { - "forecast": "mocked-forecast", - "region": "mocked-region", - "data_security_level": "mocked-data-security-level", - "data_type": "mocked-data-type", - "min_lat": 0, + "table_name": "mocked-asset", "max_lat": 0.1, "min_lon": 0, "max_lon": 0.1, @@ -55,10 +51,7 @@ } MOCKED_PARAMETER_DICT_POINT = { - "forecast": "mocked-forecast", - "region": "mocked-region", - "data_security_level": "mocked-data-security-level", - "data_type": "mocked-data-type", + "table_name": "mocked-asset", "lat": 0, "lon": 0, "start_date": "2020-01-01", diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py index 41aa0af29..0bfcd8ce6 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py @@ -42,10 +42,7 @@ MOCKED_QUERY_OFFSET_LIMIT = "LIMIT 10 OFFSET 10 " MOCKED_PARAMETER_DICT_GRID = { - "forecast": "mocked-forecast", - "region": "mocked-region", - "data_security_level": "mocked-data-security-level", - "data_type": "mocked-data-type", + "table_name": "mocked-asset", "min_lat": 1.1, "max_lat": 1.1, "min_lon": 1.1, @@ -56,10 +53,7 @@ } MOCKED_PARAMETER_DICT_POINT = { - "forecast": "mocked-forecast", - "region": "mocked-region", - "data_security_level": "mocked-data-security-level", - "data_type": "mocked-data-type", + "table_name": "mocked-asset", "lat": 1.1, "lon": 1.1, "start_date": "2020-01-01", @@ -67,12 +61,6 @@ "timestamp_column": "EventTime", } -MOCKED_NO_TAG_QUERY = "SELECT * FROM `mocked-business-unit`.`sensors`.`mocked-asset_mocked-data-security-level_events_raw` ORDER BY `TagName` " -MOCKED_PARAMETER_NO_TAGS_DICT = { - "forecast": "mocked-forecast", - "region": "mocked-region", - "data_security_level": "mocked-data-security-level", -} def test_raw_grid(mocker: MockerFixture): diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_weather_query_builder.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_weather_query_builder.py index 31ae7b67a..0ca562064 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_weather_query_builder.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_weather_query_builder.py @@ -33,10 +33,7 @@ def test_weather_query_builder_raw_point(mocker: MockerFixture): .connect(MOCK_CONNECTION) .source(MOCK_TABLE, status_column=None) .raw_point( - forecast="mock_forecast", - region="mock_region", - data_security_level="mock_data_security_level", - data_type="mock_data_type", + table_name="mock_table", start_date="2021-01-01", end_date="2021-01-02", lat=0.1, @@ -57,10 +54,7 @@ def test_query_builder_latest_point(mocker: MockerFixture): .connect(MOCK_CONNECTION) .source(MOCK_TABLE, status_column=None) .latest_point( - forecast="mock_forecast", - region="mock_region", - data_security_level="mock_data_security_level", - data_type="mock_data_type", + table_name="mock_table", lat=0.1, lon=0.1, ) @@ -79,10 +73,7 @@ def test_weather_query_builder_raw_grid(mocker: MockerFixture): .connect(MOCK_CONNECTION) .source(MOCK_TABLE, status_column=None) .raw_grid( - forecast="mock_forecast", - region="mock_region", - data_security_level="mock_data_security_level", - data_type="mock_data_type", + table_name="mock_table", start_date="2021-01-01", end_date="2021-01-02", min_lat=0.1, @@ -105,10 +96,7 @@ def test_query_builder_latest_grid(mocker: MockerFixture): .connect(MOCK_CONNECTION) .source(MOCK_TABLE, status_column=None) .latest_grid( - forecast="mock_forecast", - region="mock_region", - data_security_level="mock_data_security_level", - data_type="mock_data_type", + table_name="mock_table", min_lat=0.1, max_lat=0.1, min_lon=0.1, From 14d345aad97e14864938f29ab8c4a56c4ac4a348 Mon Sep 17 00:00:00 2001 From: Amber-Rigg Date: Tue, 19 Dec 2023 14:34:26 +0000 Subject: [PATCH 02/16] RAW query on Forecast Run Time Signed-off-by: Amber-Rigg --- .../queries/weather/_weather_query_builder.py | 13 +++++++++++++ src/sdk/python/rtdip_sdk/queries/weather/raw.py | 4 ++++ .../queries/weather/weather_query_builder.py | 15 +++++++++++++++ .../python/rtdip_sdk/queries/weather/test_raw.py | 6 ++++++ .../queries/weather/test_weather_query_builder.py | 4 ++++ 5 files changed, 42 insertions(+) diff --git a/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py index 36ca09279..5aa77202c 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py @@ -37,6 +37,8 @@ def _build_parameters( "table_name": parameters_dict.get["table_name"], "start_date": parameters_dict["start_date"], "end_date": parameters_dict["end_date"], + "forecast_run_start_date": parameters_dict["forecast_run_start_date"], + "forecast_run_end_date": parameters_dict["forecast_run_end_date"], "max_lat": parameters_dict["max_lat"], "max_lon": parameters_dict["max_lon"], "min_lat": parameters_dict["min_lat"], @@ -51,6 +53,10 @@ def _build_parameters( raw_parameters["timestamp_column"] = parameters_dict.get( "timestamp_column", "EventTime" ) + raw_parameters["forecast_run_timestamp_column"] = parameters_dict.get( + "forecast_run_timestamp_column", "EnqueuedTime" + ) + raw_parameters["include_status"] = False if area_type == "point": @@ -58,6 +64,8 @@ def _build_parameters( "table_name": parameters_dict.get["table_name"], "start_date": parameters_dict["start_date"], "end_date": parameters_dict["end_date"], + "forecast_run_start_date": parameters_dict["forecast_run_start_date"], + "forecast_run_end_date": parameters_dict["forecast_run_end_date"], "lat": parameters_dict["lat"], "lon": parameters_dict["lon"], "source": parameters_dict.get("source", None), @@ -70,6 +78,9 @@ def _build_parameters( raw_parameters["timestamp_column"] = parameters_dict.get( "timestamp_column", "EventTime" ) + raw_parameters["forecast_run_timestamp_column"] = parameters_dict.get( + "forecast_run_timestamp_column", "EnqueuedTime" + ) raw_parameters["include_status"] = False return raw_parameters @@ -80,6 +91,7 @@ def _raw_query_grid(parameters_dict: dict) -> str: "SELECT * FROM " "`{{table_name|lower }}`" 'WHERE `{{ timestamp_column }}` BETWEEN to_timestamp("{{ start_date }}") AND to_timestamp("{{ end_date }}")' + 'AND `{{ forecast_run_timestamp_column }}` BETWEEN to_timestamp("{{ forecast_run_timestamp_start_date }}") AND to_timestamp("{{ forecast_run_timestamp_end_date }}")' "AND `{{ latitude_column }}` > '{{ min_lat}}' " "AND `{{ latitude_column }}` < '{{ max_lat}}' " "AND `{{ longitude_column }}` > '{{ min_lon}}' " @@ -104,6 +116,7 @@ def _raw_query_point(parameters_dict: dict) -> str: "SELECT * FROM " "`{{table_name|lower }}`" 'WHERE `{{ timestamp_column }}` BETWEEN to_timestamp("{{ start_date }}") AND to_timestamp("{{ end_date }}")' + 'AND `{{ forecast_run_timestamp_column }}` BETWEEN to_timestamp("{{ forecast_run_timestamp_start_date }}") AND to_timestamp("{{ forecast_run_timestamp_end_date }}")' "AND `{{ latitude_column }}` > '{{lat}}' " "AND `{{ longitude_column }}` > '{{lon}}' " "{% if source is defined and source is not none %}" diff --git a/src/sdk/python/rtdip_sdk/queries/weather/raw.py b/src/sdk/python/rtdip_sdk/queries/weather/raw.py index 78d90a366..3876c752d 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/raw.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/raw.py @@ -37,6 +37,8 @@ def get_grid(connection: object, parameters_dict: dict) -> pd.DataFrame: table_name (str): Name of the table start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) + forecast_run_start_date (str): Start date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) + forecast_run_end_date (str): End date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) max_lat (float): Maximum latitude max_lon (float): Maximum longitude min_lat (float): Minimum latitude @@ -89,6 +91,8 @@ def get_point(connection: object, parameters_dict: dict) -> pd.DataFrame: table_name (str): Name of the table start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) + forecast_run_start_date (str): Start date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) + forecast_run_end_date (str): End date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) lat (float): latitude lon (float): longitude source (optional str): Source of the data ie ECMWF diff --git a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py index e69640e24..952f4de6b 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py @@ -44,6 +44,7 @@ def source( source: str, tagname_column: str = "TagName", timestamp_column: str = "EventTime", + forecast_run_timestamp_column: str = "EnqueuedTime", status_column: Union[str, None] = "Status", value_column: str = "Value", ): @@ -54,12 +55,14 @@ def source( source (str): Source of the query can be a Unity Catalog table, Hive metastore table or path tagname_column (optional str): The column name in the source that contains the tagnames or series timestamp_column (optional str): The timestamp column name in the source + forecast_run_timestamp_column (optional str): The forecast run timestamp column name in the source status_column (optional str): The status column name in the source indicating `Good` or `Bad`. If this is not available, specify `None` value_column (optional str): The value column name in the source which is normally a float or string value for the time series event """ self.data_source = "`.`".join(source.split(".")) self.tagname_column = tagname_column self.timestamp_column = timestamp_column + self.forecast_run_timestamp_column = forecast_run_timestamp_column self.status_column = status_column self.value_column = value_column return self @@ -69,6 +72,8 @@ def raw_point( table_name: str, start_date: str, end_date: str, + forecast_run_start_date: str, + forecast_run_end_date: str, lat: float, lon: float, source: str = None, @@ -83,6 +88,8 @@ def raw_point( table_name (str): Table name start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) + forecast_run_start_date (str): Start date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) + forecast_run_end_date (str): End date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) lat (float): latitude lon (float): longitude source (optional str): Source of the data ie ECMWF @@ -97,6 +104,8 @@ def raw_point( "table_name": table_name, "start_date": start_date, "end_date": end_date, + "forecast_run_start_date": forecast_run_start_date, + "forecast_run_end_date": forecast_run_end_date, "lat": lat, "lon": lon, "source": source, @@ -145,6 +154,8 @@ def raw_grid( # NOSONAR table_name: str, start_date: str, end_date: str, + forecast_run_start_date: str, + forecast_run_end_date: str, min_lat: float, min_lon: float, max_lat: float, @@ -161,6 +172,8 @@ def raw_grid( # NOSONAR table_name (str): Table name start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) + forecast_run_start_date (str): Start date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) + forecast_run_end_date (str): End date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) min_lat (float): Min latitude min_lon (float): Min longitude max_lat (float): Max latitude @@ -177,6 +190,8 @@ def raw_grid( # NOSONAR "table_name": table_name, "start_date": start_date, "end_date": end_date, + "forecast_run_start_date": forecast_run_start_date, + "forecast_run_end_date": forecast_run_end_date, "min_lat": min_lat, "min_lon": min_lon, "max_lat": max_lat, diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py index 0bfcd8ce6..59f4921c6 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py @@ -49,7 +49,10 @@ "max_lon": 1.1, "start_date": "2020-01-01", "end_date": "2020-01-02", + "forecast_run_start_date": "2020-01-01", + "forecast_run_end_date": "2020-01-02", "timestamp_column": "EventTime", + "forecast_run_timestamp_column": "EnqueuedTime", } MOCKED_PARAMETER_DICT_POINT = { @@ -58,7 +61,10 @@ "lon": 1.1, "start_date": "2020-01-01", "end_date": "2020-01-02", + "forecast_run_start_date": "2020-01-01", + "forecast_run_end_date": "2020-01-02", "timestamp_column": "EventTime", + "forecast_run_timestamp_column": "EnqueuedTime", } diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_weather_query_builder.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_weather_query_builder.py index 0ca562064..0d084011b 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_weather_query_builder.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_weather_query_builder.py @@ -36,6 +36,8 @@ def test_weather_query_builder_raw_point(mocker: MockerFixture): table_name="mock_table", start_date="2021-01-01", end_date="2021-01-02", + forecast_run_start_date="2021-01-01", + forecast_run_end_date="2021-01-02", lat=0.1, lon=0.1, ) @@ -76,6 +78,8 @@ def test_weather_query_builder_raw_grid(mocker: MockerFixture): table_name="mock_table", start_date="2021-01-01", end_date="2021-01-02", + forecast_run_start_date="2021-01-01", + forecast_run_end_date="2021-01-02", min_lat=0.1, max_lat=0.1, min_lon=0.1, From 353de8dfa4742b37b4ae0ca6c5a5f7ca4e199eab Mon Sep 17 00:00:00 2001 From: Amber-Rigg Date: Tue, 19 Dec 2023 14:41:12 +0000 Subject: [PATCH 03/16] Update Tests Signed-off-by: Amber-Rigg --- tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py | 4 ++-- tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py index 1130d4aa0..5e0990a12 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py @@ -37,8 +37,8 @@ DATABRICKS_SQL_CONNECT = "databricks.sql.connect" DATABRICKS_SQL_CONNECT_CURSOR = "databricks.sql.connect.cursor" INTERPOLATION_METHOD = "test/test/test" -MOCKED_QUERY_GRID = "SELECT * FROM `mocked-forecast`.`weather`.`mocked-region_weather_mocked-data-security-level_events_mocked-data-type` WHERE `Latitude` > '0' AND `Latitude` < '0.1' AND `Longitude` > '0' AND`Longitude` < '0.1' ORDER BY `TagName` " -MOCKED_QUERY_POINT = "SELECT * FROM `mocked-forecast`.`weather`.`mocked-region_weather_mocked-data-security-level_events_mocked-data-type` WHERE `Latitude` == '0' AND `Longitude` == '0' ORDER BY `TagName` " +MOCKED_QUERY_GRID = "SELECT * FROM `mocked-asset` WHERE `Latitude` > '0' AND `Latitude` < '0.1' AND `Longitude` > '0' AND`Longitude` < '0.1' ORDER BY `TagName` " +MOCKED_QUERY_POINT = "SELECT * FROM `mocked-asset` WHERE `Latitude` == '0' AND `Longitude` == '0' ORDER BY `TagName` " MOCKED_QUERY_OFFSET_LIMIT = "LIMIT 10 OFFSET 10 " MOCKED_PARAMETER_DICT_GRID = { diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py index 59f4921c6..20ad5883e 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py @@ -37,8 +37,8 @@ DATABRICKS_SQL_CONNECT = "databricks.sql.connect" DATABRICKS_SQL_CONNECT_CURSOR = "databricks.sql.connect.cursor" INTERPOLATION_METHOD = "test/test/test" -MOCKED_QUERY_GRID = "SELECT * FROM `mocked-forecast`.`weather`.`mocked-region_weather_mocked-data-security-level_events_mocked-data-type` WHERE `EventTime` BETWEEN to_timestamp(\"2020-01-01\") AND to_timestamp(\"2020-01-02\")AND `Latitude` > '1.1' AND `Latitude` < '1.1' AND `Longitude` > '1.1' AND`Longitude` < '1.1' ORDER BY `TagName` " -MOCKED_QUERY_POINT = "SELECT * FROM `mocked-forecast`.`weather`.`mocked-region_weather_mocked-data-security-level_events_mocked-data-type` WHERE `EventTime` BETWEEN to_timestamp(\"2020-01-01\") AND to_timestamp(\"2020-01-02\")AND `Latitude` > '1.1' AND `Longitude` > '1.1' ORDER BY `TagName` " +MOCKED_QUERY_GRID = "SELECT * FROM `mocked-asset` WHERE `EventTime` BETWEEN to_timestamp(\"2020-01-01\") AND to_timestamp(\"2020-01-02\")AND `Latitude` > '1.1' AND `Latitude` < '1.1' AND `Longitude` > '1.1' AND`Longitude` < '1.1' ORDER BY `TagName` " +MOCKED_QUERY_POINT = "SELECT * FROM `mocked-asset` WHERE `EventTime` BETWEEN to_timestamp(\"2020-01-01\") AND to_timestamp(\"2020-01-02\")AND `Latitude` > '1.1' AND `Longitude` > '1.1' ORDER BY `TagName` " MOCKED_QUERY_OFFSET_LIMIT = "LIMIT 10 OFFSET 10 " MOCKED_PARAMETER_DICT_GRID = { From b0076d66414bfddb581890ce57aa7b5f85878fbb Mon Sep 17 00:00:00 2001 From: Amber-Rigg Date: Tue, 19 Dec 2023 15:08:43 +0000 Subject: [PATCH 04/16] Update tests Signed-off-by: Amber-Rigg --- .../queries/weather/_weather_query_builder.py | 42 +++++++++---------- .../rtdip_sdk/queries/weather/test_latest.py | 10 ++--- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py index 5aa77202c..9c15a6f29 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py @@ -34,7 +34,7 @@ def _build_parameters( ) -> dict: if area_type == "grid": raw_parameters = { - "table_name": parameters_dict.get["table_name"], + "table_name": parameters_dict["table_name"], "start_date": parameters_dict["start_date"], "end_date": parameters_dict["end_date"], "forecast_run_start_date": parameters_dict["forecast_run_start_date"], @@ -61,7 +61,7 @@ def _build_parameters( if area_type == "point": raw_parameters = { - "table_name": parameters_dict.get["table_name"], + "table_name": parameters_dict["table_name"], "start_date": parameters_dict["start_date"], "end_date": parameters_dict["end_date"], "forecast_run_start_date": parameters_dict["forecast_run_start_date"], @@ -90,16 +90,16 @@ def _raw_query_grid(parameters_dict: dict) -> str: raw_query_grid = ( "SELECT * FROM " "`{{table_name|lower }}`" - 'WHERE `{{ timestamp_column }}` BETWEEN to_timestamp("{{ start_date }}") AND to_timestamp("{{ end_date }}")' - 'AND `{{ forecast_run_timestamp_column }}` BETWEEN to_timestamp("{{ forecast_run_timestamp_start_date }}") AND to_timestamp("{{ forecast_run_timestamp_end_date }}")' - "AND `{{ latitude_column }}` > '{{ min_lat}}' " - "AND `{{ latitude_column }}` < '{{ max_lat}}' " - "AND `{{ longitude_column }}` > '{{ min_lon}}' " - "AND`{{ longitude_column }}` < '{{ max_lon}}' " + "WHERE {{ timestamp_column }} BETWEEN to_timestamp('{{ start_date }}') AND to_timestamp('{{ end_date }}') " + "AND {{ forecast_run_timestamp_column }}` BETWEEN to_timestamp('{{ forecast_run_timestamp_start_date }}') AND to_timestamp('{{ forecast_run_timestamp_end_date }}') " + "AND {{ latitude_column }} > {{ min_lat}} " + "AND {{ latitude_column }} < {{ max_lat}} " + "AND {{ longitude_column }} > {{ min_lon}} " + "AND {{ longitude_column }} < {{ max_lon}} " "{% if source is defined and source is not none %}" "AND SOURCE = '{{ source }}' " "{% endif %}" - "ORDER BY `{{ tagname_column }}` " + "ORDER BY {{ tagname_column }} " "{% if limit is defined and limit is not none %}" "LIMIT {{ limit }} " "{% endif %}" @@ -115,14 +115,14 @@ def _raw_query_point(parameters_dict: dict) -> str: raw_query_point = ( "SELECT * FROM " "`{{table_name|lower }}`" - 'WHERE `{{ timestamp_column }}` BETWEEN to_timestamp("{{ start_date }}") AND to_timestamp("{{ end_date }}")' - 'AND `{{ forecast_run_timestamp_column }}` BETWEEN to_timestamp("{{ forecast_run_timestamp_start_date }}") AND to_timestamp("{{ forecast_run_timestamp_end_date }}")' - "AND `{{ latitude_column }}` > '{{lat}}' " - "AND `{{ longitude_column }}` > '{{lon}}' " + "WHERE {{ timestamp_column }} BETWEEN to_timestamp('{{ start_date }}') AND to_timestamp('{{ end_date }}') " + "AND {{ forecast_run_timestamp_column }}` BETWEEN to_timestamp('{{ forecast_run_timestamp_start_date }}') AND to_timestamp('{{ forecast_run_timestamp_end_date }}') " + "AND {{ latitude_column }} > {{lat}} " + "AND {{ longitude_column }} > {{lon}} " "{% if source is defined and source is not none %}" "AND SOURCE = '{{ source }}' " "{% endif %}" - "ORDER BY `{{ tagname_column }}` " + "ORDER BY {{ tagname_column }} " "{% if limit is defined and limit is not none %}" "LIMIT {{ limit }} " "{% endif %}" @@ -137,11 +137,11 @@ def _raw_query_point(parameters_dict: dict) -> str: def _latest_query_grid(parameters_dict: dict) -> str: latest_query_grid = ( "SELECT * FROM " - "`{{table_name|lower }}`" - "WHERE `{{ latitude_column }}` > '{{ min_lat}}' " - "AND `{{ latitude_column }}` < '{{ max_lat}}' " - "AND `{{ longitude_column }}` > '{{ min_lon}}' " - "AND`{{ longitude_column }}` < '{{ max_lon}}' " + "`{{table_name|lower }}`" + "WHERE {{ latitude_column }} > {{ min_lat}} " + "AND {{ latitude_column }} < {{ max_lat}} " + "AND {{ longitude_column }} > {{ min_lon}} " + "AND {{ longitude_column }} < {{ max_lon}} " "{% if source is defined and source is not none %}" "AND SOURCE = '{{ source }}' " "{% endif %}" @@ -161,8 +161,8 @@ def _latest_query_point(parameters_dict: dict) -> str: latest_query_point = ( "SELECT * FROM " "`{{table_name|lower }}`" - "WHERE `{{ latitude_column }}` == '{{lat}}' " - "AND `{{ longitude_column }}` == '{{lon}}' " + "WHERE {{ latitude_column }} > {{lat}} " + "AND {{ longitude_column }} > {{lon}} " "{% if source is defined and source is not none %}" "AND SOURCE = '{{ source }}' " "{% endif %}" diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py index 5e0990a12..4ec51a401 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py @@ -48,6 +48,8 @@ "max_lon": 0.1, "start_date": "2020-01-01", "end_date": "2020-01-02", + "forecast_run_start_date": "2020-01-01", + "forecast_run_end_date": "2020-01-02", } MOCKED_PARAMETER_DICT_POINT = { @@ -56,14 +58,10 @@ "lon": 0, "start_date": "2020-01-01", "end_date": "2020-01-02", + "forecast_run_start_date": "2020-01-01", + "forecast_run_end_date": "2020-01-02", } -MOCKED_NO_TAG_QUERY = "SELECT * FROM `mocked-business-unit`.`sensors`.`mocked-asset_mocked-data-security-level_events_latest` ORDER BY `TagName` " -MOCKED_PARAMETER_NO_TAGS_DICT = { - "forecast": "mocked-forecast", - "region": "mocked-region", - "data_security_level": "mocked-data-security-level", -} def test_latest_grid(mocker: MockerFixture): From cc2ca1a7c3d44b0326b00051c80d879837319482 Mon Sep 17 00:00:00 2001 From: Amber-Rigg Date: Tue, 19 Dec 2023 15:27:39 +0000 Subject: [PATCH 05/16] Update Tests Signed-off-by: Amber-Rigg --- .../queries/weather/_weather_query_builder.py | 25 +++++++++++++------ .../rtdip_sdk/queries/weather/test_latest.py | 13 ++-------- .../rtdip_sdk/queries/weather/test_raw.py | 5 ++-- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py index 9c15a6f29..64f0411d1 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py @@ -35,10 +35,6 @@ def _build_parameters( if area_type == "grid": raw_parameters = { "table_name": parameters_dict["table_name"], - "start_date": parameters_dict["start_date"], - "end_date": parameters_dict["end_date"], - "forecast_run_start_date": parameters_dict["forecast_run_start_date"], - "forecast_run_end_date": parameters_dict["forecast_run_end_date"], "max_lat": parameters_dict["max_lat"], "max_lon": parameters_dict["max_lon"], "min_lat": parameters_dict["min_lat"], @@ -50,6 +46,14 @@ def _build_parameters( "tagname_column": parameters_dict.get("tagname_column", "TagName"), } if table_type == "raw": + raw_parameters["start_date"] = parameters_dict["start_date"] + raw_parameters["end_date"] = parameters_dict["end_date"] + raw_parameters["forecast_run_start_date"] = parameters_dict[ + "forecast_run_start_date" + ] + raw_parameters["forecast_run_end_date"] = parameters_dict[ + "forecast_run_end_date" + ] raw_parameters["timestamp_column"] = parameters_dict.get( "timestamp_column", "EventTime" ) @@ -62,10 +66,6 @@ def _build_parameters( if area_type == "point": raw_parameters = { "table_name": parameters_dict["table_name"], - "start_date": parameters_dict["start_date"], - "end_date": parameters_dict["end_date"], - "forecast_run_start_date": parameters_dict["forecast_run_start_date"], - "forecast_run_end_date": parameters_dict["forecast_run_end_date"], "lat": parameters_dict["lat"], "lon": parameters_dict["lon"], "source": parameters_dict.get("source", None), @@ -75,12 +75,21 @@ def _build_parameters( "tagname_column": parameters_dict.get("tagname_column", "TagName"), } if table_type == "raw": + raw_parameters["start_date"] = parameters_dict["start_date"] + raw_parameters["end_date"] = parameters_dict["end_date"] + raw_parameters["forecast_run_start_date"] = parameters_dict[ + "forecast_run_start_date" + ] + raw_parameters["forecast_run_end_date"] = parameters_dict[ + "forecast_run_end_date" + ] raw_parameters["timestamp_column"] = parameters_dict.get( "timestamp_column", "EventTime" ) raw_parameters["forecast_run_timestamp_column"] = parameters_dict.get( "forecast_run_timestamp_column", "EnqueuedTime" ) + raw_parameters["include_status"] = False return raw_parameters diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py index 4ec51a401..22cca20fd 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py @@ -37,8 +37,8 @@ DATABRICKS_SQL_CONNECT = "databricks.sql.connect" DATABRICKS_SQL_CONNECT_CURSOR = "databricks.sql.connect.cursor" INTERPOLATION_METHOD = "test/test/test" -MOCKED_QUERY_GRID = "SELECT * FROM `mocked-asset` WHERE `Latitude` > '0' AND `Latitude` < '0.1' AND `Longitude` > '0' AND`Longitude` < '0.1' ORDER BY `TagName` " -MOCKED_QUERY_POINT = "SELECT * FROM `mocked-asset` WHERE `Latitude` == '0' AND `Longitude` == '0' ORDER BY `TagName` " +MOCKED_QUERY_GRID = "SELECT * FROM `mocked-asset` WHERE Latitude > 0 AND Latitude < 0.1 AND Longitude > 0 AND Longitude < 0.1 ORDER BY TagName" +MOCKED_QUERY_POINT = "SELECT * FROM `mocked-asset` WHERE Latitude == 0 AND Longitude == 0 ORDER BY TagName" MOCKED_QUERY_OFFSET_LIMIT = "LIMIT 10 OFFSET 10 " MOCKED_PARAMETER_DICT_GRID = { @@ -46,24 +46,15 @@ "max_lat": 0.1, "min_lon": 0, "max_lon": 0.1, - "start_date": "2020-01-01", - "end_date": "2020-01-02", - "forecast_run_start_date": "2020-01-01", - "forecast_run_end_date": "2020-01-02", } MOCKED_PARAMETER_DICT_POINT = { "table_name": "mocked-asset", "lat": 0, "lon": 0, - "start_date": "2020-01-01", - "end_date": "2020-01-02", - "forecast_run_start_date": "2020-01-01", - "forecast_run_end_date": "2020-01-02", } - def test_latest_grid(mocker: MockerFixture): mocked_cursor = mocker.spy(MockedDBConnection, "cursor") mocked_connection_close = mocker.spy(MockedDBConnection, "close") diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py index 20ad5883e..28444b724 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py @@ -37,8 +37,8 @@ DATABRICKS_SQL_CONNECT = "databricks.sql.connect" DATABRICKS_SQL_CONNECT_CURSOR = "databricks.sql.connect.cursor" INTERPOLATION_METHOD = "test/test/test" -MOCKED_QUERY_GRID = "SELECT * FROM `mocked-asset` WHERE `EventTime` BETWEEN to_timestamp(\"2020-01-01\") AND to_timestamp(\"2020-01-02\")AND `Latitude` > '1.1' AND `Latitude` < '1.1' AND `Longitude` > '1.1' AND`Longitude` < '1.1' ORDER BY `TagName` " -MOCKED_QUERY_POINT = "SELECT * FROM `mocked-asset` WHERE `EventTime` BETWEEN to_timestamp(\"2020-01-01\") AND to_timestamp(\"2020-01-02\")AND `Latitude` > '1.1' AND `Longitude` > '1.1' ORDER BY `TagName` " +MOCKED_QUERY_GRID = "SELECT * FROM `mocked-asset` WHERE EventTime BETWEEN to_timestamp('2020-01-01') AND to_timestamp('2020-01-02')AND Latitude > 1.1 AND Latitude < 1.1 AND Longitude > 1.1 AND Longitude < 1.1 ORDER BY `TagName` " +MOCKED_QUERY_POINT = "SELECT * FROM `mocked-asset` WHERE EventTime BETWEEN to_timestamp('2020-01-01') AND to_timestamp('2020-01-02')AND Latitude == 1.1 AND Longitude == 1.1 ORDER BY TagName " MOCKED_QUERY_OFFSET_LIMIT = "LIMIT 10 OFFSET 10 " MOCKED_PARAMETER_DICT_GRID = { @@ -68,7 +68,6 @@ } - def test_raw_grid(mocker: MockerFixture): mocked_cursor = mocker.spy(MockedDBConnection, "cursor") mocked_connection_close = mocker.spy(MockedDBConnection, "close") From 6368a42a58ea4f75975dbb9398ece8441f87418d Mon Sep 17 00:00:00 2001 From: Amber-Rigg Date: Wed, 20 Dec 2023 14:16:09 +0000 Subject: [PATCH 06/16] Update Tests Signed-off-by: Amber-Rigg --- .../queries/weather/_weather_query_builder.py | 24 +++++++++---------- .../queries/weather/weather_query_builder.py | 2 +- .../rtdip_sdk/queries/weather/test_latest.py | 5 ++-- .../rtdip_sdk/queries/weather/test_raw.py | 8 +++---- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py index 64f0411d1..0b5814e5b 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py @@ -98,9 +98,9 @@ def _build_parameters( def _raw_query_grid(parameters_dict: dict) -> str: raw_query_grid = ( "SELECT * FROM " - "`{{table_name|lower }}`" + "{{table_name|lower }} " "WHERE {{ timestamp_column }} BETWEEN to_timestamp('{{ start_date }}') AND to_timestamp('{{ end_date }}') " - "AND {{ forecast_run_timestamp_column }}` BETWEEN to_timestamp('{{ forecast_run_timestamp_start_date }}') AND to_timestamp('{{ forecast_run_timestamp_end_date }}') " + "AND {{ forecast_run_timestamp_column }} BETWEEN to_timestamp('{{ forecast_run_start_date }}') AND to_timestamp('{{ forecast_run_end_date }}') " "AND {{ latitude_column }} > {{ min_lat}} " "AND {{ latitude_column }} < {{ max_lat}} " "AND {{ longitude_column }} > {{ min_lon}} " @@ -123,11 +123,11 @@ def _raw_query_grid(parameters_dict: dict) -> str: def _raw_query_point(parameters_dict: dict) -> str: raw_query_point = ( "SELECT * FROM " - "`{{table_name|lower }}`" + "{{table_name|lower }} " "WHERE {{ timestamp_column }} BETWEEN to_timestamp('{{ start_date }}') AND to_timestamp('{{ end_date }}') " - "AND {{ forecast_run_timestamp_column }}` BETWEEN to_timestamp('{{ forecast_run_timestamp_start_date }}') AND to_timestamp('{{ forecast_run_timestamp_end_date }}') " - "AND {{ latitude_column }} > {{lat}} " - "AND {{ longitude_column }} > {{lon}} " + "AND {{ forecast_run_timestamp_column }} BETWEEN to_timestamp('{{ forecast_run_start_date }}') AND to_timestamp('{{ forecast_run_end_date }}') " + "AND {{ latitude_column }} == {{lat}} " + "AND {{ longitude_column }} == {{lon}} " "{% if source is defined and source is not none %}" "AND SOURCE = '{{ source }}' " "{% endif %}" @@ -146,7 +146,7 @@ def _raw_query_point(parameters_dict: dict) -> str: def _latest_query_grid(parameters_dict: dict) -> str: latest_query_grid = ( "SELECT * FROM " - "`{{table_name|lower }}`" + "{{table_name|lower }} " "WHERE {{ latitude_column }} > {{ min_lat}} " "AND {{ latitude_column }} < {{ max_lat}} " "AND {{ longitude_column }} > {{ min_lon}} " @@ -154,7 +154,7 @@ def _latest_query_grid(parameters_dict: dict) -> str: "{% if source is defined and source is not none %}" "AND SOURCE = '{{ source }}' " "{% endif %}" - "ORDER BY `{{ tagname_column }}` " + "ORDER BY {{ tagname_column }} " "{% if limit is defined and limit is not none %}" "LIMIT {{ limit }} " "{% endif %}" @@ -169,13 +169,13 @@ def _latest_query_grid(parameters_dict: dict) -> str: def _latest_query_point(parameters_dict: dict) -> str: latest_query_point = ( "SELECT * FROM " - "`{{table_name|lower }}`" - "WHERE {{ latitude_column }} > {{lat}} " - "AND {{ longitude_column }} > {{lon}} " + "{{table_name|lower }} " + "WHERE {{ latitude_column }} == {{lat}} " + "AND {{ longitude_column }} == {{lon}} " "{% if source is defined and source is not none %}" "AND SOURCE = '{{ source }}' " "{% endif %}" - "ORDER BY `{{ tagname_column }}` " + "ORDER BY {{ tagname_column }} " "{% if limit is defined and limit is not none %}" "LIMIT {{ limit }} " "{% endif %}" diff --git a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py index 952f4de6b..497a3b7eb 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py @@ -59,7 +59,7 @@ def source( status_column (optional str): The status column name in the source indicating `Good` or `Bad`. If this is not available, specify `None` value_column (optional str): The value column name in the source which is normally a float or string value for the time series event """ - self.data_source = "`.`".join(source.split(".")) + self.data_source = source self.tagname_column = tagname_column self.timestamp_column = timestamp_column self.forecast_run_timestamp_column = forecast_run_timestamp_column diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py index 22cca20fd..bdc2fdfee 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py @@ -37,13 +37,14 @@ DATABRICKS_SQL_CONNECT = "databricks.sql.connect" DATABRICKS_SQL_CONNECT_CURSOR = "databricks.sql.connect.cursor" INTERPOLATION_METHOD = "test/test/test" -MOCKED_QUERY_GRID = "SELECT * FROM `mocked-asset` WHERE Latitude > 0 AND Latitude < 0.1 AND Longitude > 0 AND Longitude < 0.1 ORDER BY TagName" -MOCKED_QUERY_POINT = "SELECT * FROM `mocked-asset` WHERE Latitude == 0 AND Longitude == 0 ORDER BY TagName" +MOCKED_QUERY_GRID = "SELECT * FROM mocked-asset WHERE Latitude > 0 AND Latitude < 0.1 AND Longitude > 0 AND Longitude < 0.1 ORDER BY TagName " +MOCKED_QUERY_POINT = "SELECT * FROM mocked-asset WHERE Latitude == 0 AND Longitude == 0 ORDER BY TagName " MOCKED_QUERY_OFFSET_LIMIT = "LIMIT 10 OFFSET 10 " MOCKED_PARAMETER_DICT_GRID = { "table_name": "mocked-asset", "max_lat": 0.1, + "min_lat": 0, "min_lon": 0, "max_lon": 0.1, } diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py index 28444b724..b9882ae8e 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py @@ -37,15 +37,15 @@ DATABRICKS_SQL_CONNECT = "databricks.sql.connect" DATABRICKS_SQL_CONNECT_CURSOR = "databricks.sql.connect.cursor" INTERPOLATION_METHOD = "test/test/test" -MOCKED_QUERY_GRID = "SELECT * FROM `mocked-asset` WHERE EventTime BETWEEN to_timestamp('2020-01-01') AND to_timestamp('2020-01-02')AND Latitude > 1.1 AND Latitude < 1.1 AND Longitude > 1.1 AND Longitude < 1.1 ORDER BY `TagName` " -MOCKED_QUERY_POINT = "SELECT * FROM `mocked-asset` WHERE EventTime BETWEEN to_timestamp('2020-01-01') AND to_timestamp('2020-01-02')AND Latitude == 1.1 AND Longitude == 1.1 ORDER BY TagName " +MOCKED_QUERY_GRID = "SELECT * FROM mocked-asset WHERE EventTime BETWEEN to_timestamp('2020-01-01') AND to_timestamp('2020-01-02') AND EnqueuedTime BETWEEN to_timestamp('2020-01-01') AND to_timestamp('2020-01-02') AND Latitude > 0 AND Latitude < 1.1 AND Longitude > 0 AND Longitude < 1.1 ORDER BY TagName " +MOCKED_QUERY_POINT = "SELECT * FROM mocked-asset WHERE EventTime BETWEEN to_timestamp('2020-01-01') AND to_timestamp('2020-01-02') AND EnqueuedTime BETWEEN to_timestamp('2020-01-01') AND to_timestamp('2020-01-02') AND Latitude == 1.1 AND Longitude == 1.1 ORDER BY TagName " MOCKED_QUERY_OFFSET_LIMIT = "LIMIT 10 OFFSET 10 " MOCKED_PARAMETER_DICT_GRID = { "table_name": "mocked-asset", - "min_lat": 1.1, + "min_lat": 0, "max_lat": 1.1, - "min_lon": 1.1, + "min_lon": 0, "max_lon": 1.1, "start_date": "2020-01-01", "end_date": "2020-01-02", From 2668770e07e8fcbc10383a78746e372fc0e1d655 Mon Sep 17 00:00:00 2001 From: Amber-Rigg Date: Wed, 20 Dec 2023 15:38:47 +0000 Subject: [PATCH 07/16] Refactor raw_parameters build Signed-off-by: Amber-Rigg --- .../queries/weather/_weather_query_builder.py | 90 +++++++------------ 1 file changed, 33 insertions(+), 57 deletions(-) diff --git a/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py index 0b5814e5b..a1e3825c5 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py @@ -32,65 +32,41 @@ def _build_parameters( area_type: str, table_type: str, ) -> dict: - if area_type == "grid": - raw_parameters = { - "table_name": parameters_dict["table_name"], - "max_lat": parameters_dict["max_lat"], - "max_lon": parameters_dict["max_lon"], - "min_lat": parameters_dict["min_lat"], - "min_lon": parameters_dict["min_lon"], - "source": parameters_dict.get("source", None), - "limit": parameters_dict.get("limit", None), - "latitude_column": parameters_dict.get("latitude_column", "Latitude"), - "longitude_column": parameters_dict.get("longitude_column", "Longitude"), - "tagname_column": parameters_dict.get("tagname_column", "TagName"), - } - if table_type == "raw": - raw_parameters["start_date"] = parameters_dict["start_date"] - raw_parameters["end_date"] = parameters_dict["end_date"] - raw_parameters["forecast_run_start_date"] = parameters_dict[ - "forecast_run_start_date" - ] - raw_parameters["forecast_run_end_date"] = parameters_dict[ - "forecast_run_end_date" - ] - raw_parameters["timestamp_column"] = parameters_dict.get( - "timestamp_column", "EventTime" - ) - raw_parameters["forecast_run_timestamp_column"] = parameters_dict.get( - "forecast_run_timestamp_column", "EnqueuedTime" - ) - - raw_parameters["include_status"] = False + raw_parameters = { + "table_name": parameters_dict["table_name"], + "source": parameters_dict.get("source", None), + "limit": parameters_dict.get("limit", None), + "latitude_column": parameters_dict.get("latitude_column", "Latitude"), + "longitude_column": parameters_dict.get("longitude_column", "Longitude"), + "tagname_column": parameters_dict.get("tagname_column", "TagName"), + } if area_type == "point": - raw_parameters = { - "table_name": parameters_dict["table_name"], - "lat": parameters_dict["lat"], - "lon": parameters_dict["lon"], - "source": parameters_dict.get("source", None), - "limit": parameters_dict.get("limit", None), - "latitude_column": parameters_dict.get("latitude_column", "Latitude"), - "longitude_column": parameters_dict.get("longitude_column", "Longitude"), - "tagname_column": parameters_dict.get("tagname_column", "TagName"), - } - if table_type == "raw": - raw_parameters["start_date"] = parameters_dict["start_date"] - raw_parameters["end_date"] = parameters_dict["end_date"] - raw_parameters["forecast_run_start_date"] = parameters_dict[ - "forecast_run_start_date" - ] - raw_parameters["forecast_run_end_date"] = parameters_dict[ - "forecast_run_end_date" - ] - raw_parameters["timestamp_column"] = parameters_dict.get( - "timestamp_column", "EventTime" - ) - raw_parameters["forecast_run_timestamp_column"] = parameters_dict.get( - "forecast_run_timestamp_column", "EnqueuedTime" - ) - - raw_parameters["include_status"] = False + raw_parameters["lat"] = parameters_dict["lat"] + raw_parameters["lon"] = parameters_dict["lon"] + + if area_type == "grid": + raw_parameters["max_lat"] = parameters_dict["max_lat"] + raw_parameters["max_lon"] = parameters_dict["max_lon"] + raw_parameters["min_lat"] = parameters_dict["min_lat"] + raw_parameters["min_lon"] = parameters_dict["min_lon"] + + if table_type == "raw": + raw_parameters["start_date"] = parameters_dict["start_date"] + raw_parameters["end_date"] = parameters_dict["end_date"] + raw_parameters["forecast_run_start_date"] = parameters_dict[ + "forecast_run_start_date" + ] + raw_parameters["forecast_run_end_date"] = parameters_dict[ + "forecast_run_end_date" + ] + raw_parameters["timestamp_column"] = parameters_dict.get( + "timestamp_column", "EventTime" + ) + raw_parameters["forecast_run_timestamp_column"] = parameters_dict.get( + "forecast_run_timestamp_column", "EnqueuedTime" + ) + raw_parameters["include_status"] = False return raw_parameters From 90ff8278f08b6c51ea96229022cdeb10d35a4506 Mon Sep 17 00:00:00 2001 From: Amber-Rigg Date: Thu, 4 Jan 2024 09:30:30 +0000 Subject: [PATCH 08/16] Update Query Builder Table Connection Signed-off-by: Amber-Rigg --- .../queries/weather/weather_query_builder.py | 16 ++++------------ .../weather/test_weather_query_builder.py | 4 ---- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py index 497a3b7eb..bdb292e0a 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py @@ -69,7 +69,6 @@ def source( def raw_point( self, - table_name: str, start_date: str, end_date: str, forecast_run_start_date: str, @@ -85,7 +84,6 @@ def raw_point( A function to return back raw data for a point. Args: - table_name (str): Table name start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) forecast_run_start_date (str): Start date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) @@ -101,7 +99,7 @@ def raw_point( DataFrame: A dataframe of raw timeseries data. """ raw_parameters = { - "table_name": table_name, + "table_name": self.data_source, "start_date": start_date, "end_date": end_date, "forecast_run_start_date": forecast_run_start_date, @@ -119,7 +117,6 @@ def raw_point( def latest_point( self, - table_name: str, lat: float, lon: float, source: str = None, @@ -129,7 +126,6 @@ def latest_point( A function to return back the latest data for a point. Args: - table_name (str): Table name lat (float): latitude lon (float): longitude source (optional str): Source of the data ie ECMWF @@ -139,7 +135,7 @@ def latest_point( DataFrame: A dataframe of raw timeseries data. """ raw_parameters = { - "table_name": table_name, + "table_name": self.data_source, "lat": lat, "lon": lon, "source": source, @@ -151,7 +147,6 @@ def latest_point( def raw_grid( # NOSONAR self, # NOSONAR - table_name: str, start_date: str, end_date: str, forecast_run_start_date: str, @@ -169,7 +164,6 @@ def raw_grid( # NOSONAR A function to return back raw data for a point. Args: - table_name (str): Table name start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) forecast_run_start_date (str): Start date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) @@ -187,7 +181,7 @@ def raw_grid( # NOSONAR DataFrame: A dataframe of raw timeseries data. """ raw_parameters = { - "table_name": table_name, + "table_name": self.data_source, "start_date": start_date, "end_date": end_date, "forecast_run_start_date": forecast_run_start_date, @@ -207,7 +201,6 @@ def raw_grid( # NOSONAR def latest_grid( self, - table_name: str, min_lat: float, min_lon: float, max_lat: float, @@ -219,7 +212,6 @@ def latest_grid( A function to return back the latest data for a point. Args: - table_name (str): Table name min_lat (float): Min latitude min_lon (float): Min longitude max_lat (float): Max latitude @@ -231,7 +223,7 @@ def latest_grid( DataFrame: A dataframe of raw timeseries data. """ raw_parameters = { - "table_name": table_name, + "table_name": self.data_source, "min_lat": min_lat, "min_lon": min_lon, "max_lat": max_lat, diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_weather_query_builder.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_weather_query_builder.py index 0d084011b..6b25f6520 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_weather_query_builder.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_weather_query_builder.py @@ -33,7 +33,6 @@ def test_weather_query_builder_raw_point(mocker: MockerFixture): .connect(MOCK_CONNECTION) .source(MOCK_TABLE, status_column=None) .raw_point( - table_name="mock_table", start_date="2021-01-01", end_date="2021-01-02", forecast_run_start_date="2021-01-01", @@ -56,7 +55,6 @@ def test_query_builder_latest_point(mocker: MockerFixture): .connect(MOCK_CONNECTION) .source(MOCK_TABLE, status_column=None) .latest_point( - table_name="mock_table", lat=0.1, lon=0.1, ) @@ -75,7 +73,6 @@ def test_weather_query_builder_raw_grid(mocker: MockerFixture): .connect(MOCK_CONNECTION) .source(MOCK_TABLE, status_column=None) .raw_grid( - table_name="mock_table", start_date="2021-01-01", end_date="2021-01-02", forecast_run_start_date="2021-01-01", @@ -100,7 +97,6 @@ def test_query_builder_latest_grid(mocker: MockerFixture): .connect(MOCK_CONNECTION) .source(MOCK_TABLE, status_column=None) .latest_grid( - table_name="mock_table", min_lat=0.1, max_lat=0.1, min_lon=0.1, From 0bc10339c517610520100b3eedea1346eb573ca6 Mon Sep 17 00:00:00 2001 From: Amber-Rigg Date: Wed, 10 Jan 2024 18:31:27 +0000 Subject: [PATCH 09/16] Review to include options and refactor test Signed-off-by: Amber-Rigg --- .../queries/weather/_weather_query_builder.py | 68 ++++++---- .../rtdip_sdk/queries/weather/latest.py | 16 ++- .../python/rtdip_sdk/queries/weather/raw.py | 16 ++- .../queries/weather/weather_query_builder.py | 17 +-- .../rtdip_sdk/queries/weather/test_latest.py | 100 ++++++++++++-- .../rtdip_sdk/queries/weather/test_raw.py | 128 +++++++++++++++--- 6 files changed, 266 insertions(+), 79 deletions(-) diff --git a/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py index a1e3825c5..a86c5b896 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py @@ -33,8 +33,12 @@ def _build_parameters( table_type: str, ) -> dict: raw_parameters = { - "table_name": parameters_dict["table_name"], "source": parameters_dict.get("source", None), + "forecast": parameters_dict.get("forecast", None), + "forecast_type": parameters_dict.get("forecast_type", "weather"), + "region": parameters_dict.get("region", None), + "data_security_level": parameters_dict.get("data_security_level", None), + "data_type": parameters_dict.get("data_type", None), "limit": parameters_dict.get("limit", None), "latitude_column": parameters_dict.get("latitude_column", "Latitude"), "longitude_column": parameters_dict.get("longitude_column", "Longitude"), @@ -74,17 +78,18 @@ def _build_parameters( def _raw_query_grid(parameters_dict: dict) -> str: raw_query_grid = ( "SELECT * FROM " - "{{table_name|lower }} " - "WHERE {{ timestamp_column }} BETWEEN to_timestamp('{{ start_date }}') AND to_timestamp('{{ end_date }}') " - "AND {{ forecast_run_timestamp_column }} BETWEEN to_timestamp('{{ forecast_run_start_date }}') AND to_timestamp('{{ forecast_run_end_date }}') " - "AND {{ latitude_column }} > {{ min_lat}} " - "AND {{ latitude_column }} < {{ max_lat}} " - "AND {{ longitude_column }} > {{ min_lon}} " - "AND {{ longitude_column }} < {{ max_lon}} " "{% if source is defined and source is not none %}" - "AND SOURCE = '{{ source }}' " + "`{{ source|lower }}` " + "{% else %}" + "`{{ forecast|lower }}`.`{{ forecast_type|lower }}`.`{{ region|lower }}_{{ data_security_level|lower }}_events_{{ data_type|lower }}` " "{% endif %}" - "ORDER BY {{ tagname_column }} " + "WHERE (`{{ timestamp_column }}` BETWEEN to_timestamp(\"{{ start_date }}\") AND to_timestamp(\"{{ end_date }}\")) " + "AND (`{{ forecast_run_timestamp_column }}` BETWEEN to_timestamp(\"{{ forecast_run_start_date }}\") AND to_timestamp(\"{{ forecast_run_end_date }}\")) " + "AND `{{ latitude_column }}` > {{ min_lat}} " + "AND `{{ latitude_column }}` < {{ max_lat}} " + "AND `{{ longitude_column }}` > {{ min_lon}} " + "AND `{{ longitude_column }}` < {{ max_lon}} " + "ORDER BY `{{ tagname_column }}` " "{% if limit is defined and limit is not none %}" "LIMIT {{ limit }} " "{% endif %}" @@ -97,17 +102,18 @@ def _raw_query_grid(parameters_dict: dict) -> str: def _raw_query_point(parameters_dict: dict) -> str: - raw_query_point = ( + raw_query_point = ( "SELECT * FROM " - "{{table_name|lower }} " - "WHERE {{ timestamp_column }} BETWEEN to_timestamp('{{ start_date }}') AND to_timestamp('{{ end_date }}') " - "AND {{ forecast_run_timestamp_column }} BETWEEN to_timestamp('{{ forecast_run_start_date }}') AND to_timestamp('{{ forecast_run_end_date }}') " - "AND {{ latitude_column }} == {{lat}} " - "AND {{ longitude_column }} == {{lon}} " "{% if source is defined and source is not none %}" - "AND SOURCE = '{{ source }}' " + "`{{ source|lower }}` " + "{% else %}" + "`{{ forecast|lower }}`.`{{ forecast_type|lower }}`.`{{ region|lower }}_{{ data_security_level|lower }}_events_{{ data_type|lower }}` " "{% endif %}" - "ORDER BY {{ tagname_column }} " + "WHERE (`{{ timestamp_column }}` BETWEEN to_timestamp(\"{{ start_date }}\") AND to_timestamp(\"{{ end_date }}\")) " + "AND (`{{ forecast_run_timestamp_column }}` BETWEEN to_timestamp(\"{{ forecast_run_start_date }}\") AND to_timestamp(\"{{ forecast_run_end_date }}\")) " + "AND `{{ latitude_column }}` == {{ lat }} " + "AND `{{ longitude_column }}` == {{ lon }} " + "ORDER BY `{{ tagname_column }}` " "{% if limit is defined and limit is not none %}" "LIMIT {{ limit }} " "{% endif %}" @@ -122,15 +128,16 @@ def _raw_query_point(parameters_dict: dict) -> str: def _latest_query_grid(parameters_dict: dict) -> str: latest_query_grid = ( "SELECT * FROM " - "{{table_name|lower }} " - "WHERE {{ latitude_column }} > {{ min_lat}} " - "AND {{ latitude_column }} < {{ max_lat}} " - "AND {{ longitude_column }} > {{ min_lon}} " - "AND {{ longitude_column }} < {{ max_lon}} " "{% if source is defined and source is not none %}" - "AND SOURCE = '{{ source }}' " + "`{{ source|lower }}` " + "{% else %}" + "`{{ forecast|lower }}`.`{{ forecast_type|lower }}`.`{{ region|lower }}_{{ data_security_level|lower }}_events_{{ data_type|lower }}_latest` " "{% endif %}" - "ORDER BY {{ tagname_column }} " + "WHERE `{{ latitude_column }}` > {{ min_lat}} " + "AND `{{ latitude_column }}` < {{ max_lat}} " + "AND `{{ longitude_column }}` > {{ min_lon}} " + "AND `{{ longitude_column }}` < {{ max_lon}} " + "ORDER BY `{{ tagname_column }}` " "{% if limit is defined and limit is not none %}" "LIMIT {{ limit }} " "{% endif %}" @@ -145,13 +152,14 @@ def _latest_query_grid(parameters_dict: dict) -> str: def _latest_query_point(parameters_dict: dict) -> str: latest_query_point = ( "SELECT * FROM " - "{{table_name|lower }} " - "WHERE {{ latitude_column }} == {{lat}} " - "AND {{ longitude_column }} == {{lon}} " "{% if source is defined and source is not none %}" - "AND SOURCE = '{{ source }}' " + "`{{ source|lower }}` " + "{% else %}" + "`{{ forecast|lower }}`.`{{ forecast_type|lower }}`.`{{ region|lower }}_{{ data_security_level|lower }}_events_{{ data_type|lower }}_latest` " "{% endif %}" - "ORDER BY {{ tagname_column }} " + "WHERE `{{ latitude_column }}` == {{ lat }} " + "AND `{{ longitude_column }}` == {{ lon }} " + "ORDER BY `{{ tagname_column }}` " "{% if limit is defined and limit is not none %}" "LIMIT {{ limit }} " "{% endif %}" diff --git a/src/sdk/python/rtdip_sdk/queries/weather/latest.py b/src/sdk/python/rtdip_sdk/queries/weather/latest.py index 6ac6e6ce2..9fb640b5e 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/latest.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/latest.py @@ -34,12 +34,16 @@ def get_grid(connection: object, parameters_dict: dict) -> pd.DataFrame: parameters_dict: A dictionary of parameters (see Attributes table below) Attributes: - table_name (str): Name of the table + source (optional str): Source of the data the full table name + forecast (str): Any specific identifier for forecast + forecast_type(str): Type of forecast ie weather, solar, power, etc + region (str): Region + data_security_level (str): Level of data security + data_type (str): Type of the data (float, integer, double, string) max_lat (float): Maximum latitude max_lon (float): Maximum longitude min_lat (float): Minimum latitude min_lon (float): Minimum longitude - source (optional str): Source of the data ie ECMWF limit (optional int): The number of rows to be returned Returns: @@ -81,10 +85,14 @@ def get_point(connection: object, parameters_dict: dict) -> pd.DataFrame: parameters_dict: A dictionary of parameters (see Attributes table below) Attributes: - table_name (str): Name of the table + source (optional str): Source of the data the full table name + forecast (str): Any specific identifier for forecast + forecast_type(str): Type of forecast ie weather, solar, power, etc + region (str): Region + data_security_level (str): Level of data security + data_type (str): Type of the data (float, integer, double, string) lat (float): latitude lon (float): longitude - source (optional str): Source of the data ie ECMWF limit (optional int): The number of rows to be returned Returns: diff --git a/src/sdk/python/rtdip_sdk/queries/weather/raw.py b/src/sdk/python/rtdip_sdk/queries/weather/raw.py index 3876c752d..c05a21775 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/raw.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/raw.py @@ -34,7 +34,12 @@ def get_grid(connection: object, parameters_dict: dict) -> pd.DataFrame: parameters_dict: A dictionary of parameters (see Attributes table below) Attributes: - table_name (str): Name of the table + source (optional str): Source of the data the full table name + forecast (str): Any specific identifier for forecast + forecast_type(str): Type of forecast ie weather, solar, power, etc + region (str): Region + data_security_level (str): Level of data security + data_type (str): Type of the data (float, integer, double, string) start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) forecast_run_start_date (str): Start date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) @@ -43,7 +48,6 @@ def get_grid(connection: object, parameters_dict: dict) -> pd.DataFrame: max_lon (float): Maximum longitude min_lat (float): Minimum latitude min_lon (float): Minimum longitude - source (optional str): Source of the data ie ECMWF limit (optional int): The number of rows to be returned } @@ -88,14 +92,18 @@ def get_point(connection: object, parameters_dict: dict) -> pd.DataFrame: parameters_dict: A dictionary of parameters (see Attributes table below) Attributes: - table_name (str): Name of the table + source (optional str): Source of the data the full table name + forecast (str): Any specific identifier for forecast + forecast_type(str): Type of forecast ie weather, solar, power, etc + region (str): Region + data_security_level (str): Level of data security + data_type (str): Type of the data (float, integer, double, string) start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) forecast_run_start_date (str): Start date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) forecast_run_end_date (str): End date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) lat (float): latitude lon (float): longitude - source (optional str): Source of the data ie ECMWF limit (optional int): The number of rows to be returned } diff --git a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py index bdb292e0a..5d29029ae 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py @@ -59,7 +59,7 @@ def source( status_column (optional str): The status column name in the source indicating `Good` or `Bad`. If this is not available, specify `None` value_column (optional str): The value column name in the source which is normally a float or string value for the time series event """ - self.data_source = source + self.data_source = "`.`".join(source.split(".")) self.tagname_column = tagname_column self.timestamp_column = timestamp_column self.forecast_run_timestamp_column = forecast_run_timestamp_column @@ -75,7 +75,6 @@ def raw_point( forecast_run_end_date: str, lat: float, lon: float, - source: str = None, time_zone: str = None, include_bad_data: bool = False, limit: int = None, @@ -99,14 +98,13 @@ def raw_point( DataFrame: A dataframe of raw timeseries data. """ raw_parameters = { - "table_name": self.data_source, + "source": self.data_source, "start_date": start_date, "end_date": end_date, "forecast_run_start_date": forecast_run_start_date, "forecast_run_end_date": forecast_run_end_date, "lat": lat, "lon": lon, - "source": source, "time_zone": time_zone, "include_bad_data": include_bad_data, "limit": limit, @@ -119,7 +117,6 @@ def latest_point( self, lat: float, lon: float, - source: str = None, limit: int = None, ) -> DataFrame: """ @@ -135,7 +132,7 @@ def latest_point( DataFrame: A dataframe of raw timeseries data. """ raw_parameters = { - "table_name": self.data_source, + "source": self.data_source, "lat": lat, "lon": lon, "source": source, @@ -155,7 +152,6 @@ def raw_grid( # NOSONAR min_lon: float, max_lat: float, max_lon: float, - source: str = None, time_zone: str = None, include_bad_data: bool = False, limit: int = None, # NOSONAR @@ -181,7 +177,7 @@ def raw_grid( # NOSONAR DataFrame: A dataframe of raw timeseries data. """ raw_parameters = { - "table_name": self.data_source, + "source": self.data_source, "start_date": start_date, "end_date": end_date, "forecast_run_start_date": forecast_run_start_date, @@ -190,7 +186,6 @@ def raw_grid( # NOSONAR "min_lon": min_lon, "max_lat": max_lat, "max_lon": max_lon, - "source": source, "time_zone": time_zone, "include_bad_data": include_bad_data, "limit": limit, @@ -205,7 +200,6 @@ def latest_grid( min_lon: float, max_lat: float, max_lon: float, - source: str = None, limit: int = None, ) -> DataFrame: """ @@ -223,12 +217,11 @@ def latest_grid( DataFrame: A dataframe of raw timeseries data. """ raw_parameters = { - "table_name": self.data_source, + "source": self.data_source, "min_lat": min_lat, "min_lon": min_lon, "max_lat": max_lat, "max_lon": max_lon, - "source": source, "limit": limit, "supress_warning": True, } diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py index bdc2fdfee..71a4870a6 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py @@ -37,25 +37,50 @@ DATABRICKS_SQL_CONNECT = "databricks.sql.connect" DATABRICKS_SQL_CONNECT_CURSOR = "databricks.sql.connect.cursor" INTERPOLATION_METHOD = "test/test/test" -MOCKED_QUERY_GRID = "SELECT * FROM mocked-asset WHERE Latitude > 0 AND Latitude < 0.1 AND Longitude > 0 AND Longitude < 0.1 ORDER BY TagName " -MOCKED_QUERY_POINT = "SELECT * FROM mocked-asset WHERE Latitude == 0 AND Longitude == 0 ORDER BY TagName " +MOCKED_QUERY_GRID = 'SELECT * FROM `forecast`.`weather`.`mock_region_mock_security_events_mock_data_type_latest` WHERE `Latitude` > 36 AND `Latitude` < 38 AND `Longitude` > -109.1 AND `Longitude` < -107.1 ORDER BY `TagName` ' +MOCKED_QUERY_POINT = 'SELECT * FROM `forecast`.`weather`.`mock_region_mock_security_events_mock_data_type_latest` WHERE `Latitude` == 37 AND `Longitude` == -108.1 ORDER BY `TagName` ' MOCKED_QUERY_OFFSET_LIMIT = "LIMIT 10 OFFSET 10 " + MOCKED_PARAMETER_DICT_GRID = { - "table_name": "mocked-asset", - "max_lat": 0.1, - "min_lat": 0, - "min_lon": 0, - "max_lon": 0.1, + "forecast": "forecast", + "forecast_type" : "weather", + "region": "mock_region", + "data_security_level": "mock_security", + "data_type": "mock_data_type", + "min_lat": 36, + "max_lat": 38, + "min_lon": -109.1, + "max_lon": -107.1, } MOCKED_PARAMETER_DICT_POINT = { - "table_name": "mocked-asset", - "lat": 0, - "lon": 0, + "forecast": "forecast", + "forecast_type" : "weather", + "region": "mock_region", + "data_security_level": "mock_security", + "data_type": "mock_data_type", + "lat": 37, + "lon": -108.1, +} + +MOCKED_PARAMETER_DICT_GRID_SOURCE = { + "source": "forecast`.`weather`.`mock_region_mock_security_events_mock_data_type_latest", + "min_lat": 36, + "max_lat": 38, + "min_lon": -109.1, + "max_lon": -107.1, + +} + +MOCKED_PARAMETER_DICT_POINT_SOURCE = { + "source": "forecast`.`weather`.`mock_region_mock_security_events_mock_data_type_latest", + "lat": 37, + "lon": -108.1, } + def test_latest_grid(mocker: MockerFixture): mocked_cursor = mocker.spy(MockedDBConnection, "cursor") mocked_connection_close = mocker.spy(MockedDBConnection, "close") @@ -83,6 +108,33 @@ def test_latest_grid(mocker: MockerFixture): mocked_close.assert_called_once() assert isinstance(actual, pd.DataFrame) +def test_latest_grid_source(mocker: MockerFixture): + mocked_cursor = mocker.spy(MockedDBConnection, "cursor") + mocked_connection_close = mocker.spy(MockedDBConnection, "close") + mocked_execute = mocker.spy(MockedCursor, "execute") + mocked_fetch_all = mocker.patch.object( + MockedCursor, + "fetchall_arrow", + return_value=pa.Table.from_pandas( + pd.DataFrame(data={"e": [1], "f": [2], "g": [3], "h": [4]}) + ), + ) + mocked_close = mocker.spy(MockedCursor, "close") + mocker.patch(DATABRICKS_SQL_CONNECT, return_value=MockedDBConnection()) + + mocked_connection = DatabricksSQLConnection( + SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN + ) + + actual = latest_grid(mocked_connection, MOCKED_PARAMETER_DICT_GRID_SOURCE) + + mocked_cursor.assert_called_once() + mocked_connection_close.assert_called_once() + mocked_execute.assert_called_once_with(mocker.ANY, query=MOCKED_QUERY_GRID) + mocked_fetch_all.assert_called_once() + mocked_close.assert_called_once() + assert isinstance(actual, pd.DataFrame) + def test_latest_grid_fails(mocker: MockerFixture): mocker.spy(MockedDBConnection, "cursor") @@ -127,6 +179,34 @@ def test_latest_point(mocker: MockerFixture): mocked_close.assert_called_once() assert isinstance(actual, pd.DataFrame) +def test_latest_point_source(mocker: MockerFixture): + mocked_cursor = mocker.spy(MockedDBConnection, "cursor") + mocked_connection_close = mocker.spy(MockedDBConnection, "close") + mocked_execute = mocker.spy(MockedCursor, "execute") + mocked_fetch_all = mocker.patch.object( + MockedCursor, + "fetchall_arrow", + return_value=pa.Table.from_pandas( + pd.DataFrame(data={"i": [1], "j": [2], "k": [3], "l": [4]}) + ), + ) + mocked_close = mocker.spy(MockedCursor, "close") + mocker.patch(DATABRICKS_SQL_CONNECT, return_value=MockedDBConnection()) + + mocked_connection = DatabricksSQLConnection( + SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN + ) + + actual = latest_point(mocked_connection, MOCKED_PARAMETER_DICT_POINT_SOURCE) + + mocked_cursor.assert_called_once() + mocked_connection_close.assert_called_once() + mocked_execute.assert_called_once_with(mocker.ANY, query=MOCKED_QUERY_POINT) + mocked_fetch_all.assert_called_once() + mocked_close.assert_called_once() + assert isinstance(actual, pd.DataFrame) + + def test_latest_point_fails(mocker: MockerFixture): mocker.spy(MockedDBConnection, "cursor") diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py index b9882ae8e..c074500f0 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py @@ -37,38 +37,72 @@ DATABRICKS_SQL_CONNECT = "databricks.sql.connect" DATABRICKS_SQL_CONNECT_CURSOR = "databricks.sql.connect.cursor" INTERPOLATION_METHOD = "test/test/test" -MOCKED_QUERY_GRID = "SELECT * FROM mocked-asset WHERE EventTime BETWEEN to_timestamp('2020-01-01') AND to_timestamp('2020-01-02') AND EnqueuedTime BETWEEN to_timestamp('2020-01-01') AND to_timestamp('2020-01-02') AND Latitude > 0 AND Latitude < 1.1 AND Longitude > 0 AND Longitude < 1.1 ORDER BY TagName " -MOCKED_QUERY_POINT = "SELECT * FROM mocked-asset WHERE EventTime BETWEEN to_timestamp('2020-01-01') AND to_timestamp('2020-01-02') AND EnqueuedTime BETWEEN to_timestamp('2020-01-01') AND to_timestamp('2020-01-02') AND Latitude == 1.1 AND Longitude == 1.1 ORDER BY TagName " +MOCKED_QUERY_GRID = 'SELECT * FROM `forecast`.`weather`.`mock_region_mock_security_events_mock_data_type` WHERE (`EventTime` BETWEEN to_timestamp("2024-01-01") AND to_timestamp("2024-01-03")) AND (`EnqueuedTime` BETWEEN to_timestamp("2023-12-28") AND to_timestamp("2023-12-31")) AND `Latitude` > 36 AND `Latitude` < 38 AND `Longitude` > -109.1 AND `Longitude` < -107.1 ORDER BY `TagName` ' +MOCKED_QUERY_POINT = 'SELECT * FROM `forecast`.`weather`.`mock_region_mock_security_events_mock_data_type` WHERE (`EventTime` BETWEEN to_timestamp("2024-01-01") AND to_timestamp("2024-01-03")) AND (`EnqueuedTime` BETWEEN to_timestamp("2023-12-28") AND to_timestamp("2023-12-31")) AND `Latitude` == 37 AND `Longitude` == -108.1 ORDER BY `TagName` ' MOCKED_QUERY_OFFSET_LIMIT = "LIMIT 10 OFFSET 10 " + MOCKED_PARAMETER_DICT_GRID = { - "table_name": "mocked-asset", - "min_lat": 0, - "max_lat": 1.1, - "min_lon": 0, - "max_lon": 1.1, - "start_date": "2020-01-01", - "end_date": "2020-01-02", - "forecast_run_start_date": "2020-01-01", - "forecast_run_end_date": "2020-01-02", + "forecast": "forecast", + "forecast_type" : "weather", + "region": "mock_region", + "data_security_level": "mock_security", + "data_type": "mock_data_type", + "min_lat": 36, + "max_lat": 38, + "min_lon": -109.1, + "max_lon": -107.1, + "start_date": "2024-01-01", + "end_date": "2024-01-03", + "forecast_run_start_date": "2023-12-28", + "forecast_run_end_date": "2023-12-31", "timestamp_column": "EventTime", "forecast_run_timestamp_column": "EnqueuedTime", } MOCKED_PARAMETER_DICT_POINT = { - "table_name": "mocked-asset", - "lat": 1.1, - "lon": 1.1, - "start_date": "2020-01-01", - "end_date": "2020-01-02", - "forecast_run_start_date": "2020-01-01", - "forecast_run_end_date": "2020-01-02", + "forecast": "forecast", + "forecast_type" : "weather", + "region": "mock_region", + "data_security_level": "mock_security", + "data_type": "mock_data_type", + "lat": 37, + "lon": -108.1, + "start_date": "2024-01-01", + "end_date": "2024-01-03", + "forecast_run_start_date": "2023-12-28", + "forecast_run_end_date": "2023-12-31", + "timestamp_column": "EventTime", + "forecast_run_timestamp_column": "EnqueuedTime", +} + +MOCKED_PARAMETER_DICT_GRID_SOURCE = { + "source": "forecast`.`weather`.`mock_region_mock_security_events_mock_data_type", + "min_lat": 36, + "max_lat": 38, + "min_lon": -109.1, + "max_lon": -107.1, + "start_date": "2024-01-01", + "end_date": "2024-01-03", + "forecast_run_start_date": "2023-12-28", + "forecast_run_end_date": "2023-12-31", "timestamp_column": "EventTime", "forecast_run_timestamp_column": "EnqueuedTime", } +MOCKED_PARAMETER_DICT_POINT_SOURCE = { + "source": "forecast`.`weather`.`mock_region_mock_security_events_mock_data_type", + "lat": 37, + "lon": -108.1, + "start_date": "2024-01-01", + "end_date": "2024-01-03", + "forecast_run_start_date": "2023-12-28", + "forecast_run_end_date": "2023-12-31", + "timestamp_column": "EventTime", + "forecast_run_timestamp_column": "EnqueuedTime", +} -def test_raw_grid(mocker: MockerFixture): +def test_raw_grid_(mocker: MockerFixture): mocked_cursor = mocker.spy(MockedDBConnection, "cursor") mocked_connection_close = mocker.spy(MockedDBConnection, "close") mocked_execute = mocker.spy(MockedCursor, "execute") @@ -95,6 +129,34 @@ def test_raw_grid(mocker: MockerFixture): mocked_close.assert_called_once() assert isinstance(actual, pd.DataFrame) +def test_raw_grid_source(mocker: MockerFixture): + mocked_cursor = mocker.spy(MockedDBConnection, "cursor") + mocked_connection_close = mocker.spy(MockedDBConnection, "close") + mocked_execute = mocker.spy(MockedCursor, "execute") + mocked_fetch_all = mocker.patch.object( + MockedCursor, + "fetchall_arrow", + return_value=pa.Table.from_pandas( + pd.DataFrame(data={"m": [1], "n": [2], "o": [3], "p": [4]}) + ), + ) + mocked_close = mocker.spy(MockedCursor, "close") + mocker.patch(DATABRICKS_SQL_CONNECT, return_value=MockedDBConnection()) + + mocked_connection = DatabricksSQLConnection( + SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN + ) + + actual = raw_grid(mocked_connection, MOCKED_PARAMETER_DICT_GRID_SOURCE) + + mocked_cursor.assert_called_once() + mocked_connection_close.assert_called_once() + mocked_execute.assert_called_once_with(mocker.ANY, query=MOCKED_QUERY_GRID) + mocked_fetch_all.assert_called_once() + mocked_close.assert_called_once() + assert isinstance(actual, pd.DataFrame) + + def test_raw_grid_fails(mocker: MockerFixture): mocker.spy(MockedDBConnection, "cursor") @@ -139,6 +201,34 @@ def test_raw_point(mocker: MockerFixture): mocked_close.assert_called_once() assert isinstance(actual, pd.DataFrame) +def test_raw_point_source(mocker: MockerFixture): + mocked_cursor = mocker.spy(MockedDBConnection, "cursor") + mocked_connection_close = mocker.spy(MockedDBConnection, "close") + mocked_execute = mocker.spy(MockedCursor, "execute") + mocked_fetch_all = mocker.patch.object( + MockedCursor, + "fetchall_arrow", + return_value=pa.Table.from_pandas( + pd.DataFrame(data={"q": [1], "r": [2], "s": [3], "t": [4]}) + ), + ) + mocked_close = mocker.spy(MockedCursor, "close") + mocker.patch(DATABRICKS_SQL_CONNECT, return_value=MockedDBConnection()) + + mocked_connection = DatabricksSQLConnection( + SERVER_HOSTNAME, HTTP_PATH, ACCESS_TOKEN + ) + + actual = raw_point(mocked_connection, MOCKED_PARAMETER_DICT_POINT_SOURCE) + + mocked_cursor.assert_called_once() + mocked_connection_close.assert_called_once() + mocked_execute.assert_called_once_with(mocker.ANY, query=MOCKED_QUERY_POINT) + mocked_fetch_all.assert_called_once() + mocked_close.assert_called_once() + assert isinstance(actual, pd.DataFrame) + + def test_raw_point_fails(mocker: MockerFixture): mocker.spy(MockedDBConnection, "cursor") From 489fe94df2a37d5a5d8dad0cf2c209a2aa82dfcd Mon Sep 17 00:00:00 2001 From: Amber-Rigg Date: Wed, 10 Jan 2024 18:33:08 +0000 Subject: [PATCH 10/16] black formatting Signed-off-by: Amber-Rigg --- .../queries/weather/_weather_query_builder.py | 10 +++++----- src/sdk/python/rtdip_sdk/queries/weather/latest.py | 8 ++++---- src/sdk/python/rtdip_sdk/queries/weather/raw.py | 8 ++++---- .../python/rtdip_sdk/queries/weather/test_latest.py | 13 ++++++------- .../python/rtdip_sdk/queries/weather/test_raw.py | 9 +++++---- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py index a86c5b896..d7dfea5fc 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/_weather_query_builder.py @@ -83,8 +83,8 @@ def _raw_query_grid(parameters_dict: dict) -> str: "{% else %}" "`{{ forecast|lower }}`.`{{ forecast_type|lower }}`.`{{ region|lower }}_{{ data_security_level|lower }}_events_{{ data_type|lower }}` " "{% endif %}" - "WHERE (`{{ timestamp_column }}` BETWEEN to_timestamp(\"{{ start_date }}\") AND to_timestamp(\"{{ end_date }}\")) " - "AND (`{{ forecast_run_timestamp_column }}` BETWEEN to_timestamp(\"{{ forecast_run_start_date }}\") AND to_timestamp(\"{{ forecast_run_end_date }}\")) " + 'WHERE (`{{ timestamp_column }}` BETWEEN to_timestamp("{{ start_date }}") AND to_timestamp("{{ end_date }}")) ' + 'AND (`{{ forecast_run_timestamp_column }}` BETWEEN to_timestamp("{{ forecast_run_start_date }}") AND to_timestamp("{{ forecast_run_end_date }}")) ' "AND `{{ latitude_column }}` > {{ min_lat}} " "AND `{{ latitude_column }}` < {{ max_lat}} " "AND `{{ longitude_column }}` > {{ min_lon}} " @@ -102,15 +102,15 @@ def _raw_query_grid(parameters_dict: dict) -> str: def _raw_query_point(parameters_dict: dict) -> str: - raw_query_point = ( + raw_query_point = ( "SELECT * FROM " "{% if source is defined and source is not none %}" "`{{ source|lower }}` " "{% else %}" "`{{ forecast|lower }}`.`{{ forecast_type|lower }}`.`{{ region|lower }}_{{ data_security_level|lower }}_events_{{ data_type|lower }}` " "{% endif %}" - "WHERE (`{{ timestamp_column }}` BETWEEN to_timestamp(\"{{ start_date }}\") AND to_timestamp(\"{{ end_date }}\")) " - "AND (`{{ forecast_run_timestamp_column }}` BETWEEN to_timestamp(\"{{ forecast_run_start_date }}\") AND to_timestamp(\"{{ forecast_run_end_date }}\")) " + 'WHERE (`{{ timestamp_column }}` BETWEEN to_timestamp("{{ start_date }}") AND to_timestamp("{{ end_date }}")) ' + 'AND (`{{ forecast_run_timestamp_column }}` BETWEEN to_timestamp("{{ forecast_run_start_date }}") AND to_timestamp("{{ forecast_run_end_date }}")) ' "AND `{{ latitude_column }}` == {{ lat }} " "AND `{{ longitude_column }}` == {{ lon }} " "ORDER BY `{{ tagname_column }}` " diff --git a/src/sdk/python/rtdip_sdk/queries/weather/latest.py b/src/sdk/python/rtdip_sdk/queries/weather/latest.py index 9fb640b5e..124e0f98a 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/latest.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/latest.py @@ -34,8 +34,8 @@ def get_grid(connection: object, parameters_dict: dict) -> pd.DataFrame: parameters_dict: A dictionary of parameters (see Attributes table below) Attributes: - source (optional str): Source of the data the full table name - forecast (str): Any specific identifier for forecast + source (optional str): Source of the data the full table name + forecast (str): Any specific identifier for forecast forecast_type(str): Type of forecast ie weather, solar, power, etc region (str): Region data_security_level (str): Level of data security @@ -85,8 +85,8 @@ def get_point(connection: object, parameters_dict: dict) -> pd.DataFrame: parameters_dict: A dictionary of parameters (see Attributes table below) Attributes: - source (optional str): Source of the data the full table name - forecast (str): Any specific identifier for forecast + source (optional str): Source of the data the full table name + forecast (str): Any specific identifier for forecast forecast_type(str): Type of forecast ie weather, solar, power, etc region (str): Region data_security_level (str): Level of data security diff --git a/src/sdk/python/rtdip_sdk/queries/weather/raw.py b/src/sdk/python/rtdip_sdk/queries/weather/raw.py index c05a21775..1f6f116ca 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/raw.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/raw.py @@ -34,8 +34,8 @@ def get_grid(connection: object, parameters_dict: dict) -> pd.DataFrame: parameters_dict: A dictionary of parameters (see Attributes table below) Attributes: - source (optional str): Source of the data the full table name - forecast (str): Any specific identifier for forecast + source (optional str): Source of the data the full table name + forecast (str): Any specific identifier for forecast forecast_type(str): Type of forecast ie weather, solar, power, etc region (str): Region data_security_level (str): Level of data security @@ -92,8 +92,8 @@ def get_point(connection: object, parameters_dict: dict) -> pd.DataFrame: parameters_dict: A dictionary of parameters (see Attributes table below) Attributes: - source (optional str): Source of the data the full table name - forecast (str): Any specific identifier for forecast + source (optional str): Source of the data the full table name + forecast (str): Any specific identifier for forecast forecast_type(str): Type of forecast ie weather, solar, power, etc region (str): Region data_security_level (str): Level of data security diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py index 71a4870a6..4d5609a2c 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py @@ -37,14 +37,14 @@ DATABRICKS_SQL_CONNECT = "databricks.sql.connect" DATABRICKS_SQL_CONNECT_CURSOR = "databricks.sql.connect.cursor" INTERPOLATION_METHOD = "test/test/test" -MOCKED_QUERY_GRID = 'SELECT * FROM `forecast`.`weather`.`mock_region_mock_security_events_mock_data_type_latest` WHERE `Latitude` > 36 AND `Latitude` < 38 AND `Longitude` > -109.1 AND `Longitude` < -107.1 ORDER BY `TagName` ' -MOCKED_QUERY_POINT = 'SELECT * FROM `forecast`.`weather`.`mock_region_mock_security_events_mock_data_type_latest` WHERE `Latitude` == 37 AND `Longitude` == -108.1 ORDER BY `TagName` ' +MOCKED_QUERY_GRID = "SELECT * FROM `forecast`.`weather`.`mock_region_mock_security_events_mock_data_type_latest` WHERE `Latitude` > 36 AND `Latitude` < 38 AND `Longitude` > -109.1 AND `Longitude` < -107.1 ORDER BY `TagName` " +MOCKED_QUERY_POINT = "SELECT * FROM `forecast`.`weather`.`mock_region_mock_security_events_mock_data_type_latest` WHERE `Latitude` == 37 AND `Longitude` == -108.1 ORDER BY `TagName` " MOCKED_QUERY_OFFSET_LIMIT = "LIMIT 10 OFFSET 10 " MOCKED_PARAMETER_DICT_GRID = { "forecast": "forecast", - "forecast_type" : "weather", + "forecast_type": "weather", "region": "mock_region", "data_security_level": "mock_security", "data_type": "mock_data_type", @@ -56,7 +56,7 @@ MOCKED_PARAMETER_DICT_POINT = { "forecast": "forecast", - "forecast_type" : "weather", + "forecast_type": "weather", "region": "mock_region", "data_security_level": "mock_security", "data_type": "mock_data_type", @@ -70,7 +70,6 @@ "max_lat": 38, "min_lon": -109.1, "max_lon": -107.1, - } MOCKED_PARAMETER_DICT_POINT_SOURCE = { @@ -80,7 +79,6 @@ } - def test_latest_grid(mocker: MockerFixture): mocked_cursor = mocker.spy(MockedDBConnection, "cursor") mocked_connection_close = mocker.spy(MockedDBConnection, "close") @@ -108,6 +106,7 @@ def test_latest_grid(mocker: MockerFixture): mocked_close.assert_called_once() assert isinstance(actual, pd.DataFrame) + def test_latest_grid_source(mocker: MockerFixture): mocked_cursor = mocker.spy(MockedDBConnection, "cursor") mocked_connection_close = mocker.spy(MockedDBConnection, "close") @@ -179,6 +178,7 @@ def test_latest_point(mocker: MockerFixture): mocked_close.assert_called_once() assert isinstance(actual, pd.DataFrame) + def test_latest_point_source(mocker: MockerFixture): mocked_cursor = mocker.spy(MockedDBConnection, "cursor") mocked_connection_close = mocker.spy(MockedDBConnection, "close") @@ -207,7 +207,6 @@ def test_latest_point_source(mocker: MockerFixture): assert isinstance(actual, pd.DataFrame) - def test_latest_point_fails(mocker: MockerFixture): mocker.spy(MockedDBConnection, "cursor") mocker.spy(MockedDBConnection, "close") diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py index c074500f0..75889bc4d 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py @@ -44,7 +44,7 @@ MOCKED_PARAMETER_DICT_GRID = { "forecast": "forecast", - "forecast_type" : "weather", + "forecast_type": "weather", "region": "mock_region", "data_security_level": "mock_security", "data_type": "mock_data_type", @@ -62,7 +62,7 @@ MOCKED_PARAMETER_DICT_POINT = { "forecast": "forecast", - "forecast_type" : "weather", + "forecast_type": "weather", "region": "mock_region", "data_security_level": "mock_security", "data_type": "mock_data_type", @@ -102,6 +102,7 @@ "forecast_run_timestamp_column": "EnqueuedTime", } + def test_raw_grid_(mocker: MockerFixture): mocked_cursor = mocker.spy(MockedDBConnection, "cursor") mocked_connection_close = mocker.spy(MockedDBConnection, "close") @@ -129,6 +130,7 @@ def test_raw_grid_(mocker: MockerFixture): mocked_close.assert_called_once() assert isinstance(actual, pd.DataFrame) + def test_raw_grid_source(mocker: MockerFixture): mocked_cursor = mocker.spy(MockedDBConnection, "cursor") mocked_connection_close = mocker.spy(MockedDBConnection, "close") @@ -157,7 +159,6 @@ def test_raw_grid_source(mocker: MockerFixture): assert isinstance(actual, pd.DataFrame) - def test_raw_grid_fails(mocker: MockerFixture): mocker.spy(MockedDBConnection, "cursor") mocker.spy(MockedDBConnection, "close") @@ -201,6 +202,7 @@ def test_raw_point(mocker: MockerFixture): mocked_close.assert_called_once() assert isinstance(actual, pd.DataFrame) + def test_raw_point_source(mocker: MockerFixture): mocked_cursor = mocker.spy(MockedDBConnection, "cursor") mocked_connection_close = mocker.spy(MockedDBConnection, "close") @@ -229,7 +231,6 @@ def test_raw_point_source(mocker: MockerFixture): assert isinstance(actual, pd.DataFrame) - def test_raw_point_fails(mocker: MockerFixture): mocker.spy(MockedDBConnection, "cursor") mocker.spy(MockedDBConnection, "close") From 41b119f89b5f5a9f294befa58bc4a27ccad31123 Mon Sep 17 00:00:00 2001 From: Amber-Rigg Date: Wed, 10 Jan 2024 18:50:10 +0000 Subject: [PATCH 11/16] Remove 'source' parameter from WeatherQueryBuilder methods Signed-off-by: Amber-Rigg --- .../queries/weather/weather_query_builder.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py index 5d29029ae..68556b7ea 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py @@ -24,10 +24,18 @@ class WeatherQueryBuilder: """ A builder for developing RTDIP queries using any delta table + """ parameters: dict connection: ConnectionInterface + close_connection: bool + data_source: str + tagname_column: str + timestamp_column: str + status_column: str + value_column: str + def connect(self, connection: ConnectionInterface): """ @@ -89,7 +97,6 @@ def raw_point( forecast_run_end_date (str): End date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) lat (float): latitude lon (float): longitude - source (optional str): Source of the data ie ECMWF time_zone (str): Timezone of the data include_bad_data (bool): Include "Bad" data points with True or remove "Bad" data points with False limit (optional int): The number of rows to be returned @@ -125,7 +132,6 @@ def latest_point( Args: lat (float): latitude lon (float): longitude - source (optional str): Source of the data ie ECMWF limit (optional int): The number of rows to be returned Returns: @@ -135,7 +141,6 @@ def latest_point( "source": self.data_source, "lat": lat, "lon": lon, - "source": source, "limit": limit, "supress_warning": True, } @@ -168,7 +173,6 @@ def raw_grid( # NOSONAR min_lon (float): Min longitude max_lat (float): Max latitude max_lon (float): Max longitude - source (optional str): Source of the data ie ECMWF time_zone (str): Timezone of the data include_bad_data (bool): Include "Bad" data points with True or remove "Bad" data points with False limit (optional int): The number of rows to be returned @@ -210,7 +214,6 @@ def latest_grid( min_lon (float): Min longitude max_lat (float): Max latitude max_lon (float): Max longitude - source (optional str): Source of the data ie ECMWF limit (optional int): The number of rows to be returned Returns: From c0563780f6f7ef0347b88804f319f7eacaf13f74 Mon Sep 17 00:00:00 2001 From: Amber-Rigg Date: Thu, 11 Jan 2024 09:32:54 +0000 Subject: [PATCH 12/16] Examples Added to Documentation Signed-off-by: Amber-Rigg --- .../query/functions/weather/latest.md | 113 ++++++++++++++- .../query/functions/weather/raw.md | 137 +++++++++++++++++- .../weather/weather_query_builder.md | 80 +++++++++- .../queries/weather/weather_query_builder.py | 1 - 4 files changed, 327 insertions(+), 4 deletions(-) diff --git a/docs/sdk/code-reference/query/functions/weather/latest.md b/docs/sdk/code-reference/query/functions/weather/latest.md index 6e7d33f49..0a36f50ef 100644 --- a/docs/sdk/code-reference/query/functions/weather/latest.md +++ b/docs/sdk/code-reference/query/functions/weather/latest.md @@ -1,2 +1,113 @@ # Weather Latest Function -::: src.sdk.python.rtdip_sdk.queries.weather.latest \ No newline at end of file +::: src.sdk.python.rtdip_sdk.queries.weather.latest + +## Examples get_point + +```python +from rtdip_sdk.authentication.azure import DefaultAuth +from rtdip_sdk.queries.weather.latest import get_point +from rtdip_sdk.connectors import DatabricksSQLConnection + +auth = DefaultAuth().authenticate() +token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token +connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + +params = { + "source": "forecast`.`weather`.`mock_region_mock_security_events_mock_data_type", + "lat": 1.1, + "lon": 1.1, +} + +x = get_point(connection, params) + +print(x) +``` + +```python +from rtdip_sdk.authentication.azure import DefaultAuth +from rtdip_sdk.queries.weather.latest import get_point +from rtdip_sdk.connectors import DatabricksSQLConnection + +auth = DefaultAuth().authenticate() +token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token +connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + +params = { + "forecast": "forecast", + "forecast_type": "weather", + "region": "mock_region", + "data_security_level": "mock_security", + "data_type": "mock_data_type", + "lat": 1.1, + "lon": 1.1, +} + +x = get_point(connection, params) + +print(x) +``` + +The above examples use two different routes to select the table, one directly uses a table name provided the other builds the table name from the variables. + +These examples are using [```DefaultAuth()```](../../../authentication/azure.md) and [```DatabricksSQLConnection()```](../../connectors/db-sql-connector.md) to authenticate and connect. You can find other ways to authenticate [here](../../../authentication/azure.md). The alternative built in connection methods are either by [```PYODBCSQLConnection()```](../../connectors/pyodbc-sql-connector.md), [```TURBODBCSQLConnection()```](../../connectors/turbodbc-sql-connector.md) or [```SparkConnection()```](../../connectors/spark-connector.md). + +!!! note "Note" + ```server_hostname``` and ```http_path``` can be found on the [SQL Warehouses Page](../../../../queries/databricks/sql-warehouses.md).
+ + +## Examples get_grid + +```python +from rtdip_sdk.authentication.azure import DefaultAuth +from rtdip_sdk.queries.weather.latest import get_grid +from rtdip_sdk.connectors import DatabricksSQLConnection + +auth = DefaultAuth().authenticate() +token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token +connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + +params = { + "source": "forecast`.`weather`.`mock_region_mock_security_events_mock_data_type", + "min_lat": 36, + "max_lat": 38, + "min_lon": -109.1, + "max_lon": -107.1, +} + +x = get_grid(connection, params) + +print(x) +``` + +```python +from rtdip_sdk.authentication.azure import DefaultAuth +from rtdip_sdk.queries.weather.latest import get_point +from rtdip_sdk.connectors import DatabricksSQLConnection + +auth = DefaultAuth().authenticate() +token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token +connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + +params = { + "forecast": "forecast", + "forecast_type": "weather", + "region": "mock_region", + "data_security_level": "mock_security", + "data_type": "mock_data_type", + "min_lat": 36, + "max_lat": 38, + "min_lon": -109.1, + "max_lon": -107.1, +} + +x = get_grid(connection, params) + +print(x) +``` + +The above examples use two different routes to select the table, one directly uses a table name provided the other builds the table name from the variables. + +These examples are using [```DefaultAuth()```](../../../authentication/azure.md) and [```DatabricksSQLConnection()```](../../connectors/db-sql-connector.md) to authenticate and connect. You can find other ways to authenticate [here](../../../authentication/azure.md). The alternative built in connection methods are either by [```PYODBCSQLConnection()```](../../connectors/pyodbc-sql-connector.md), [```TURBODBCSQLConnection()```](../../connectors/turbodbc-sql-connector.md) or [```SparkConnection()```](../../connectors/spark-connector.md). + +!!! note "Note" + ```server_hostname``` and ```http_path``` can be found on the [SQL Warehouses Page](../../../../queries/databricks/sql-warehouses.md).
\ No newline at end of file diff --git a/docs/sdk/code-reference/query/functions/weather/raw.md b/docs/sdk/code-reference/query/functions/weather/raw.md index 4e209aff0..e27bc17a6 100644 --- a/docs/sdk/code-reference/query/functions/weather/raw.md +++ b/docs/sdk/code-reference/query/functions/weather/raw.md @@ -1,2 +1,137 @@ # Weather Raw Function -::: src.sdk.python.rtdip_sdk.queries.weather.raw \ No newline at end of file +::: src.sdk.python.rtdip_sdk.queries.weather.raw + +## Examples get_point + +```python +from rtdip_sdk.authentication.azure import DefaultAuth +from rtdip_sdk.queries.weather.raw import get_point +from rtdip_sdk.connectors import DatabricksSQLConnection + +auth = DefaultAuth().authenticate() +token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token +connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + +params = { + "source": "forecast`.`weather`.`mock_region_mock_security_events_mock_data_type", + "lat": 1.1, + "lon": 1.1, + "start_date": "2020-01-01", + "end_date": "2020-01-02", + "forecast_run_start_date": "2020-01-01", + "forecast_run_end_date": "2020-01-02", + "timestamp_column": "EventTime", + "forecast_run_timestamp_column": "EnqueuedTime", +} + +x = get_point(connection, params) + +print(x) +``` + +```python +from rtdip_sdk.authentication.azure import DefaultAuth +from rtdip_sdk.queries.weather.raw import get_point +from rtdip_sdk.connectors import DatabricksSQLConnection + +auth = DefaultAuth().authenticate() +token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token +connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + +params = { + "forecast": "forecast", + "forecast_type": "weather", + "region": "mock_region", + "data_security_level": "mock_security", + "data_type": "mock_data_type", + "lat": 1.1, + "lon": 1.1, + "start_date": "2020-01-01", + "end_date": "2020-01-02", + "forecast_run_start_date": "2020-01-01", + "forecast_run_end_date": "2020-01-02", + "timestamp_column": "EventTime", + "forecast_run_timestamp_column": "EnqueuedTime", +} + +x = get_point(connection, params) + +print(x) +``` + +The above examples use two different routes to select the table, one directly uses a table name provided the other builds the table name from the variables. + +These examples are using [```DefaultAuth()```](../../../authentication/azure.md) and [```DatabricksSQLConnection()```](../../connectors/db-sql-connector.md) to authenticate and connect. You can find other ways to authenticate [here](../../../authentication/azure.md). The alternative built in connection methods are either by [```PYODBCSQLConnection()```](../../connectors/pyodbc-sql-connector.md), [```TURBODBCSQLConnection()```](../../connectors/turbodbc-sql-connector.md) or [```SparkConnection()```](../../connectors/spark-connector.md). + +!!! note "Note" + ```server_hostname``` and ```http_path``` can be found on the [SQL Warehouses Page](../../../../queries/databricks/sql-warehouses.md).
+ + +## Examples get_grid + +```python +from rtdip_sdk.authentication.azure import DefaultAuth +from rtdip_sdk.queries.weather.raw import get_grid +from rtdip_sdk.connectors import DatabricksSQLConnection + +auth = DefaultAuth().authenticate() +token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token +connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + +params = { + "source": "forecast`.`weather`.`mock_region_mock_security_events_mock_data_type", + "min_lat": 36, + "max_lat": 38, + "min_lon": -109.1, + "max_lon": -107.1, + "start_date": "2020-01-01", + "end_date": "2020-01-02", + "forecast_run_start_date": "2020-01-01", + "forecast_run_end_date": "2020-01-02", + "timestamp_column": "EventTime", + "forecast_run_timestamp_column": "EnqueuedTime", +} + +x = get_grid(connection, params) + +print(x) +``` + +```python +from rtdip_sdk.authentication.azure import DefaultAuth +from rtdip_sdk.queries.weather.raw import get_grid +from rtdip_sdk.connectors import DatabricksSQLConnection + +auth = DefaultAuth().authenticate() +token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token +connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + +params = { + "forecast": "forecast", + "forecast_type": "weather", + "region": "mock_region", + "data_security_level": "mock_security", + "data_type": "mock_data_type", + "min_lat": 36, + "max_lat": 38, + "min_lon": -109.1, + "max_lon": -107.1, + "start_date": "2020-01-01", + "end_date": "2020-01-02", + "forecast_run_start_date": "2020-01-01", + "forecast_run_end_date": "2020-01-02", + "timestamp_column": "EventTime", + "forecast_run_timestamp_column": "EnqueuedTime", +} + +x = get_grid(connection, params) + +print(x) +``` + +The above examples use two different routes to select the table, one directly uses a table name provided the other builds the table name from the variables. + +These examples are using [```DefaultAuth()```](../../../authentication/azure.md) and [```DatabricksSQLConnection()```](../../connectors/db-sql-connector.md) to authenticate and connect. You can find other ways to authenticate [here](../../../authentication/azure.md). The alternative built in connection methods are either by [```PYODBCSQLConnection()```](../../connectors/pyodbc-sql-connector.md), [```TURBODBCSQLConnection()```](../../connectors/turbodbc-sql-connector.md) or [```SparkConnection()```](../../connectors/spark-connector.md). + +!!! note "Note" + ```server_hostname``` and ```http_path``` can be found on the [SQL Warehouses Page](../../../../queries/databricks/sql-warehouses.md).
\ No newline at end of file diff --git a/docs/sdk/code-reference/query/functions/weather/weather_query_builder.md b/docs/sdk/code-reference/query/functions/weather/weather_query_builder.md index 369267444..9344d5415 100644 --- a/docs/sdk/code-reference/query/functions/weather/weather_query_builder.md +++ b/docs/sdk/code-reference/query/functions/weather/weather_query_builder.md @@ -1,2 +1,80 @@ # Weather Query Builder -::: src.sdk.python.rtdip_sdk.queries.weather.weather_query_builder \ No newline at end of file +::: src.sdk.python.rtdip_sdk.queries.weather.weather_query_builder + +## Examples + +```python +from rtdip_sdk.queries.weather.weather_query_builder import ( + WeatherQueryBuilder, +) +from rtdip_sdk.authentication.azure import DefaultAuth +from rtdip_sdk.connectors import DatabricksSQLConnection + + +auth = DefaultAuth().authenticate() +token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token +connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + +table = "weather.forecast.table" + + +data = ( + WeatherQueryBuilder() + .connect(connection) + .source(table, status_column=None) + .raw_point( + start_date="2021-01-01", + end_date="2021-01-02", + forecast_run_start_date="2021-01-01", + forecast_run_end_date="2021-01-02", + lat=0.1, + lon=0.1, + ) + ) + +data = ( + WeatherQueryBuilder() + .connect(connection) + .source(table, status_column=None) + .latest_point( + lat=0.1, + lon=0.1, + ) + ) + +data = ( + WeatherQueryBuilder() + .connect(connection) + .source(table, status_column=None) + .raw_grid( + start_date="2021-01-01", + end_date="2021-01-02", + forecast_run_start_date="2021-01-01", + forecast_run_end_date="2021-01-02", + min_lat=0.1, + max_lat=0.1, + min_lon=0.1, + max_lon=0.1, + ) + ) + +data = ( + WeatherQueryBuilder() + .connect(connection) + .source(table, status_column=None) + .latest_grid( + min_lat=0.1, + max_lat=0.1, + min_lon=0.1, + max_lon=0.1, + ) + ) + +``` + +The above example shows the 4 ways the Weather Query Builder can be use to get_point, get_grid, latest_point and latest_grid. + +These examples are using [```DefaultAuth()```](../../../authentication/azure.md) and [```DatabricksSQLConnection()```](../../connectors/db-sql-connector.md) to authenticate and connect. You can find other ways to authenticate [here](../../../authentication/azure.md). The alternative built in connection methods are either by [```PYODBCSQLConnection()```](../../connectors/pyodbc-sql-connector.md), [```TURBODBCSQLConnection()```](../../connectors/turbodbc-sql-connector.md) or [```SparkConnection()```](../../connectors/spark-connector.md). + +!!! note "Note" + ```server_hostname``` and ```http_path``` can be found on the [SQL Warehouses Page](../../../../queries/databricks/sql-warehouses.md).
\ No newline at end of file diff --git a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py index 68556b7ea..0ad65bc76 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py @@ -36,7 +36,6 @@ class WeatherQueryBuilder: status_column: str value_column: str - def connect(self, connection: ConnectionInterface): """ Specifies the connection to be used for the query From f6c025bfa153f430c7cc5aa237865f8f772b0434 Mon Sep 17 00:00:00 2001 From: Amber-Rigg Date: Thu, 11 Jan 2024 09:49:47 +0000 Subject: [PATCH 13/16] Remove duplication in tests Signed-off-by: Amber-Rigg --- tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py | 4 ++-- tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py index 4d5609a2c..1e85a62d3 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_latest.py @@ -115,7 +115,7 @@ def test_latest_grid_source(mocker: MockerFixture): MockedCursor, "fetchall_arrow", return_value=pa.Table.from_pandas( - pd.DataFrame(data={"e": [1], "f": [2], "g": [3], "h": [4]}) + pd.DataFrame(data={"a": [1], "c": [2], "g": [3], "i": [4]}) ), ) mocked_close = mocker.spy(MockedCursor, "close") @@ -187,7 +187,7 @@ def test_latest_point_source(mocker: MockerFixture): MockedCursor, "fetchall_arrow", return_value=pa.Table.from_pandas( - pd.DataFrame(data={"i": [1], "j": [2], "k": [3], "l": [4]}) + pd.DataFrame(data={"h": [1], "o": [2], "u": [3], "w": [4]}) ), ) mocked_close = mocker.spy(MockedCursor, "close") diff --git a/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py b/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py index 75889bc4d..aeea9b887 100644 --- a/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py +++ b/tests/sdk/python/rtdip_sdk/queries/weather/test_raw.py @@ -139,7 +139,7 @@ def test_raw_grid_source(mocker: MockerFixture): MockedCursor, "fetchall_arrow", return_value=pa.Table.from_pandas( - pd.DataFrame(data={"m": [1], "n": [2], "o": [3], "p": [4]}) + pd.DataFrame(data={"a": [1], "d": [2], "e": [3], "r": [4]}) ), ) mocked_close = mocker.spy(MockedCursor, "close") @@ -211,7 +211,7 @@ def test_raw_point_source(mocker: MockerFixture): MockedCursor, "fetchall_arrow", return_value=pa.Table.from_pandas( - pd.DataFrame(data={"q": [1], "r": [2], "s": [3], "t": [4]}) + pd.DataFrame(data={"u": [1], "v": [2], "w": [3], "x": [4]}) ), ) mocked_close = mocker.spy(MockedCursor, "close") From 0368eb71414c7ae8f708b9d31d21b3dcb55f9663 Mon Sep 17 00:00:00 2001 From: Amber-Rigg Date: Thu, 11 Jan 2024 10:21:36 +0000 Subject: [PATCH 14/16] Remove unused Variables and include timestamp coulmn name default Signed-off-by: Amber-Rigg --- src/sdk/python/rtdip_sdk/queries/weather/raw.py | 4 ++++ .../queries/weather/weather_query_builder.py | 16 ++++------------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/sdk/python/rtdip_sdk/queries/weather/raw.py b/src/sdk/python/rtdip_sdk/queries/weather/raw.py index 1f6f116ca..48f4af5a4 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/raw.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/raw.py @@ -44,6 +44,8 @@ def get_grid(connection: object, parameters_dict: dict) -> pd.DataFrame: end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) forecast_run_start_date (str): Start date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) forecast_run_end_date (str): End date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) + timestamp_column (str): The column which contains the the forecast output time. Default "EventTime". + forecast_run_timestamp_column (str): The column which contains whent the forecast was run. Default "EnqueuedTime". max_lat (float): Maximum latitude max_lon (float): Maximum longitude min_lat (float): Minimum latitude @@ -102,6 +104,8 @@ def get_point(connection: object, parameters_dict: dict) -> pd.DataFrame: end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) forecast_run_start_date (str): Start date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) forecast_run_end_date (str): End date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) + timestamp_column (str): The column which contains the the forecast output time. Default "EventTime". + forecast_run_timestamp_column (str): The column which contains whent the forecast was run. Default "EnqueuedTime. lat (float): latitude lon (float): longitude limit (optional int): The number of rows to be returned diff --git a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py index 0ad65bc76..b8f28c192 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py @@ -82,8 +82,6 @@ def raw_point( forecast_run_end_date: str, lat: float, lon: float, - time_zone: str = None, - include_bad_data: bool = False, limit: int = None, ) -> DataFrame: """ @@ -96,8 +94,6 @@ def raw_point( forecast_run_end_date (str): End date of the forecast run (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) lat (float): latitude lon (float): longitude - time_zone (str): Timezone of the data - include_bad_data (bool): Include "Bad" data points with True or remove "Bad" data points with False limit (optional int): The number of rows to be returned Returns: @@ -109,10 +105,10 @@ def raw_point( "end_date": end_date, "forecast_run_start_date": forecast_run_start_date, "forecast_run_end_date": forecast_run_end_date, + "timestamp_column": self.timestamp_column, + "forecast_run_timestamp_column": self.forecast_run_timestamp_column, "lat": lat, "lon": lon, - "time_zone": time_zone, - "include_bad_data": include_bad_data, "limit": limit, "supress_warning": True, } @@ -156,8 +152,6 @@ def raw_grid( # NOSONAR min_lon: float, max_lat: float, max_lon: float, - time_zone: str = None, - include_bad_data: bool = False, limit: int = None, # NOSONAR ) -> DataFrame: """ @@ -172,8 +166,6 @@ def raw_grid( # NOSONAR min_lon (float): Min longitude max_lat (float): Max latitude max_lon (float): Max longitude - time_zone (str): Timezone of the data - include_bad_data (bool): Include "Bad" data points with True or remove "Bad" data points with False limit (optional int): The number of rows to be returned Returns: @@ -185,12 +177,12 @@ def raw_grid( # NOSONAR "end_date": end_date, "forecast_run_start_date": forecast_run_start_date, "forecast_run_end_date": forecast_run_end_date, + "timestamp_column": self.timestamp_column, + "forecast_run_timestamp_column": self.forecast_run_timestamp_column, "min_lat": min_lat, "min_lon": min_lon, "max_lat": max_lat, "max_lon": max_lon, - "time_zone": time_zone, - "include_bad_data": include_bad_data, "limit": limit, "supress_warning": True, } From 6509e0670f30b6fc39a2bb9d6bd96a208763e39f Mon Sep 17 00:00:00 2001 From: Amber-Rigg Date: Thu, 11 Jan 2024 11:55:00 +0000 Subject: [PATCH 15/16] Documentation and Examples Formatting Signed-off-by: Amber-Rigg --- .../query/functions/weather/latest.md | 4 + .../query/functions/weather/raw.md | 4 + .../weather/weather_query_builder.md | 76 +--------- .../queries/weather/weather_query_builder.py | 134 +++++++++++++++++- 4 files changed, 141 insertions(+), 77 deletions(-) diff --git a/docs/sdk/code-reference/query/functions/weather/latest.md b/docs/sdk/code-reference/query/functions/weather/latest.md index 0a36f50ef..543866d34 100644 --- a/docs/sdk/code-reference/query/functions/weather/latest.md +++ b/docs/sdk/code-reference/query/functions/weather/latest.md @@ -8,6 +8,7 @@ from rtdip_sdk.authentication.azure import DefaultAuth from rtdip_sdk.queries.weather.latest import get_point from rtdip_sdk.connectors import DatabricksSQLConnection + auth = DefaultAuth().authenticate() token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) @@ -28,6 +29,7 @@ from rtdip_sdk.authentication.azure import DefaultAuth from rtdip_sdk.queries.weather.latest import get_point from rtdip_sdk.connectors import DatabricksSQLConnection + auth = DefaultAuth().authenticate() token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) @@ -62,6 +64,7 @@ from rtdip_sdk.authentication.azure import DefaultAuth from rtdip_sdk.queries.weather.latest import get_grid from rtdip_sdk.connectors import DatabricksSQLConnection + auth = DefaultAuth().authenticate() token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) @@ -84,6 +87,7 @@ from rtdip_sdk.authentication.azure import DefaultAuth from rtdip_sdk.queries.weather.latest import get_point from rtdip_sdk.connectors import DatabricksSQLConnection + auth = DefaultAuth().authenticate() token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) diff --git a/docs/sdk/code-reference/query/functions/weather/raw.md b/docs/sdk/code-reference/query/functions/weather/raw.md index e27bc17a6..0d3f434fd 100644 --- a/docs/sdk/code-reference/query/functions/weather/raw.md +++ b/docs/sdk/code-reference/query/functions/weather/raw.md @@ -8,6 +8,7 @@ from rtdip_sdk.authentication.azure import DefaultAuth from rtdip_sdk.queries.weather.raw import get_point from rtdip_sdk.connectors import DatabricksSQLConnection + auth = DefaultAuth().authenticate() token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) @@ -34,6 +35,7 @@ from rtdip_sdk.authentication.azure import DefaultAuth from rtdip_sdk.queries.weather.raw import get_point from rtdip_sdk.connectors import DatabricksSQLConnection + auth = DefaultAuth().authenticate() token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) @@ -74,6 +76,7 @@ from rtdip_sdk.authentication.azure import DefaultAuth from rtdip_sdk.queries.weather.raw import get_grid from rtdip_sdk.connectors import DatabricksSQLConnection + auth = DefaultAuth().authenticate() token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) @@ -102,6 +105,7 @@ from rtdip_sdk.authentication.azure import DefaultAuth from rtdip_sdk.queries.weather.raw import get_grid from rtdip_sdk.connectors import DatabricksSQLConnection + auth = DefaultAuth().authenticate() token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) diff --git a/docs/sdk/code-reference/query/functions/weather/weather_query_builder.md b/docs/sdk/code-reference/query/functions/weather/weather_query_builder.md index 9344d5415..917bf15b9 100644 --- a/docs/sdk/code-reference/query/functions/weather/weather_query_builder.md +++ b/docs/sdk/code-reference/query/functions/weather/weather_query_builder.md @@ -1,80 +1,8 @@ # Weather Query Builder ::: src.sdk.python.rtdip_sdk.queries.weather.weather_query_builder -## Examples - -```python -from rtdip_sdk.queries.weather.weather_query_builder import ( - WeatherQueryBuilder, -) -from rtdip_sdk.authentication.azure import DefaultAuth -from rtdip_sdk.connectors import DatabricksSQLConnection - - -auth = DefaultAuth().authenticate() -token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token -connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) - -table = "weather.forecast.table" - - -data = ( - WeatherQueryBuilder() - .connect(connection) - .source(table, status_column=None) - .raw_point( - start_date="2021-01-01", - end_date="2021-01-02", - forecast_run_start_date="2021-01-01", - forecast_run_end_date="2021-01-02", - lat=0.1, - lon=0.1, - ) - ) - -data = ( - WeatherQueryBuilder() - .connect(connection) - .source(table, status_column=None) - .latest_point( - lat=0.1, - lon=0.1, - ) - ) - -data = ( - WeatherQueryBuilder() - .connect(connection) - .source(table, status_column=None) - .raw_grid( - start_date="2021-01-01", - end_date="2021-01-02", - forecast_run_start_date="2021-01-01", - forecast_run_end_date="2021-01-02", - min_lat=0.1, - max_lat=0.1, - min_lon=0.1, - max_lon=0.1, - ) - ) - -data = ( - WeatherQueryBuilder() - .connect(connection) - .source(table, status_column=None) - .latest_grid( - min_lat=0.1, - max_lat=0.1, - min_lon=0.1, - max_lon=0.1, - ) - ) - -``` - -The above example shows the 4 ways the Weather Query Builder can be use to get_point, get_grid, latest_point and latest_grid. - -These examples are using [```DefaultAuth()```](../../../authentication/azure.md) and [```DatabricksSQLConnection()```](../../connectors/db-sql-connector.md) to authenticate and connect. You can find other ways to authenticate [here](../../../authentication/azure.md). The alternative built in connection methods are either by [```PYODBCSQLConnection()```](../../connectors/pyodbc-sql-connector.md), [```TURBODBCSQLConnection()```](../../connectors/turbodbc-sql-connector.md) or [```SparkConnection()```](../../connectors/spark-connector.md). +!!! note "Note" + These examples are using [```DefaultAuth()```](../../../authentication/azure.md) and [```DatabricksSQLConnection()```](../../connectors/db-sql-connector.md) to authenticate and connect. You can find other ways to authenticate [here](../../../authentication/azure.md). The alternative built in connection methods are either by [```PYODBCSQLConnection()```](../../connectors/pyodbc-sql-connector.md), [```TURBODBCSQLConnection()```](../../connectors/turbodbc-sql-connector.md) or [```SparkConnection()```](../../connectors/spark-connector.md).
!!! note "Note" ```server_hostname``` and ```http_path``` can be found on the [SQL Warehouses Page](../../../../queries/databricks/sql-warehouses.md).
\ No newline at end of file diff --git a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py index b8f28c192..5485f606a 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py @@ -23,7 +23,7 @@ class WeatherQueryBuilder: """ - A builder for developing RTDIP queries using any delta table + A builder for developing RTDIP forecast queries using any delta table """ @@ -87,6 +87,39 @@ def raw_point( """ A function to return back raw data for a point. + **Example:** + + ```python + from rtdip_sdk.queries.weather.weather_query_builder import ( + WeatherQueryBuilder, + ) + from rtdip_sdk.authentication.azure import DefaultAuth + from rtdip_sdk.connectors import DatabricksSQLConnection + + + auth = DefaultAuth().authenticate() + token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token + connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + + table = "weather.forecast.table" + + data = ( + WeatherQueryBuilder() + .connect(connection) + .source(table, status_column=None) + .raw_point( + start_date="2021-01-01", + end_date="2021-01-02", + forecast_run_start_date="2021-01-01", + forecast_run_end_date="2021-01-02", + lat=0.1, + lon=0.1, + ) + ) + + print(data) + ``` + Args: start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) end_date (str): End date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) @@ -124,6 +157,35 @@ def latest_point( """ A function to return back the latest data for a point. + **Example:** + + ```python + from rtdip_sdk.queries.weather.weather_query_builder import ( + WeatherQueryBuilder, + ) + from rtdip_sdk.authentication.azure import DefaultAuth + from rtdip_sdk.connectors import DatabricksSQLConnection + + + auth = DefaultAuth().authenticate() + token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token + connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + + table = "weather.forecast.table" + + data = ( + WeatherQueryBuilder() + .connect(connection) + .source(table, status_column=None) + .latest_point( + lat=0.1, + lon=0.1, + ) + ) + + print(data) + ``` + Args: lat (float): latitude lon (float): longitude @@ -155,7 +217,42 @@ def raw_grid( # NOSONAR limit: int = None, # NOSONAR ) -> DataFrame: """ - A function to return back raw data for a point. + A function to return back raw data for a grid. + + **Example:** + + ```python + from rtdip_sdk.queries.weather.weather_query_builder import ( + WeatherQueryBuilder, + ) + from rtdip_sdk.authentication.azure import DefaultAuth + from rtdip_sdk.connectors import DatabricksSQLConnection + + + auth = DefaultAuth().authenticate() + token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token + connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + + table = "weather.forecast.table" + + data = ( + WeatherQueryBuilder() + .connect(connection) + .source(table, status_column=None) + .raw_grid( + start_date="2021-01-01", + end_date="2021-01-02", + forecast_run_start_date="2021-01-01", + forecast_run_end_date="2021-01-02", + min_lat=0.1, + max_lat=0.1, + min_lon=0.1, + max_lon=0.1, + ) + ) + + print(data) + ``` Args: start_date (str): Start date (Either a date in the format YY-MM-DD or a datetime in the format YYY-MM-DDTHH:MM:SS or specify the timezone offset in the format YYYY-MM-DDTHH:MM:SS+zz:zz) @@ -198,7 +295,38 @@ def latest_grid( limit: int = None, ) -> DataFrame: """ - A function to return back the latest data for a point. + A function to return back the latest data for a grid. + + **Example:** + + ```python + from rtdip_sdk.queries.weather.weather_query_builder import ( + WeatherQueryBuilder, + ) + from rtdip_sdk.authentication.azure import DefaultAuth + from rtdip_sdk.connectors import DatabricksSQLConnection + + + auth = DefaultAuth().authenticate() + token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token + connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) + + table = "weather.forecast.table" + + data = ( + WeatherQueryBuilder() + .connect(connection) + .source(table, status_column=None) + .latest_grid( + min_lat=0.1, + max_lat=0.1, + min_lon=0.1, + max_lon=0.1, + ) + ) + + print(data) + ``` Args: min_lat (float): Min latitude From 805208a64f4cfb23e87e5923122628efda60726a Mon Sep 17 00:00:00 2001 From: Amber-Rigg Date: Thu, 11 Jan 2024 14:04:08 +0000 Subject: [PATCH 16/16] Documentation Update Signed-off-by: Amber-Rigg --- .../query/functions/weather/latest.md | 68 ++-------------- .../query/functions/weather/raw.md | 80 ++----------------- .../queries/weather/weather_query_builder.py | 24 +----- 3 files changed, 16 insertions(+), 156 deletions(-) diff --git a/docs/sdk/code-reference/query/functions/weather/latest.md b/docs/sdk/code-reference/query/functions/weather/latest.md index 543866d34..01532d537 100644 --- a/docs/sdk/code-reference/query/functions/weather/latest.md +++ b/docs/sdk/code-reference/query/functions/weather/latest.md @@ -1,42 +1,20 @@ # Weather Latest Function ::: src.sdk.python.rtdip_sdk.queries.weather.latest -## Examples get_point +## Example get_point ```python from rtdip_sdk.authentication.azure import DefaultAuth from rtdip_sdk.queries.weather.latest import get_point from rtdip_sdk.connectors import DatabricksSQLConnection - auth = DefaultAuth().authenticate() token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) params = { - "source": "forecast`.`weather`.`mock_region_mock_security_events_mock_data_type", - "lat": 1.1, - "lon": 1.1, -} - -x = get_point(connection, params) - -print(x) -``` - -```python -from rtdip_sdk.authentication.azure import DefaultAuth -from rtdip_sdk.queries.weather.latest import get_point -from rtdip_sdk.connectors import DatabricksSQLConnection - - -auth = DefaultAuth().authenticate() -token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token -connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) - -params = { - "forecast": "forecast", - "forecast_type": "weather", + "forecast": "mock_forecast", + "forecast_type": "mock_weather", "region": "mock_region", "data_security_level": "mock_security", "data_type": "mock_data_type", @@ -48,53 +26,21 @@ x = get_point(connection, params) print(x) ``` - -The above examples use two different routes to select the table, one directly uses a table name provided the other builds the table name from the variables. - -These examples are using [```DefaultAuth()```](../../../authentication/azure.md) and [```DatabricksSQLConnection()```](../../connectors/db-sql-connector.md) to authenticate and connect. You can find other ways to authenticate [here](../../../authentication/azure.md). The alternative built in connection methods are either by [```PYODBCSQLConnection()```](../../connectors/pyodbc-sql-connector.md), [```TURBODBCSQLConnection()```](../../connectors/turbodbc-sql-connector.md) or [```SparkConnection()```](../../connectors/spark-connector.md). - -!!! note "Note" - ```server_hostname``` and ```http_path``` can be found on the [SQL Warehouses Page](../../../../queries/databricks/sql-warehouses.md).
- -## Examples get_grid - -```python -from rtdip_sdk.authentication.azure import DefaultAuth -from rtdip_sdk.queries.weather.latest import get_grid -from rtdip_sdk.connectors import DatabricksSQLConnection - - -auth = DefaultAuth().authenticate() -token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token -connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) - -params = { - "source": "forecast`.`weather`.`mock_region_mock_security_events_mock_data_type", - "min_lat": 36, - "max_lat": 38, - "min_lon": -109.1, - "max_lon": -107.1, -} - -x = get_grid(connection, params) - -print(x) -``` +## Example get_grid ```python from rtdip_sdk.authentication.azure import DefaultAuth from rtdip_sdk.queries.weather.latest import get_point from rtdip_sdk.connectors import DatabricksSQLConnection - auth = DefaultAuth().authenticate() token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) params = { - "forecast": "forecast", - "forecast_type": "weather", + "forecast": "mock_forecast", + "forecast_type": "mock_weather", "region": "mock_region", "data_security_level": "mock_security", "data_type": "mock_data_type", @@ -109,8 +55,6 @@ x = get_grid(connection, params) print(x) ``` -The above examples use two different routes to select the table, one directly uses a table name provided the other builds the table name from the variables. - These examples are using [```DefaultAuth()```](../../../authentication/azure.md) and [```DatabricksSQLConnection()```](../../connectors/db-sql-connector.md) to authenticate and connect. You can find other ways to authenticate [here](../../../authentication/azure.md). The alternative built in connection methods are either by [```PYODBCSQLConnection()```](../../connectors/pyodbc-sql-connector.md), [```TURBODBCSQLConnection()```](../../connectors/turbodbc-sql-connector.md) or [```SparkConnection()```](../../connectors/spark-connector.md). !!! note "Note" diff --git a/docs/sdk/code-reference/query/functions/weather/raw.md b/docs/sdk/code-reference/query/functions/weather/raw.md index 0d3f434fd..bdbaa0f73 100644 --- a/docs/sdk/code-reference/query/functions/weather/raw.md +++ b/docs/sdk/code-reference/query/functions/weather/raw.md @@ -1,48 +1,20 @@ # Weather Raw Function ::: src.sdk.python.rtdip_sdk.queries.weather.raw -## Examples get_point +## Example get_point ```python from rtdip_sdk.authentication.azure import DefaultAuth from rtdip_sdk.queries.weather.raw import get_point from rtdip_sdk.connectors import DatabricksSQLConnection - auth = DefaultAuth().authenticate() token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) params = { - "source": "forecast`.`weather`.`mock_region_mock_security_events_mock_data_type", - "lat": 1.1, - "lon": 1.1, - "start_date": "2020-01-01", - "end_date": "2020-01-02", - "forecast_run_start_date": "2020-01-01", - "forecast_run_end_date": "2020-01-02", - "timestamp_column": "EventTime", - "forecast_run_timestamp_column": "EnqueuedTime", -} - -x = get_point(connection, params) - -print(x) -``` - -```python -from rtdip_sdk.authentication.azure import DefaultAuth -from rtdip_sdk.queries.weather.raw import get_point -from rtdip_sdk.connectors import DatabricksSQLConnection - - -auth = DefaultAuth().authenticate() -token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token -connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) - -params = { - "forecast": "forecast", - "forecast_type": "weather", + "forecast": "mock_forecast", + "forecast_type": "mock_weather", "region": "mock_region", "data_security_level": "mock_security", "data_type": "mock_data_type", @@ -61,58 +33,20 @@ x = get_point(connection, params) print(x) ``` -The above examples use two different routes to select the table, one directly uses a table name provided the other builds the table name from the variables. - -These examples are using [```DefaultAuth()```](../../../authentication/azure.md) and [```DatabricksSQLConnection()```](../../connectors/db-sql-connector.md) to authenticate and connect. You can find other ways to authenticate [here](../../../authentication/azure.md). The alternative built in connection methods are either by [```PYODBCSQLConnection()```](../../connectors/pyodbc-sql-connector.md), [```TURBODBCSQLConnection()```](../../connectors/turbodbc-sql-connector.md) or [```SparkConnection()```](../../connectors/spark-connector.md). - -!!! note "Note" - ```server_hostname``` and ```http_path``` can be found on the [SQL Warehouses Page](../../../../queries/databricks/sql-warehouses.md).
- - -## Examples get_grid +## Example get_grid ```python from rtdip_sdk.authentication.azure import DefaultAuth from rtdip_sdk.queries.weather.raw import get_grid from rtdip_sdk.connectors import DatabricksSQLConnection - auth = DefaultAuth().authenticate() token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) params = { - "source": "forecast`.`weather`.`mock_region_mock_security_events_mock_data_type", - "min_lat": 36, - "max_lat": 38, - "min_lon": -109.1, - "max_lon": -107.1, - "start_date": "2020-01-01", - "end_date": "2020-01-02", - "forecast_run_start_date": "2020-01-01", - "forecast_run_end_date": "2020-01-02", - "timestamp_column": "EventTime", - "forecast_run_timestamp_column": "EnqueuedTime", -} - -x = get_grid(connection, params) - -print(x) -``` - -```python -from rtdip_sdk.authentication.azure import DefaultAuth -from rtdip_sdk.queries.weather.raw import get_grid -from rtdip_sdk.connectors import DatabricksSQLConnection - - -auth = DefaultAuth().authenticate() -token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token -connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) - -params = { - "forecast": "forecast", - "forecast_type": "weather", + "forecast": "mock_forecast", + "forecast_type": "mock_weather", "region": "mock_region", "data_security_level": "mock_security", "data_type": "mock_data_type", @@ -133,8 +67,6 @@ x = get_grid(connection, params) print(x) ``` -The above examples use two different routes to select the table, one directly uses a table name provided the other builds the table name from the variables. - These examples are using [```DefaultAuth()```](../../../authentication/azure.md) and [```DatabricksSQLConnection()```](../../connectors/db-sql-connector.md) to authenticate and connect. You can find other ways to authenticate [here](../../../authentication/azure.md). The alternative built in connection methods are either by [```PYODBCSQLConnection()```](../../connectors/pyodbc-sql-connector.md), [```TURBODBCSQLConnection()```](../../connectors/turbodbc-sql-connector.md) or [```SparkConnection()```](../../connectors/spark-connector.md). !!! note "Note" diff --git a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py index 5485f606a..f6be94175 100644 --- a/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py +++ b/src/sdk/python/rtdip_sdk/queries/weather/weather_query_builder.py @@ -88,7 +88,6 @@ def raw_point( A function to return back raw data for a point. **Example:** - ```python from rtdip_sdk.queries.weather.weather_query_builder import ( WeatherQueryBuilder, @@ -96,17 +95,14 @@ def raw_point( from rtdip_sdk.authentication.azure import DefaultAuth from rtdip_sdk.connectors import DatabricksSQLConnection - auth = DefaultAuth().authenticate() token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) - table = "weather.forecast.table" - data = ( WeatherQueryBuilder() .connect(connection) - .source(table, status_column=None) + .source("example.forecast.table") .raw_point( start_date="2021-01-01", end_date="2021-01-02", @@ -158,7 +154,6 @@ def latest_point( A function to return back the latest data for a point. **Example:** - ```python from rtdip_sdk.queries.weather.weather_query_builder import ( WeatherQueryBuilder, @@ -166,17 +161,14 @@ def latest_point( from rtdip_sdk.authentication.azure import DefaultAuth from rtdip_sdk.connectors import DatabricksSQLConnection - auth = DefaultAuth().authenticate() token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) - table = "weather.forecast.table" - data = ( WeatherQueryBuilder() .connect(connection) - .source(table, status_column=None) + .source("example.forecast.table") .latest_point( lat=0.1, lon=0.1, @@ -220,7 +212,6 @@ def raw_grid( # NOSONAR A function to return back raw data for a grid. **Example:** - ```python from rtdip_sdk.queries.weather.weather_query_builder import ( WeatherQueryBuilder, @@ -228,17 +219,14 @@ def raw_grid( # NOSONAR from rtdip_sdk.authentication.azure import DefaultAuth from rtdip_sdk.connectors import DatabricksSQLConnection - auth = DefaultAuth().authenticate() token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) - table = "weather.forecast.table" - data = ( WeatherQueryBuilder() .connect(connection) - .source(table, status_column=None) + .source("example.forecast.table") .raw_grid( start_date="2021-01-01", end_date="2021-01-02", @@ -298,7 +286,6 @@ def latest_grid( A function to return back the latest data for a grid. **Example:** - ```python from rtdip_sdk.queries.weather.weather_query_builder import ( WeatherQueryBuilder, @@ -306,17 +293,14 @@ def latest_grid( from rtdip_sdk.authentication.azure import DefaultAuth from rtdip_sdk.connectors import DatabricksSQLConnection - auth = DefaultAuth().authenticate() token = auth.get_token("2ff814a6-3304-4ab8-85cb-cd0e6f879c1d/.default").token connection = DatabricksSQLConnection("{server_hostname}", "{http_path}", token) - table = "weather.forecast.table" - data = ( WeatherQueryBuilder() .connect(connection) - .source(table, status_column=None) + .source("example.forecast.table") .latest_grid( min_lat=0.1, max_lat=0.1,