From c0b840fd3e3894ae2377c9f648a87f331de104a0 Mon Sep 17 00:00:00 2001 From: David Fairbrother Date: Fri, 3 Jan 2025 15:45:20 +0000 Subject: [PATCH] WIP: Add placement API to support getting HV/provider resources In Yoga+ placement should be used for determining usage...etc. with Horizon having switched to it upstream. Unfortunately, we don't have full support in the latest SDK for getting usage, so add some static methods and data classes to support this until we/someone else upstream the work. --- openstackquery/__init__.py | 1 + openstackquery/api/query_objects.py | 8 ++ .../enums/props/placement_properties.py | 68 +++++++++++ .../handlers/server_side_handler.py | 2 + openstackquery/mappings/placement_mapping.py | 115 ++++++++++++++++++ openstackquery/runners/placement_runner.py | 112 +++++++++++++++++ .../structs/resource_provider_usage.py | 23 ++++ 7 files changed, 329 insertions(+) create mode 100644 openstackquery/enums/props/placement_properties.py create mode 100644 openstackquery/mappings/placement_mapping.py create mode 100644 openstackquery/runners/placement_runner.py create mode 100644 openstackquery/structs/resource_provider_usage.py diff --git a/openstackquery/__init__.py b/openstackquery/__init__.py index 55e7261..73c0f5f 100644 --- a/openstackquery/__init__.py +++ b/openstackquery/__init__.py @@ -7,6 +7,7 @@ ProjectQuery, ImageQuery, HypervisorQuery, + PlacementQuery, ) # Create logger diff --git a/openstackquery/api/query_objects.py b/openstackquery/api/query_objects.py index d522b9c..0db9460 100644 --- a/openstackquery/api/query_objects.py +++ b/openstackquery/api/query_objects.py @@ -5,6 +5,7 @@ from openstackquery.mappings.hypervisor_mapping import HypervisorMapping from openstackquery.mappings.image_mapping import ImageMapping from openstackquery.mappings.mapping_interface import MappingInterface +from openstackquery.mappings.placement_mapping import PlacementMapping from openstackquery.mappings.project_mapping import ProjectMapping from openstackquery.mappings.server_mapping import ServerMapping from openstackquery.mappings.user_mapping import UserMapping @@ -70,3 +71,10 @@ def HypervisorQuery() -> "QueryAPI": Simple helper function to setup a query using a factory """ return get_common(HypervisorMapping) + + +def PlacementQuery() -> "QueryAPI": + """ + Simple helper function to setup a query using a factory + """ + return get_common(PlacementMapping) diff --git a/openstackquery/enums/props/placement_properties.py b/openstackquery/enums/props/placement_properties.py new file mode 100644 index 0000000..91bacda --- /dev/null +++ b/openstackquery/enums/props/placement_properties.py @@ -0,0 +1,68 @@ +from enum import auto +from typing import Dict, Optional + +from openstackquery.enums.props.prop_enum import PropEnum, PropFunc +from openstackquery.exceptions.query_property_mapping_error import ( + QueryPropertyMappingError, +) + + +class PlacementProperties(PropEnum): + """ + An enum class for currently used placement properties + """ + RESOURCE_PROVIDER_ID = auto() + RESOURCE_PROVIDER_NAME = auto() + VCPUS_USED = auto() + VCPUS_AVAIL = auto() + MEMORY_MB_USED = auto() + MEMORY_MB_AVAIL = auto() + DISK_GB_USED = auto() + DISK_GB_AVAIL = auto() + + @staticmethod + def _get_aliases() -> Dict: + """ + A method that returns all valid string alias mappings + """ + return { + PlacementProperties.RESOURCE_PROVIDER_ID: ["resource_provider_id", "resource_provider_uuid", "id"], + PlacementProperties.RESOURCE_PROVIDER_NAME: ["resource_name", "name", "provider_name"], + PlacementProperties.VCPUS_USED: ["vcpus_used"], + PlacementProperties.VCPUS_AVAIL: ["vcpus_avail"], + PlacementProperties.MEMORY_MB_USED: ["memory_mb_used"], + PlacementProperties.MEMORY_MB_AVAIL: ["memory_mb_avail"], + PlacementProperties.DISK_GB_USED: ["disk_gb_used"], + PlacementProperties.DISK_GB_AVAIL: ["disk_gb_avail"], + } + + @staticmethod + def get_prop_mapping(prop) -> Optional[PropFunc]: + """ + Method that returns the property function if function mapping exists for a given Hypervisor Enum + how to get specified property from a ResourceProviderUsage object + :param prop: A HypervisorProperty Enum for which a function may exist for + """ + mapping = { + PlacementProperties.RESOURCE_PROVIDER_ID: lambda a: a["id"], + PlacementProperties.RESOURCE_PROVIDER_NAME: lambda a: a["name"], + PlacementProperties.VCPUS_AVAIL: lambda a: a["VCPU_AVAIL"], + PlacementProperties.MEMORY_MB_AVAIL: lambda a: a["MEMORY_MB_AVAIL"], + PlacementProperties.DISK_GB_AVAIL: lambda a: a["DISK_GB_AVAIL"], + PlacementProperties.VCPUS_USED: lambda a: a["VCPU_USED"], + PlacementProperties.MEMORY_MB_USED: lambda a: a["MEMORY_MB_USED"], + PlacementProperties.DISK_GB_USED: lambda a: a["DISK_GB_USED"], + } + try: + return mapping[prop] + except KeyError as exp: + raise QueryPropertyMappingError( + f"Error: failed to get property mapping, property {prop.name} is not supported in PlacementProperties" + ) from exp + + @staticmethod + def get_marker_prop_func(): + """ + A getter method to return marker property function for pagination + """ + return PlacementProperties.get_prop_mapping(PlacementProperties.RESOURCE_PROVIDER_ID) diff --git a/openstackquery/handlers/server_side_handler.py b/openstackquery/handlers/server_side_handler.py index 50e3fc6..391744f 100644 --- a/openstackquery/handlers/server_side_handler.py +++ b/openstackquery/handlers/server_side_handler.py @@ -87,6 +87,8 @@ def get_filters( try: filters = filter_func(**params) except (KeyError, TypeError) as err: + # Dev note: your lambda must take "value" as the lambda + # argument if you arrive here adding new mappings raise QueryPresetMappingError( "Preset Argument Error: failed to build server-side openstacksdk filters for preset:prop: " f"'{preset.name}':'{prop.name}' " diff --git a/openstackquery/mappings/placement_mapping.py b/openstackquery/mappings/placement_mapping.py new file mode 100644 index 0000000..1a25f53 --- /dev/null +++ b/openstackquery/mappings/placement_mapping.py @@ -0,0 +1,115 @@ +from typing import Type + +from openstackquery.enums.props.hypervisor_properties import HypervisorProperties +from openstackquery.enums.props.placement_properties import PlacementProperties +from openstackquery.enums.props.prop_enum import PropEnum +from openstackquery.enums.query_presets import ( + QueryPresetsGeneric, + QueryPresetsString, + QueryPresetsInteger, +) +from openstackquery.handlers.client_side_handler_generic import ( + ClientSideHandlerGeneric, +) +from openstackquery.handlers.client_side_handler_integer import ( + ClientSideHandlerInteger, +) +from openstackquery.handlers.client_side_handler_string import ClientSideHandlerString +from openstackquery.handlers.server_side_handler import ServerSideHandler +from openstackquery.mappings.mapping_interface import MappingInterface +from openstackquery.runners.placement_runner import PlacementRunner + +from openstackquery.runners.runner_wrapper import RunnerWrapper +from openstackquery.structs.query_client_side_handlers import QueryClientSideHandlers + + +class PlacementMapping(MappingInterface): + """ + Mapping class for querying Openstack placement and resource objects + Define property mappings, kwarg mappings and filter function mappings, + and runner mapping related to placement and resources here + """ + + @staticmethod + def get_chain_mappings(): + """ + Should return a dictionary containing property pairs mapped to query mappings. + This is used to define how to chain results from this query to other possible queries + """ + return {PlacementProperties.RESOURCE_PROVIDER_NAME: HypervisorProperties.HYPERVISOR_NAME} + + @staticmethod + def get_runner_mapping() -> Type[RunnerWrapper]: + """ + Returns a mapping to associated Runner class for the Query (placement and resourceRunner) + """ + return PlacementRunner + + @staticmethod + def get_prop_mapping() -> Type[PropEnum]: + """ + Returns a mapping of valid presets for server side attributes (placement and resourceProperties) + """ + return PlacementProperties + + @staticmethod + def get_server_side_handler() -> ServerSideHandler: + """ + method to configure a server handler which can be used to get 'filter' keyword arguments that + can be passed to openstack function conn.placement.resource_providers() to filter results for a valid preset-property + pair + + valid filters documented here: + https://docs.openstack.org/openstacksdk/latest/user/proxies/placement.html + """ + return ServerSideHandler({ + QueryPresetsGeneric.EQUAL_TO: { + PlacementProperties.RESOURCE_PROVIDER_ID: lambda value: {"id": value}, + PlacementProperties.RESOURCE_PROVIDER_NAME: lambda value: {"name": value}, + } + }) + + @staticmethod + def get_client_side_handlers() -> QueryClientSideHandlers: + """ + method to configure a set of client-side handlers which can be used to get local filter functions + corresponding to valid preset-property pairs. These filter functions can be used to filter results after + listing all placement and resources. + """ + integer_prop_list = [ + PlacementProperties.VCPUS_AVAIL, + PlacementProperties.MEMORY_MB_AVAIL, + PlacementProperties.DISK_GB_AVAIL, + PlacementProperties.VCPUS_USED, + PlacementProperties.MEMORY_MB_USED, + PlacementProperties.DISK_GB_USED, + ] + + return QueryClientSideHandlers( + generic_handler = ClientSideHandlerGeneric( + { + QueryPresetsGeneric.EQUAL_TO: ["*"], + QueryPresetsGeneric.NOT_EQUAL_TO: ["*"], + QueryPresetsGeneric.ANY_IN: ["*"], + QueryPresetsGeneric.NOT_ANY_IN: ["*"], + } + ), + # set string query preset mappings + string_handler = ClientSideHandlerString( + { + QueryPresetsString.MATCHES_REGEX: [ + PlacementProperties.RESOURCE_PROVIDER_ID, + PlacementProperties.RESOURCE_PROVIDER_NAME, + ] + } + ), + datetime_handler = None, + integer_handler=ClientSideHandlerInteger( + { + QueryPresetsInteger.LESS_THAN: integer_prop_list, + QueryPresetsInteger.LESS_THAN_OR_EQUAL_TO: integer_prop_list, + QueryPresetsInteger.GREATER_THAN: integer_prop_list, + QueryPresetsInteger.GREATER_THAN_OR_EQUAL_TO: integer_prop_list, + } + ), + ) diff --git a/openstackquery/runners/placement_runner.py b/openstackquery/runners/placement_runner.py new file mode 100644 index 0000000..d3844f8 --- /dev/null +++ b/openstackquery/runners/placement_runner.py @@ -0,0 +1,112 @@ +import logging +from typing import List, Optional, Dict + +from openstack.placement.v1.resource_provider import ResourceProvider + +from openstackquery.aliases import OpenstackResourceObj, ServerSideFilters, ServerSideFilter +from openstackquery.openstack_connection import OpenstackConnection +from openstackquery.runners.runner_utils import RunnerUtils +from openstackquery.runners.runner_wrapper import RunnerWrapper +from openstackquery.structs.resource_provider_usage import ResourceProviderUsage + +logger = logging.getLogger(__name__) + + +class PlacementRunner(RunnerWrapper): + """ + Runner class for openstack Hypervisor resource + HypervisorRunner encapsulates running any openstacksdk Hypervisor commands + """ + + RESOURCE_TYPE = ResourceProvider + + def parse_meta_params(self, conn: OpenstackConnection, **kwargs): + """ + This class has no meta-params available, so this method is a no-op + """ + return super().parse_meta_params(conn, **kwargs) + + def _convert_to_custom_obj(self, conn: OpenstackConnection, obj: ResourceProvider) -> OpenstackResourceObj: + """ + Converts an openstacksdk ResourceProvider object to a ResourceProviderUsage object + including populating the available and used resources from the placement API + :param conn: Openstack connection + :param obj: Openstack placement resource provider object + :return: A ResourceProviderUsage object + """ + usage = self._get_usage_info(conn, obj) + avail = self._get_availability_info(conn, obj) + return ResourceProviderUsage( + id=obj.id, + name=obj.name, + VCPU_USED=usage["VCPU"], + MEMORY_MB_USED=usage["MEMORY_MB"], + DISK_GB_USED=usage["DISK_GB"], + VCPU_AVAIL=avail["VCPU"], + MEMORY_MB_AVAIL=avail["MEMORY_MB"], + DISK_GB_AVAIL=avail["DISK_GB"], + ) + + @staticmethod + def _get_availability_info(conn: OpenstackConnection, resource_provider_obj: ResourceProvider) -> Dict: + """ + Gets availability stats for a given placement resource provider + across the following resource classes: VCPU, MEMORY_MB, DISK_GB + :param conn: Openstack connection + :param resource_provider_obj: Openstack placement resource provider object + :return: A dictionary with the summed availability stats using the class name as a key + """ + summed_classes = {} + for resource_class in ["VCPU", "MEMORY_MB", "DISK_GB"]: + placement_inventories = conn.placement.resource_provider_inventories(resource_provider_obj, resource_class=resource_class) + # A resource provider can have n number of inventories for a given resource class + if not placement_inventories: + logger.warning("No available resources found for resource provider: %s", resource_provider_obj.id) + summed_classes[resource_class] = 0 + else: + summed_classes[resource_class] = sum(i["total"] for i in placement_inventories) + return summed_classes + + @staticmethod + def _get_usage_info( + conn: OpenstackConnection, resource_provider_obj: ResourceProvider + ) -> Dict: + """ + Gets usage stats for a given placement resource provider + :param conn: Openstack connection + :param resource_provider_obj: Openstack placement resource provider object + :return: A ResourceProviderUsage object with usage stats + """ + # The following should be up-streamed to openstacksdk at some point + # It is based on the existing `resource_provider.py:fetch_aggregates` method + # found in the OpenStack SDK + from openstack import exceptions, utils + url = utils.urljoin(ResourceProvider.base_path, resource_provider_obj.id, "usages") + + response = conn.session.get(url, endpoint_filter={"service_type": "placement"}) + exceptions.raise_from_response(response) + return response.json()["usages"] + + + # pylint: disable=unused-argument + def run_query( + self, + conn: OpenstackConnection, + filter_kwargs: Optional[ServerSideFilter] = None, + **kwargs, + ) -> List[OpenstackResourceObj]: + """ + This method runs the query by running openstacksdk commands + + For HypervisorQuery, this command finds all hypervisors that match a given set of filter_kwargs + :param conn: An OpenstackConnection object - used to connect to openstacksdk + :param filter_kwargs: An Optional list of filter kwargs to pass to conn.compute.hypervisors() + to limit the hypervisors being returned. + - see https://docs.openstack.org/api-ref/compute/?expanded=list-hypervisors-detail + """ + logger.debug( + "running openstacksdk command conn.placement.resource_providers(%s)", + ",".join(f"{key}={value}" for key, value in filter_kwargs.items()), + ) + resource_providers = conn.placement.resource_providers(**filter_kwargs) + return [self._convert_to_custom_obj(conn, provider) for provider in resource_providers] diff --git a/openstackquery/structs/resource_provider_usage.py b/openstackquery/structs/resource_provider_usage.py new file mode 100644 index 0000000..fcaf6ca --- /dev/null +++ b/openstackquery/structs/resource_provider_usage.py @@ -0,0 +1,23 @@ +from dataclasses import dataclass + +from openstack.placement.v1.resource_provider import ResourceProvider + + +@dataclass +class ResourceProviderUsage: + """ + Upstream has a resource provider class which only provides available information + usage is not supported at all. Instead, create a custom class to store usage information + until upstream has a dedicated class for usage + """ + # Lower case to maintain compatibility with existing ResourceProvider object + name: str + id: str + + VCPU_AVAIL: int + MEMORY_MB_AVAIL: int + DISK_GB_AVAIL: int + + VCPU_USED: int + MEMORY_MB_USED: int + DISK_GB_USED: int