diff --git a/openstackquery/__init__.py b/openstackquery/__init__.py index 55e7261..73c0f5f 100644 --- a/openstackquery/__init__.py +++ b/openstackquery/__init__.py @@ -7,6 +7,7 @@ ProjectQuery, ImageQuery, HypervisorQuery, + PlacementQuery, ) # Create logger diff --git a/openstackquery/api/query_objects.py b/openstackquery/api/query_objects.py index d522b9c..0db9460 100644 --- a/openstackquery/api/query_objects.py +++ b/openstackquery/api/query_objects.py @@ -5,6 +5,7 @@ from openstackquery.mappings.hypervisor_mapping import HypervisorMapping from openstackquery.mappings.image_mapping import ImageMapping from openstackquery.mappings.mapping_interface import MappingInterface +from openstackquery.mappings.placement_mapping import PlacementMapping from openstackquery.mappings.project_mapping import ProjectMapping from openstackquery.mappings.server_mapping import ServerMapping from openstackquery.mappings.user_mapping import UserMapping @@ -70,3 +71,10 @@ def HypervisorQuery() -> "QueryAPI": Simple helper function to setup a query using a factory """ return get_common(HypervisorMapping) + + +def PlacementQuery() -> "QueryAPI": + """ + Simple helper function to setup a query using a factory + """ + return get_common(PlacementMapping) diff --git a/openstackquery/enums/props/placement_properties.py b/openstackquery/enums/props/placement_properties.py new file mode 100644 index 0000000..91bacda --- /dev/null +++ b/openstackquery/enums/props/placement_properties.py @@ -0,0 +1,68 @@ +from enum import auto +from typing import Dict, Optional + +from openstackquery.enums.props.prop_enum import PropEnum, PropFunc +from openstackquery.exceptions.query_property_mapping_error import ( + QueryPropertyMappingError, +) + + +class PlacementProperties(PropEnum): + """ + An enum class for currently used placement properties + """ + RESOURCE_PROVIDER_ID = auto() + RESOURCE_PROVIDER_NAME = auto() + VCPUS_USED = auto() + VCPUS_AVAIL = auto() + MEMORY_MB_USED = auto() + MEMORY_MB_AVAIL = auto() + DISK_GB_USED = auto() + DISK_GB_AVAIL = auto() + + @staticmethod + def _get_aliases() -> Dict: + """ + A method that returns all valid string alias mappings + """ + return { + PlacementProperties.RESOURCE_PROVIDER_ID: ["resource_provider_id", "resource_provider_uuid", "id"], + PlacementProperties.RESOURCE_PROVIDER_NAME: ["resource_name", "name", "provider_name"], + PlacementProperties.VCPUS_USED: ["vcpus_used"], + PlacementProperties.VCPUS_AVAIL: ["vcpus_avail"], + PlacementProperties.MEMORY_MB_USED: ["memory_mb_used"], + PlacementProperties.MEMORY_MB_AVAIL: ["memory_mb_avail"], + PlacementProperties.DISK_GB_USED: ["disk_gb_used"], + PlacementProperties.DISK_GB_AVAIL: ["disk_gb_avail"], + } + + @staticmethod + def get_prop_mapping(prop) -> Optional[PropFunc]: + """ + Method that returns the property function if function mapping exists for a given Hypervisor Enum + how to get specified property from a ResourceProviderUsage object + :param prop: A HypervisorProperty Enum for which a function may exist for + """ + mapping = { + PlacementProperties.RESOURCE_PROVIDER_ID: lambda a: a["id"], + PlacementProperties.RESOURCE_PROVIDER_NAME: lambda a: a["name"], + PlacementProperties.VCPUS_AVAIL: lambda a: a["VCPU_AVAIL"], + PlacementProperties.MEMORY_MB_AVAIL: lambda a: a["MEMORY_MB_AVAIL"], + PlacementProperties.DISK_GB_AVAIL: lambda a: a["DISK_GB_AVAIL"], + PlacementProperties.VCPUS_USED: lambda a: a["VCPU_USED"], + PlacementProperties.MEMORY_MB_USED: lambda a: a["MEMORY_MB_USED"], + PlacementProperties.DISK_GB_USED: lambda a: a["DISK_GB_USED"], + } + try: + return mapping[prop] + except KeyError as exp: + raise QueryPropertyMappingError( + f"Error: failed to get property mapping, property {prop.name} is not supported in PlacementProperties" + ) from exp + + @staticmethod + def get_marker_prop_func(): + """ + A getter method to return marker property function for pagination + """ + return PlacementProperties.get_prop_mapping(PlacementProperties.RESOURCE_PROVIDER_ID) diff --git a/openstackquery/handlers/server_side_handler.py b/openstackquery/handlers/server_side_handler.py index 50e3fc6..391744f 100644 --- a/openstackquery/handlers/server_side_handler.py +++ b/openstackquery/handlers/server_side_handler.py @@ -87,6 +87,8 @@ def get_filters( try: filters = filter_func(**params) except (KeyError, TypeError) as err: + # Dev note: your lambda must take "value" as the lambda + # argument if you arrive here adding new mappings raise QueryPresetMappingError( "Preset Argument Error: failed to build server-side openstacksdk filters for preset:prop: " f"'{preset.name}':'{prop.name}' " diff --git a/openstackquery/mappings/placement_mapping.py b/openstackquery/mappings/placement_mapping.py new file mode 100644 index 0000000..1a25f53 --- /dev/null +++ b/openstackquery/mappings/placement_mapping.py @@ -0,0 +1,115 @@ +from typing import Type + +from openstackquery.enums.props.hypervisor_properties import HypervisorProperties +from openstackquery.enums.props.placement_properties import PlacementProperties +from openstackquery.enums.props.prop_enum import PropEnum +from openstackquery.enums.query_presets import ( + QueryPresetsGeneric, + QueryPresetsString, + QueryPresetsInteger, +) +from openstackquery.handlers.client_side_handler_generic import ( + ClientSideHandlerGeneric, +) +from openstackquery.handlers.client_side_handler_integer import ( + ClientSideHandlerInteger, +) +from openstackquery.handlers.client_side_handler_string import ClientSideHandlerString +from openstackquery.handlers.server_side_handler import ServerSideHandler +from openstackquery.mappings.mapping_interface import MappingInterface +from openstackquery.runners.placement_runner import PlacementRunner + +from openstackquery.runners.runner_wrapper import RunnerWrapper +from openstackquery.structs.query_client_side_handlers import QueryClientSideHandlers + + +class PlacementMapping(MappingInterface): + """ + Mapping class for querying Openstack placement and resource objects + Define property mappings, kwarg mappings and filter function mappings, + and runner mapping related to placement and resources here + """ + + @staticmethod + def get_chain_mappings(): + """ + Should return a dictionary containing property pairs mapped to query mappings. + This is used to define how to chain results from this query to other possible queries + """ + return {PlacementProperties.RESOURCE_PROVIDER_NAME: HypervisorProperties.HYPERVISOR_NAME} + + @staticmethod + def get_runner_mapping() -> Type[RunnerWrapper]: + """ + Returns a mapping to associated Runner class for the Query (placement and resourceRunner) + """ + return PlacementRunner + + @staticmethod + def get_prop_mapping() -> Type[PropEnum]: + """ + Returns a mapping of valid presets for server side attributes (placement and resourceProperties) + """ + return PlacementProperties + + @staticmethod + def get_server_side_handler() -> ServerSideHandler: + """ + method to configure a server handler which can be used to get 'filter' keyword arguments that + can be passed to openstack function conn.placement.resource_providers() to filter results for a valid preset-property + pair + + valid filters documented here: + https://docs.openstack.org/openstacksdk/latest/user/proxies/placement.html + """ + return ServerSideHandler({ + QueryPresetsGeneric.EQUAL_TO: { + PlacementProperties.RESOURCE_PROVIDER_ID: lambda value: {"id": value}, + PlacementProperties.RESOURCE_PROVIDER_NAME: lambda value: {"name": value}, + } + }) + + @staticmethod + def get_client_side_handlers() -> QueryClientSideHandlers: + """ + method to configure a set of client-side handlers which can be used to get local filter functions + corresponding to valid preset-property pairs. These filter functions can be used to filter results after + listing all placement and resources. + """ + integer_prop_list = [ + PlacementProperties.VCPUS_AVAIL, + PlacementProperties.MEMORY_MB_AVAIL, + PlacementProperties.DISK_GB_AVAIL, + PlacementProperties.VCPUS_USED, + PlacementProperties.MEMORY_MB_USED, + PlacementProperties.DISK_GB_USED, + ] + + return QueryClientSideHandlers( + generic_handler = ClientSideHandlerGeneric( + { + QueryPresetsGeneric.EQUAL_TO: ["*"], + QueryPresetsGeneric.NOT_EQUAL_TO: ["*"], + QueryPresetsGeneric.ANY_IN: ["*"], + QueryPresetsGeneric.NOT_ANY_IN: ["*"], + } + ), + # set string query preset mappings + string_handler = ClientSideHandlerString( + { + QueryPresetsString.MATCHES_REGEX: [ + PlacementProperties.RESOURCE_PROVIDER_ID, + PlacementProperties.RESOURCE_PROVIDER_NAME, + ] + } + ), + datetime_handler = None, + integer_handler=ClientSideHandlerInteger( + { + QueryPresetsInteger.LESS_THAN: integer_prop_list, + QueryPresetsInteger.LESS_THAN_OR_EQUAL_TO: integer_prop_list, + QueryPresetsInteger.GREATER_THAN: integer_prop_list, + QueryPresetsInteger.GREATER_THAN_OR_EQUAL_TO: integer_prop_list, + } + ), + ) diff --git a/openstackquery/runners/placement_runner.py b/openstackquery/runners/placement_runner.py new file mode 100644 index 0000000..d3844f8 --- /dev/null +++ b/openstackquery/runners/placement_runner.py @@ -0,0 +1,112 @@ +import logging +from typing import List, Optional, Dict + +from openstack.placement.v1.resource_provider import ResourceProvider + +from openstackquery.aliases import OpenstackResourceObj, ServerSideFilters, ServerSideFilter +from openstackquery.openstack_connection import OpenstackConnection +from openstackquery.runners.runner_utils import RunnerUtils +from openstackquery.runners.runner_wrapper import RunnerWrapper +from openstackquery.structs.resource_provider_usage import ResourceProviderUsage + +logger = logging.getLogger(__name__) + + +class PlacementRunner(RunnerWrapper): + """ + Runner class for openstack Hypervisor resource + HypervisorRunner encapsulates running any openstacksdk Hypervisor commands + """ + + RESOURCE_TYPE = ResourceProvider + + def parse_meta_params(self, conn: OpenstackConnection, **kwargs): + """ + This class has no meta-params available, so this method is a no-op + """ + return super().parse_meta_params(conn, **kwargs) + + def _convert_to_custom_obj(self, conn: OpenstackConnection, obj: ResourceProvider) -> OpenstackResourceObj: + """ + Converts an openstacksdk ResourceProvider object to a ResourceProviderUsage object + including populating the available and used resources from the placement API + :param conn: Openstack connection + :param obj: Openstack placement resource provider object + :return: A ResourceProviderUsage object + """ + usage = self._get_usage_info(conn, obj) + avail = self._get_availability_info(conn, obj) + return ResourceProviderUsage( + id=obj.id, + name=obj.name, + VCPU_USED=usage["VCPU"], + MEMORY_MB_USED=usage["MEMORY_MB"], + DISK_GB_USED=usage["DISK_GB"], + VCPU_AVAIL=avail["VCPU"], + MEMORY_MB_AVAIL=avail["MEMORY_MB"], + DISK_GB_AVAIL=avail["DISK_GB"], + ) + + @staticmethod + def _get_availability_info(conn: OpenstackConnection, resource_provider_obj: ResourceProvider) -> Dict: + """ + Gets availability stats for a given placement resource provider + across the following resource classes: VCPU, MEMORY_MB, DISK_GB + :param conn: Openstack connection + :param resource_provider_obj: Openstack placement resource provider object + :return: A dictionary with the summed availability stats using the class name as a key + """ + summed_classes = {} + for resource_class in ["VCPU", "MEMORY_MB", "DISK_GB"]: + placement_inventories = conn.placement.resource_provider_inventories(resource_provider_obj, resource_class=resource_class) + # A resource provider can have n number of inventories for a given resource class + if not placement_inventories: + logger.warning("No available resources found for resource provider: %s", resource_provider_obj.id) + summed_classes[resource_class] = 0 + else: + summed_classes[resource_class] = sum(i["total"] for i in placement_inventories) + return summed_classes + + @staticmethod + def _get_usage_info( + conn: OpenstackConnection, resource_provider_obj: ResourceProvider + ) -> Dict: + """ + Gets usage stats for a given placement resource provider + :param conn: Openstack connection + :param resource_provider_obj: Openstack placement resource provider object + :return: A ResourceProviderUsage object with usage stats + """ + # The following should be up-streamed to openstacksdk at some point + # It is based on the existing `resource_provider.py:fetch_aggregates` method + # found in the OpenStack SDK + from openstack import exceptions, utils + url = utils.urljoin(ResourceProvider.base_path, resource_provider_obj.id, "usages") + + response = conn.session.get(url, endpoint_filter={"service_type": "placement"}) + exceptions.raise_from_response(response) + return response.json()["usages"] + + + # pylint: disable=unused-argument + def run_query( + self, + conn: OpenstackConnection, + filter_kwargs: Optional[ServerSideFilter] = None, + **kwargs, + ) -> List[OpenstackResourceObj]: + """ + This method runs the query by running openstacksdk commands + + For HypervisorQuery, this command finds all hypervisors that match a given set of filter_kwargs + :param conn: An OpenstackConnection object - used to connect to openstacksdk + :param filter_kwargs: An Optional list of filter kwargs to pass to conn.compute.hypervisors() + to limit the hypervisors being returned. + - see https://docs.openstack.org/api-ref/compute/?expanded=list-hypervisors-detail + """ + logger.debug( + "running openstacksdk command conn.placement.resource_providers(%s)", + ",".join(f"{key}={value}" for key, value in filter_kwargs.items()), + ) + resource_providers = conn.placement.resource_providers(**filter_kwargs) + return [self._convert_to_custom_obj(conn, provider) for provider in resource_providers] diff --git a/openstackquery/structs/resource_provider_usage.py b/openstackquery/structs/resource_provider_usage.py new file mode 100644 index 0000000..fcaf6ca --- /dev/null +++ b/openstackquery/structs/resource_provider_usage.py @@ -0,0 +1,23 @@ +from dataclasses import dataclass + +from openstack.placement.v1.resource_provider import ResourceProvider + + +@dataclass +class ResourceProviderUsage: + """ + Upstream has a resource provider class which only provides available information + usage is not supported at all. Instead, create a custom class to store usage information + until upstream has a dedicated class for usage + """ + # Lower case to maintain compatibility with existing ResourceProvider object + name: str + id: str + + VCPU_AVAIL: int + MEMORY_MB_AVAIL: int + DISK_GB_AVAIL: int + + VCPU_USED: int + MEMORY_MB_USED: int + DISK_GB_USED: int