From e4863aa115db8306f0479b89996ad0b57505f8bd Mon Sep 17 00:00:00 2001 From: Dean Hudek Date: Thu, 23 Mar 2023 14:05:20 +0100 Subject: [PATCH] topology-gocdb-connector refactored, args pulled from singleton-config file --- exec/topology-gocdb-connector.py | 63 ++----------- modules/config.py | 13 ++- modules/singleton_config.py | 149 +++++++++++++++++++++++++++++++ modules/tasks/gocdb_topology.py | 51 ++++++----- 4 files changed, 191 insertions(+), 85 deletions(-) create mode 100755 modules/singleton_config.py diff --git a/exec/topology-gocdb-connector.py b/exec/topology-gocdb-connector.py index 19409265..212e8813 100755 --- a/exec/topology-gocdb-connector.py +++ b/exec/topology-gocdb-connector.py @@ -7,7 +7,7 @@ import asyncio import uvloop -from argo_connectors.config import Global, CustomerConf +from argo_connectors.singleton_config import ConfigClass from argo_connectors.exceptions import ConnectorError, ConnectorParseError, ConnectorHttpError from argo_connectors.log import Logger from argo_connectors.tasks.common import write_state @@ -57,67 +57,16 @@ def main(): parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() - group_endpoints, group_groups = [], [] - logger = Logger(os.path.basename(sys.argv[0])) - - fixed_date = None - if args.date and date_check(args.date): - fixed_date = args.date - - confpath = args.gloconf[0] if args.gloconf else None - cglob = Global(sys.argv[0], confpath) - globopts = cglob.parse() - pass_extensions = eval(globopts['GeneralPassExtensions'.lower()]) - - confpath = args.custconf[0] if args.custconf else None - confcust = CustomerConf(sys.argv[0], confpath) - confcust.parse() - confcust.make_dirstruct() - confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) - topofeed = confcust.get_topofeed() - topofeedpaging = confcust.get_topofeedpaging() - uidservendp = confcust.get_uidserviceendpoints() - topofetchtype = confcust.get_topofetchtype() - custname = confcust.get_custname() - logger.customer = custname - - auth_custopts = confcust.get_authopts() - auth_opts = cglob.merge_opts(auth_custopts, 'authentication') - auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') - if not auth_complete: - logger.error('%s options incomplete, missing %s' % - ('authentication', ' '.join(missing))) - raise SystemExit(1) - - bdii_opts = get_bdii_opts(confcust) - webapi_opts = get_webapi_opts(cglob, confcust) - - toposcope = confcust.get_toposcope() - topofeedendpoints = confcust.get_topofeedendpoints() - topofeedservicegroups = confcust.get_topofeedservicegroups() - topofeedsites = confcust.get_topofeedsites() - notiflag = confcust.get_notif_flag() - - if toposcope: - SERVICE_ENDPOINTS_PI = topofeedendpoints + toposcope - SERVICE_GROUPS_PI = topofeedservicegroups + toposcope - SITES_PI = topofeedsites + toposcope - - else: - SERVICE_ENDPOINTS_PI = topofeedendpoints - SERVICE_GROUPS_PI = topofeedservicegroups - SITES_PI = topofeedsites loop = uvloop.new_event_loop() asyncio.set_event_loop(loop) + + config = ConfigClass(args) + fixed_date = config.get_fixed_date() try: - task = TaskGocdbTopology( - loop, logger, sys.argv[0], SERVICE_ENDPOINTS_PI, SERVICE_GROUPS_PI, - SITES_PI, globopts, auth_opts, webapi_opts, bdii_opts, confcust, - custname, topofeed, topofetchtype, fixed_date, uidservendp, - pass_extensions, topofeedpaging, notiflag - ) + task = TaskGocdbTopology(config, loop) + loop.run_until_complete(task.run()) except (ConnectorError, ConnectorParseError, ConnectorHttpError, KeyboardInterrupt) as exc: diff --git a/modules/config.py b/modules/config.py index 0ca4afb6..31c0707b 100644 --- a/modules/config.py +++ b/modules/config.py @@ -6,7 +6,16 @@ from .log import Logger -class Global(object): +class Singleton(type): + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super().__call__(*args, **kwargs) + return cls._instances[cls] + + +class Global(metaclass=Singleton): """ Class represents parser for global.conf """ @@ -195,7 +204,7 @@ def parse(self): return options -class CustomerConf(object): +class CustomerConf(metaclass=Singleton): """ Class with parser for customer.conf and additional helper methods """ diff --git a/modules/singleton_config.py b/modules/singleton_config.py new file mode 100755 index 00000000..88e44283 --- /dev/null +++ b/modules/singleton_config.py @@ -0,0 +1,149 @@ +import os +import sys + +import asyncio +import uvloop + +from argo_connectors.log import Logger +from argo_connectors.config import Global, CustomerConf +from argo_connectors.utils import date_check + + +logger = None +globopts = {} +custname = '' +isok = True + + +def get_webapi_opts(cglob, confcust): + webapi_custopts = confcust.get_webapiopts() + webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') + webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') + if not webapi_complete: + logger.error('Customer:%s %s options incomplete, missing %s' % + (logger.customer, 'webapi', ' '.join(missopt))) + raise SystemExit(1) + return webapi_opts + + +def get_bdii_opts(confcust): + bdii_custopts = confcust._get_cust_options('BDIIOpts') + if bdii_custopts: + bdii_complete, missing = confcust.is_complete_bdii(bdii_custopts) + if not bdii_complete: + logger.error('%s options incomplete, missing %s' % + ('bdii', ' '.join(missing))) + raise SystemExit(1) + return bdii_custopts + else: + return None + +class Singleton(type): + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super().__call__(*args, **kwargs) + return cls._instances[cls] + + + +class ConfigClass(metaclass=Singleton): + def __init__(self, args): + self.args = args + + def get_logger(self): + logger = Logger(os.path.basename(sys.argv[0])) + return logger + + def get_connector_name(self): + return sys.argv[0] + + def get_fixed_date(self): + fixed_date = None + if self.args.date and date_check(self.args.date): + fixed_date = self.args.date + return fixed_date + + def get_globopts_n_pass_ext(self): + confpath = self.args.gloconf[0] if self.args.gloconf else None + cglob = Global(sys.argv[0], confpath) + globopts = cglob.parse() + pass_extensions = eval(globopts['GeneralPassExtensions'.lower()]) + return globopts, pass_extensions + + def get_confcust(self, globopts): + confpath = self.args.custconf[0] if self.args.custconf else None + confcust = CustomerConf(sys.argv[0], confpath) + confcust.parse() + confcust.make_dirstruct() + confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) + return confcust + + def topofeed_data(self, confcust): + topofeed = confcust.get_topofeed() + return topofeed + + def topofeedpaging_data(self, confcust): + topofeedpaging = confcust.get_topofeedpaging() + return topofeedpaging + + def uidservendp_data(self, confcust): + uidservendp = confcust.get_uidserviceendpoints() + return uidservendp + + def topofetchtype_data(self, confcust): + topofetchtype = confcust.get_topofetchtype() + return topofetchtype + + def custname_data(self, confcust): + custname = confcust.get_custname() + return custname + + #logger.customer = custname #TODO: VIDITI DAL MI TREBA KASNIJE + + def get_auth_opts(self, confcust, logger): + confpath = self.args.gloconf[0] if self.args.gloconf else None + cglob = Global(sys.argv[0], confpath) + auth_custopts = confcust.get_authopts() + auth_opts = cglob.merge_opts(auth_custopts, 'authentication') + auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') + if not auth_complete: + logger.error('%s options incomplete, missing %s' % + ('authentication', ' '.join(missing))) + raise SystemExit(1) + return auth_opts + + def bdii_opts_data(self, confcust): + bdii_opts = get_bdii_opts(confcust) + return bdii_opts + + def get_webapi_opts_data(self, confcust): + confpath = self.args.gloconf[0] if self.args.gloconf else None + cglob = Global(sys.argv[0], confpath) + webapi_opts = get_webapi_opts(cglob, confcust) + return webapi_opts + + def notiflag_data(self, confcust): + notiflag = confcust.get_notif_flag() + return notiflag + + def service_data(self, confcust): + toposcope = confcust.get_toposcope() + topofeedendpoints = confcust.get_topofeedendpoints() + topofeedservicegroups = confcust.get_topofeedservicegroups() + topofeedsites = confcust.get_topofeedsites() + + if toposcope: + SERVICE_ENDPOINTS_PI = topofeedendpoints + toposcope + SERVICE_GROUPS_PI = topofeedservicegroups + toposcope + SITES_PI = topofeedsites + toposcope + + else: + SERVICE_ENDPOINTS_PI = topofeedendpoints + SERVICE_GROUPS_PI = topofeedservicegroups + SITES_PI = topofeedsites + + return SERVICE_ENDPOINTS_PI, SERVICE_GROUPS_PI, SITES_PI + + # return loop, logger, sys.argv[0], SERVICE_ENDPOINTS_PI, SERVICE_GROUPS_PI, SITES_PI, globopts, auth_opts, webapi_opts, bdii_opts, confcust, custname, topofeed, topofetchtype, fixed_date, uidservendp, pass_extensions, topofeedpaging, notiflag diff --git a/modules/tasks/gocdb_topology.py b/modules/tasks/gocdb_topology.py index c052e864..5cdb8248 100644 --- a/modules/tasks/gocdb_topology.py +++ b/modules/tasks/gocdb_topology.py @@ -171,33 +171,32 @@ def parse_serviceendpoints_contacts(self, res): class TaskGocdbTopology(TaskParseContacts, TaskParseTopology): - def __init__(self, loop, logger, connector_name, SERVICE_ENDPOINTS_PI, - SERVICE_GROUPS_PI, SITES_PI, globopts, auth_opts, webapi_opts, - bdii_opts, confcust, custname, topofeed, topofetchtype, - fixed_date, uidservendp, pass_extensions, topofeedpaging, - notiflag): - TaskParseTopology.__init__(self, logger, custname, uidservendp, - pass_extensions, notiflag) - super(TaskGocdbTopology, self).__init__(logger) + def __init__(self, config, loop): + self.config = config self.loop = loop - self.logger = logger - self.connector_name = connector_name - self.SERVICE_ENDPOINTS_PI = SERVICE_ENDPOINTS_PI - self.SERVICE_GROUPS_PI = SERVICE_GROUPS_PI - self.SITES_PI = SITES_PI - self.globopts = globopts - self.auth_opts = auth_opts - self.webapi_opts = webapi_opts - self.bdii_opts = bdii_opts - self.confcust = confcust - self.custname = custname - self.topofeed = topofeed - self.topofetchtype = topofetchtype - self.fixed_date = fixed_date - self.uidservendp = uidservendp - self.pass_extensions = pass_extensions - self.topofeedpaging = topofeedpaging - self.notification_flag = notiflag + + self.logger = self.config.get_logger() + self.connector_name = self.config.get_connector_name() + self.fixed_date = self.config.get_fixed_date() + self.globopts, self.pass_extensions = self.config.get_globopts_n_pass_ext() + self.confcust = self.config.get_confcust(self.globopts) + self.topofeed = self.config.topofeed_data(self.confcust) + self.topofeedpaging = self.config.topofeedpaging_data(self.confcust) + self.uidservendp = self.config.uidservendp_data(self.confcust) + self.topofetchtype = self.config.topofetchtype_data(self.confcust) + self.custname = self.config.custname_data(self.confcust) + self.auth_opts = self.config.get_auth_opts(self.confcust, self.logger) + self.bdii_opts = self.config.bdii_opts_data(self.confcust) + self.webapi_opts = self.config.get_webapi_opts_data(self.confcust) + self.notiflag = self.config.notiflag_data(self.confcust) + self.SERVICE_ENDPOINTS_PI, self.SERVICE_GROUPS_PI, self.SITES_PI = self.config.service_data(self.confcust) + + + TaskParseTopology.__init__(self, self.logger, self.custname, self.uidservendp, self.pass_extensions, + self.notiflag) + super(TaskGocdbTopology, self).__init__(self.logger) + + async def fetch_ldap_data(self, host, port, base, filter, attributes): ldap_session = LDAPSessionWithRetry(self.logger, int(self.globopts['ConnectionRetry'.lower()]),