From cef7b2956db0cd923682c664ecf665919c3b8665 Mon Sep 17 00:00:00 2001 From: Dean Hudek Date: Fri, 24 Mar 2023 09:22:09 +0100 Subject: [PATCH 01/11] quotation marks removed from hostnames --- modules/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/utils.py b/modules/utils.py index 2b179a4f..963c18e1 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -56,4 +56,7 @@ def remove_non_utf(string): if ' ' in string: string = string.replace(' ', '_') + if '"' in string: + string = string.replace('"', '') + return unidecode(string) \ No newline at end of file From edffbfac7eff682e1f919baa8029d678f3610ee4 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 23 Mar 2023 08:05:18 +0100 Subject: [PATCH 02/11] add depedency on python3-unidecode pkg --- argo-connectors.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/argo-connectors.spec b/argo-connectors.spec index 1c254589..da4c4114 100644 --- a/argo-connectors.spec +++ b/argo-connectors.spec @@ -20,6 +20,7 @@ Requires: python3-requests Requires: python3-typing-extensions Requires: python3-uvloop Requires: python3-bonsai +Requires: python3-unidecode Requires: python36-lxml BuildRequires: python3-devel python3-setuptools From e4863aa115db8306f0479b89996ad0b57505f8bd Mon Sep 17 00:00:00 2001 From: Dean Hudek Date: Thu, 23 Mar 2023 14:05:20 +0100 Subject: [PATCH 03/11] topology-gocdb-connector refactored, args pulled from singleton-config file --- exec/topology-gocdb-connector.py | 63 ++----------- modules/config.py | 13 ++- modules/singleton_config.py | 149 +++++++++++++++++++++++++++++++ modules/tasks/gocdb_topology.py | 51 ++++++----- 4 files changed, 191 insertions(+), 85 deletions(-) create mode 100755 modules/singleton_config.py diff --git a/exec/topology-gocdb-connector.py b/exec/topology-gocdb-connector.py index 19409265..212e8813 100755 --- a/exec/topology-gocdb-connector.py +++ b/exec/topology-gocdb-connector.py @@ -7,7 +7,7 @@ import asyncio import uvloop -from argo_connectors.config import Global, CustomerConf +from argo_connectors.singleton_config import ConfigClass from argo_connectors.exceptions import ConnectorError, ConnectorParseError, ConnectorHttpError from argo_connectors.log import Logger from argo_connectors.tasks.common import write_state @@ -57,67 +57,16 @@ def main(): parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() - group_endpoints, group_groups = [], [] - logger = Logger(os.path.basename(sys.argv[0])) - - fixed_date = None - if args.date and date_check(args.date): - fixed_date = args.date - - confpath = args.gloconf[0] if args.gloconf else None - cglob = Global(sys.argv[0], confpath) - globopts = cglob.parse() - pass_extensions = eval(globopts['GeneralPassExtensions'.lower()]) - - confpath = args.custconf[0] if args.custconf else None - confcust = CustomerConf(sys.argv[0], confpath) - confcust.parse() - confcust.make_dirstruct() - confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) - topofeed = confcust.get_topofeed() - topofeedpaging = confcust.get_topofeedpaging() - uidservendp = confcust.get_uidserviceendpoints() - topofetchtype = confcust.get_topofetchtype() - custname = confcust.get_custname() - logger.customer = custname - - auth_custopts = confcust.get_authopts() - auth_opts = cglob.merge_opts(auth_custopts, 'authentication') - auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') - if not auth_complete: - logger.error('%s options incomplete, missing %s' % - ('authentication', ' '.join(missing))) - raise SystemExit(1) - - bdii_opts = get_bdii_opts(confcust) - webapi_opts = get_webapi_opts(cglob, confcust) - - toposcope = confcust.get_toposcope() - topofeedendpoints = confcust.get_topofeedendpoints() - topofeedservicegroups = confcust.get_topofeedservicegroups() - topofeedsites = confcust.get_topofeedsites() - notiflag = confcust.get_notif_flag() - - if toposcope: - SERVICE_ENDPOINTS_PI = topofeedendpoints + toposcope - SERVICE_GROUPS_PI = topofeedservicegroups + toposcope - SITES_PI = topofeedsites + toposcope - - else: - SERVICE_ENDPOINTS_PI = topofeedendpoints - SERVICE_GROUPS_PI = topofeedservicegroups - SITES_PI = topofeedsites loop = uvloop.new_event_loop() asyncio.set_event_loop(loop) + + config = ConfigClass(args) + fixed_date = config.get_fixed_date() try: - task = TaskGocdbTopology( - loop, logger, sys.argv[0], SERVICE_ENDPOINTS_PI, SERVICE_GROUPS_PI, - SITES_PI, globopts, auth_opts, webapi_opts, bdii_opts, confcust, - custname, topofeed, topofetchtype, fixed_date, uidservendp, - pass_extensions, topofeedpaging, notiflag - ) + task = TaskGocdbTopology(config, loop) + loop.run_until_complete(task.run()) except (ConnectorError, ConnectorParseError, ConnectorHttpError, KeyboardInterrupt) as exc: diff --git a/modules/config.py b/modules/config.py index 0ca4afb6..31c0707b 100644 --- a/modules/config.py +++ b/modules/config.py @@ -6,7 +6,16 @@ from .log import Logger -class Global(object): +class Singleton(type): + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super().__call__(*args, **kwargs) + return cls._instances[cls] + + +class Global(metaclass=Singleton): """ Class represents parser for global.conf """ @@ -195,7 +204,7 @@ def parse(self): return options -class CustomerConf(object): +class CustomerConf(metaclass=Singleton): """ Class with parser for customer.conf and additional helper methods """ diff --git a/modules/singleton_config.py b/modules/singleton_config.py new file mode 100755 index 00000000..88e44283 --- /dev/null +++ b/modules/singleton_config.py @@ -0,0 +1,149 @@ +import os +import sys + +import asyncio +import uvloop + +from argo_connectors.log import Logger +from argo_connectors.config import Global, CustomerConf +from argo_connectors.utils import date_check + + +logger = None +globopts = {} +custname = '' +isok = True + + +def get_webapi_opts(cglob, confcust): + webapi_custopts = confcust.get_webapiopts() + webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') + webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') + if not webapi_complete: + logger.error('Customer:%s %s options incomplete, missing %s' % + (logger.customer, 'webapi', ' '.join(missopt))) + raise SystemExit(1) + return webapi_opts + + +def get_bdii_opts(confcust): + bdii_custopts = confcust._get_cust_options('BDIIOpts') + if bdii_custopts: + bdii_complete, missing = confcust.is_complete_bdii(bdii_custopts) + if not bdii_complete: + logger.error('%s options incomplete, missing %s' % + ('bdii', ' '.join(missing))) + raise SystemExit(1) + return bdii_custopts + else: + return None + +class Singleton(type): + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super().__call__(*args, **kwargs) + return cls._instances[cls] + + + +class ConfigClass(metaclass=Singleton): + def __init__(self, args): + self.args = args + + def get_logger(self): + logger = Logger(os.path.basename(sys.argv[0])) + return logger + + def get_connector_name(self): + return sys.argv[0] + + def get_fixed_date(self): + fixed_date = None + if self.args.date and date_check(self.args.date): + fixed_date = self.args.date + return fixed_date + + def get_globopts_n_pass_ext(self): + confpath = self.args.gloconf[0] if self.args.gloconf else None + cglob = Global(sys.argv[0], confpath) + globopts = cglob.parse() + pass_extensions = eval(globopts['GeneralPassExtensions'.lower()]) + return globopts, pass_extensions + + def get_confcust(self, globopts): + confpath = self.args.custconf[0] if self.args.custconf else None + confcust = CustomerConf(sys.argv[0], confpath) + confcust.parse() + confcust.make_dirstruct() + confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) + return confcust + + def topofeed_data(self, confcust): + topofeed = confcust.get_topofeed() + return topofeed + + def topofeedpaging_data(self, confcust): + topofeedpaging = confcust.get_topofeedpaging() + return topofeedpaging + + def uidservendp_data(self, confcust): + uidservendp = confcust.get_uidserviceendpoints() + return uidservendp + + def topofetchtype_data(self, confcust): + topofetchtype = confcust.get_topofetchtype() + return topofetchtype + + def custname_data(self, confcust): + custname = confcust.get_custname() + return custname + + #logger.customer = custname #TODO: VIDITI DAL MI TREBA KASNIJE + + def get_auth_opts(self, confcust, logger): + confpath = self.args.gloconf[0] if self.args.gloconf else None + cglob = Global(sys.argv[0], confpath) + auth_custopts = confcust.get_authopts() + auth_opts = cglob.merge_opts(auth_custopts, 'authentication') + auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') + if not auth_complete: + logger.error('%s options incomplete, missing %s' % + ('authentication', ' '.join(missing))) + raise SystemExit(1) + return auth_opts + + def bdii_opts_data(self, confcust): + bdii_opts = get_bdii_opts(confcust) + return bdii_opts + + def get_webapi_opts_data(self, confcust): + confpath = self.args.gloconf[0] if self.args.gloconf else None + cglob = Global(sys.argv[0], confpath) + webapi_opts = get_webapi_opts(cglob, confcust) + return webapi_opts + + def notiflag_data(self, confcust): + notiflag = confcust.get_notif_flag() + return notiflag + + def service_data(self, confcust): + toposcope = confcust.get_toposcope() + topofeedendpoints = confcust.get_topofeedendpoints() + topofeedservicegroups = confcust.get_topofeedservicegroups() + topofeedsites = confcust.get_topofeedsites() + + if toposcope: + SERVICE_ENDPOINTS_PI = topofeedendpoints + toposcope + SERVICE_GROUPS_PI = topofeedservicegroups + toposcope + SITES_PI = topofeedsites + toposcope + + else: + SERVICE_ENDPOINTS_PI = topofeedendpoints + SERVICE_GROUPS_PI = topofeedservicegroups + SITES_PI = topofeedsites + + return SERVICE_ENDPOINTS_PI, SERVICE_GROUPS_PI, SITES_PI + + # return loop, logger, sys.argv[0], SERVICE_ENDPOINTS_PI, SERVICE_GROUPS_PI, SITES_PI, globopts, auth_opts, webapi_opts, bdii_opts, confcust, custname, topofeed, topofetchtype, fixed_date, uidservendp, pass_extensions, topofeedpaging, notiflag diff --git a/modules/tasks/gocdb_topology.py b/modules/tasks/gocdb_topology.py index c052e864..5cdb8248 100644 --- a/modules/tasks/gocdb_topology.py +++ b/modules/tasks/gocdb_topology.py @@ -171,33 +171,32 @@ def parse_serviceendpoints_contacts(self, res): class TaskGocdbTopology(TaskParseContacts, TaskParseTopology): - def __init__(self, loop, logger, connector_name, SERVICE_ENDPOINTS_PI, - SERVICE_GROUPS_PI, SITES_PI, globopts, auth_opts, webapi_opts, - bdii_opts, confcust, custname, topofeed, topofetchtype, - fixed_date, uidservendp, pass_extensions, topofeedpaging, - notiflag): - TaskParseTopology.__init__(self, logger, custname, uidservendp, - pass_extensions, notiflag) - super(TaskGocdbTopology, self).__init__(logger) + def __init__(self, config, loop): + self.config = config self.loop = loop - self.logger = logger - self.connector_name = connector_name - self.SERVICE_ENDPOINTS_PI = SERVICE_ENDPOINTS_PI - self.SERVICE_GROUPS_PI = SERVICE_GROUPS_PI - self.SITES_PI = SITES_PI - self.globopts = globopts - self.auth_opts = auth_opts - self.webapi_opts = webapi_opts - self.bdii_opts = bdii_opts - self.confcust = confcust - self.custname = custname - self.topofeed = topofeed - self.topofetchtype = topofetchtype - self.fixed_date = fixed_date - self.uidservendp = uidservendp - self.pass_extensions = pass_extensions - self.topofeedpaging = topofeedpaging - self.notification_flag = notiflag + + self.logger = self.config.get_logger() + self.connector_name = self.config.get_connector_name() + self.fixed_date = self.config.get_fixed_date() + self.globopts, self.pass_extensions = self.config.get_globopts_n_pass_ext() + self.confcust = self.config.get_confcust(self.globopts) + self.topofeed = self.config.topofeed_data(self.confcust) + self.topofeedpaging = self.config.topofeedpaging_data(self.confcust) + self.uidservendp = self.config.uidservendp_data(self.confcust) + self.topofetchtype = self.config.topofetchtype_data(self.confcust) + self.custname = self.config.custname_data(self.confcust) + self.auth_opts = self.config.get_auth_opts(self.confcust, self.logger) + self.bdii_opts = self.config.bdii_opts_data(self.confcust) + self.webapi_opts = self.config.get_webapi_opts_data(self.confcust) + self.notiflag = self.config.notiflag_data(self.confcust) + self.SERVICE_ENDPOINTS_PI, self.SERVICE_GROUPS_PI, self.SITES_PI = self.config.service_data(self.confcust) + + + TaskParseTopology.__init__(self, self.logger, self.custname, self.uidservendp, self.pass_extensions, + self.notiflag) + super(TaskGocdbTopology, self).__init__(self.logger) + + async def fetch_ldap_data(self, host, port, base, filter, attributes): ldap_session = LDAPSessionWithRetry(self.logger, int(self.globopts['ConnectionRetry'.lower()]), From 401e3ed907618d37c8b5ffb294dadae701f30b5c Mon Sep 17 00:00:00 2001 From: Dean Hudek Date: Thu, 23 Mar 2023 15:41:00 +0100 Subject: [PATCH 04/11] weights-vapor-connector refactored --- exec/topology-gocdb-connector.py | 109 ++++++++++++++++++++++++++++++- exec/weights-vapor-connector.py | 53 ++++++++++----- modules/singleton_config.py | 10 ++- modules/tasks/gocdb_topology.py | 38 ++++++++++- modules/tasks/vapor_weights.py | 33 +++++++--- 5 files changed, 210 insertions(+), 33 deletions(-) diff --git a/exec/topology-gocdb-connector.py b/exec/topology-gocdb-connector.py index 212e8813..02ce4e32 100755 --- a/exec/topology-gocdb-connector.py +++ b/exec/topology-gocdb-connector.py @@ -8,6 +8,7 @@ import uvloop from argo_connectors.singleton_config import ConfigClass +from argo_connectors.config import Global, CustomerConf from argo_connectors.exceptions import ConnectorError, ConnectorParseError, ConnectorHttpError from argo_connectors.log import Logger from argo_connectors.tasks.common import write_state @@ -58,17 +59,95 @@ def main(): help='write data for this date', type=str, required=False) args = parser.parse_args() + # logger = Logger(os.path.basename(sys.argv[0])) + + # fixed_date = None + # if args.date and date_check(args.date): + # fixed_date = args.date + + # confpath = args.gloconf[0] if args.gloconf else None + # cglob = Global(sys.argv[0], confpath) + # globopts = cglob.parse() + # pass_extensions = eval(globopts['GeneralPassExtensions'.lower()]) + + # confpath = args.custconf[0] if args.custconf else None + # confcust = CustomerConf(sys.argv[0], confpath) + # confcust.parse() + # confcust.make_dirstruct() + # confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) + + # topofeed = confcust.get_topofeed() + # topofeedpaging = confcust.get_topofeedpaging() + # uidservendp = confcust.get_uidserviceendpoints() + # topofetchtype = confcust.get_topofetchtype() + # custname = confcust.get_custname() + # #logger.customer = custname #TODO: VIDIT DAL NAM TREBA + + # auth_custopts = confcust.get_authopts() + # auth_opts = cglob.merge_opts(auth_custopts, 'authentication') + + # auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') + # if not auth_complete: + # logger.error('%s options incomplete, missing %s' % + # ('authentication', ' '.join(missing))) + # raise SystemExit(1) + + # bdii_opts = get_bdii_opts(confcust) + # # print("bdii_opts: ", bdii_opts) # za EGI: {'bdii': 'True', 'bdiihost': 'bdii.egi.cro-ngi.hr', 'bdiiport': '2170', 'bdiiquerybase': 'o=grid', 'bdiiqueryfiltersrm': '(&(objectClass=GlueService)(|(GlueServiceType=srm_v1)(GlueServiceType=srm)))', 'bdiiqueryattributessrm': 'GlueServiceEndpoint', 'bdiiqueryfiltersepath': '(objectClass=GlueSATop)', 'bdiiqueryattributessepath': 'GlueVOInfoAccessControlBaseRule GlueVOInfoPath'} + + # webapi_opts = get_webapi_opts(cglob, confcust) + # # print("webapi_opts: ", webapi_opts) # za EGI: {'webapitoken': '505c3be00e9e30400b72dbfb0c06268aa73f694b', 'webapihost': 'api.devel.argo.grnet.gr'} + + # toposcope = confcust.get_toposcope() + # topofeedendpoints = confcust.get_topofeedendpoints() + # topofeedservicegroups = confcust.get_topofeedservicegroups() + # topofeedsites = confcust.get_topofeedsites() + # notiflag = confcust.get_notif_flag() + + # if toposcope: + # SERVICE_ENDPOINTS_PI = topofeedendpoints + toposcope + # SERVICE_GROUPS_PI = topofeedservicegroups + toposcope + # SITES_PI = topofeedsites + toposcope + + # else: + # SERVICE_ENDPOINTS_PI = topofeedendpoints + # SERVICE_GROUPS_PI = topofeedservicegroups + # SITES_PI = topofeedsites + + # loop = uvloop.new_event_loop() + # asyncio.set_event_loop(loop) + + + ############################################################################################### + loop = uvloop.new_event_loop() asyncio.set_event_loop(loop) - + config = ConfigClass(args) fixed_date = config.get_fixed_date() + ############################################################################################### + + try: - task = TaskGocdbTopology(config, loop) + # task = TaskGocdbTopology( + # loop, logger, sys.argv[0], SERVICE_ENDPOINTS_PI, SERVICE_GROUPS_PI, + # SITES_PI, globopts, auth_opts, webapi_opts, bdii_opts, confcust, + # custname, topofeed, topofetchtype, fixed_date, uidservendp, + # pass_extensions, topofeedpaging, notiflag) + + ############################################################################################### + + + task = TaskGocdbTopology(config, loop) #TODO: OVAKO TREBA IZGLEDATI + + + ############################################################################################### loop.run_until_complete(task.run()) + + except (ConnectorError, ConnectorParseError, ConnectorHttpError, KeyboardInterrupt) as exc: logger.error(repr(exc)) loop.run_until_complete( @@ -81,3 +160,29 @@ def main(): if __name__ == '__main__': main() + + + + + + + +# print("loop: ", loop) # +# print("logger: ", logger) # +# print("sys.argv[0]: ", sys.argv[0]) # /usr/libexec/argo-connectors/topology-gocdb-connector.py +# print("SERVICE_ENDPOINTS_PI: ", SERVICE_ENDPOINTS_PI) # https://goc-sdc.argo.grnet.gr//gocdbpi/private/?method=get_service_endpoint&scope= +# print("SERVICE_GROUPS_PI: ", SERVICE_GROUPS_PI) # https://goc-sdc.argo.grnet.gr//gocdbpi/private/?method=get_service_group&scope= +# print("SITES_PI: ", SITES_PI) # https://goc-sdc.argo.grnet.gr//gocdbpi/private/?method=get_site&scope= +# print("globopts: ", globopts) # {'generalwritejson': 'True', 'generalpublishwebapi': 'False', 'generalpassextensions': 'True', 'generalcompressjson': 'False', 'authenticationhostkey': '/etc/grid-security/hostkey.pem', 'authenticationhostcert': '/etc/grid-security/hostcert.pem', 'authenticationcapath': '/etc/grid-security/certificates', 'authenticationcafile': '/etc/pki/tls/certs/ca-bundle.crt', 'authenticationverifyservercert': 'True', 'authenticationuseplainhttpauth': 'False', 'authenticationhttpuser': 'xxxx', 'authenticationhttppass': 'xxxx', 'connectiontimeout': '180', 'connectionretry': '60', 'connectionsleepretry': '60', 'connectionretryrandom': 'True', 'connectionsleeprandomretrymax': '300', 'inputstatesavedir': '/var/lib/argo-connectors/states/', 'inputstatedays': '3', 'webapihost': 'api.devel.argo.grnet.gr', 'outputtopologygroupofendpoints': 'group_endpoints_DATE.json', 'outputtopologygroupofgroups': 'group_groups_DATE.json'} +# print("auth_opts: ", auth_opts) # {'authenticationhostkey': '/etc/grid-security/hostkey.pem', 'authenticationhostcert': '/etc/grid-security/hostcert.pem', 'authenticationcapath': '/etc/grid-security/certificates', 'authenticationcafile': '/etc/pki/tls/certs/ca-bundle.crt', 'authenticationverifyservercert': 'True', 'authenticationuseplainhttpauth': 'False', 'authenticationhttpuser': 'xxxx', 'authenticationhttppass': 'xxxx'} +# print("webapi_opts: ", webapi_opts) # {'webapitoken': '4473153af6c67a650a74d81d367e9e83f70e2b7b', 'webapihost': 'api.devel.argo.grnet.gr'} +# print("bdii_opts: ", bdii_opts) # None +# print("confcust: ", confcust) # +# print("custname: ", custname) # SDC +# print("topofeed: ", topofeed) # https://goc-sdc.argo.grnet.gr/ +# print("topofetchtype: ", topofetchtype) # ['sites', 'servicegroups'] +# print("fixed_date: ", fixed_date) # None +# print("uidservendp: ", uidservendp) # False +# print("pass_extensions: ", pass_extensions) # True +# print("topofeedpaging: ", topofeedpaging) # False +# print("notiflag: ", notiflag) # True \ No newline at end of file diff --git a/exec/weights-vapor-connector.py b/exec/weights-vapor-connector.py index b5aa728c..bb58dec9 100755 --- a/exec/weights-vapor-connector.py +++ b/exec/weights-vapor-connector.py @@ -7,12 +7,13 @@ import asyncio import uvloop +from argo_connectors.singleton_config import ConfigClass from argo_connectors.exceptions import ConnectorHttpError, ConnectorParseError from argo_connectors.tasks.vapor_weights import TaskVaporWeights from argo_connectors.tasks.common import write_weights_metricprofile_state as write_state from argo_connectors.log import Logger -from argo_connectors.config import Global, CustomerConf +#from argo_connectors.config import Global, CustomerConf from argo_connectors.utils import date_check globopts = {} @@ -20,7 +21,7 @@ def main(): - global logger, globopts + #global logger, globopts parser = argparse.ArgumentParser(description="""Fetch weights information from Gstat provider for every job listed in customer.conf""") parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', @@ -33,22 +34,36 @@ def main(): logger = Logger(os.path.basename(sys.argv[0])) - fixed_date = None - if args.date and date_check(args.date): - fixed_date = args.date + # fixed_date = None + # if args.date and date_check(args.date): + # fixed_date = args.date - confpath = args.gloconf[0] if args.gloconf else None - cglob = Global(sys.argv[0], confpath) - globopts = cglob.parse() + # confpath = args.gloconf[0] if args.gloconf else None + # cglob = Global(sys.argv[0], confpath) + # globopts = cglob.parse() - confpath = args.custconf[0] if args.custconf else None - confcust = CustomerConf(sys.argv[0], confpath) - confcust.parse() - confcust.make_dirstruct() - confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) + # confpath = args.custconf[0] if args.custconf else None + # confcust = CustomerConf(sys.argv[0], confpath) + # confcust.parse() + # confcust.make_dirstruct() + # confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) + + # VAPORPI = confcust.get_vaporpi() + # feeds = confcust.get_mapfeedjobs(sys.argv[0], deffeed=VAPORPI) + + ##################################################################### + + config = ConfigClass(args) + + fixed_date = config.get_fixed_date() + globopts, pass_extensions, cglob = config.get_globopts_n_pass_ext() + confcust = config.get_confcust(globopts) + VAPORPI = config.vaporrpi_data(confcust) + feeds = config.get_feeds(confcust, VAPORPI) + + + ##################################################################### - VAPORPI = confcust.get_vaporpi() - feeds = confcust.get_mapfeedjobs(sys.argv[0], deffeed=VAPORPI) loop = uvloop.new_event_loop() asyncio.set_event_loop(loop) @@ -64,9 +79,11 @@ def main(): logger.customer = customers try: - task = TaskVaporWeights(loop, logger, sys.argv[0], globopts, - confcust, VAPORPI, jobcust, cglob, - fixed_date) + # task = TaskVaporWeights(loop, logger, sys.argv[0], globopts, + # confcust, VAPORPI, jobcust, cglob, + # fixed_date) + + task = TaskVaporWeights(config, loop, jobcust) loop.run_until_complete(task.run()) except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc: diff --git a/modules/singleton_config.py b/modules/singleton_config.py index 88e44283..0fd9ae2a 100755 --- a/modules/singleton_config.py +++ b/modules/singleton_config.py @@ -70,7 +70,7 @@ def get_globopts_n_pass_ext(self): cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() pass_extensions = eval(globopts['GeneralPassExtensions'.lower()]) - return globopts, pass_extensions + return globopts, pass_extensions, cglob def get_confcust(self, globopts): confpath = self.args.custconf[0] if self.args.custconf else None @@ -146,4 +146,10 @@ def service_data(self, confcust): return SERVICE_ENDPOINTS_PI, SERVICE_GROUPS_PI, SITES_PI - # return loop, logger, sys.argv[0], SERVICE_ENDPOINTS_PI, SERVICE_GROUPS_PI, SITES_PI, globopts, auth_opts, webapi_opts, bdii_opts, confcust, custname, topofeed, topofetchtype, fixed_date, uidservendp, pass_extensions, topofeedpaging, notiflag + def vaporrpi_data(self, confcust): + VAPORPI = confcust.get_vaporpi() + return VAPORPI + + def get_feeds(self, confcust, VAPORPI): + feeds = confcust.get_mapfeedjobs(sys.argv[0], deffeed=VAPORPI) + return feeds diff --git a/modules/tasks/gocdb_topology.py b/modules/tasks/gocdb_topology.py index 5cdb8248..a98f5107 100644 --- a/modules/tasks/gocdb_topology.py +++ b/modules/tasks/gocdb_topology.py @@ -8,6 +8,7 @@ from concurrent.futures import ProcessPoolExecutor from functools import partial +from argo_connectors.singleton_config import ConfigClass from argo_connectors.parse.gocdb_topology import ParseServiceGroups, ParseServiceEndpoints, ParseSites from argo_connectors.parse.gocdb_contacts import ParseServiceEndpointContacts, ParseSitesWithContacts, ParseServiceGroupWithContacts from argo_connectors.exceptions import ConnectorError, ConnectorParseError, ConnectorHttpError @@ -93,7 +94,7 @@ def _parse(self): return count, cursor -class TaskParseTopology(object): +class TaskParseTopology(): def __init__(self, logger, custname, uidservendp, pass_extensions, notiflag): self.logger = logger @@ -156,21 +157,54 @@ def parse_servicegroups(logger, custname, uidservendp, pass_extensions, class TaskParseContacts(object): def __init__(self, logger): self.logger = logger + #print("self.logger: ", self.logger) def parse_siteswith_contacts(self, res): contacts = ParseSitesWithContacts(self.logger, res) + #print("contacts1: ", contacts) return contacts.get_contacts() def parse_servicegroups_contacts(self, res): contacts = ParseServiceGroupWithContacts(self.logger, res) + #print("contacts2: ", contacts) return contacts.get_contacts() def parse_serviceendpoints_contacts(self, res): contacts = ParseServiceEndpointContacts(self.logger, res) + return contacts.get_contacts() class TaskGocdbTopology(TaskParseContacts, TaskParseTopology): + # def __init__(self, loop, logger, connector_name, SERVICE_ENDPOINTS_PI, + # SERVICE_GROUPS_PI, SITES_PI, globopts, auth_opts, webapi_opts, + # bdii_opts, confcust, custname, topofeed, topofetchtype, + # fixed_date, uidservendp, pass_extensions, topofeedpaging, + # notiflag): + # TaskParseTopology.__init__(self, logger, custname, uidservendp, + # pass_extensions, notiflag) + # super(TaskGocdbTopology, self).__init__(logger) + # self.loop = loop + # self.logger = logger + # self.connector_name = connector_name + # self.SERVICE_ENDPOINTS_PI = SERVICE_ENDPOINTS_PI + # self.SERVICE_GROUPS_PI = SERVICE_GROUPS_PI + # self.SITES_PI = SITES_PI + # self.globopts = globopts + # self.auth_opts = auth_opts + # self.webapi_opts = webapi_opts + # self.bdii_opts = bdii_opts + # self.confcust = confcust + # self.custname = custname + # self.topofeed = topofeed + # self.topofetchtype = topofetchtype + # self.fixed_date = fixed_date + # self.uidservendp = uidservendp + # self.pass_extensions = pass_extensions + # self.topofeedpaging = topofeedpaging + # self.notification_flag = notiflag + + def __init__(self, config, loop): self.config = config self.loop = loop @@ -178,7 +212,7 @@ def __init__(self, config, loop): self.logger = self.config.get_logger() self.connector_name = self.config.get_connector_name() self.fixed_date = self.config.get_fixed_date() - self.globopts, self.pass_extensions = self.config.get_globopts_n_pass_ext() + self.globopts, self.pass_extensions, self.cglob = self.config.get_globopts_n_pass_ext() self.confcust = self.config.get_confcust(self.globopts) self.topofeed = self.config.topofeed_data(self.confcust) self.topofeedpaging = self.config.topofeedpaging_data(self.confcust) diff --git a/modules/tasks/vapor_weights.py b/modules/tasks/vapor_weights.py index c4f4986b..1f6380e8 100644 --- a/modules/tasks/vapor_weights.py +++ b/modules/tasks/vapor_weights.py @@ -9,17 +9,32 @@ class TaskVaporWeights(object): - def __init__(self, loop, logger, connector_name, globopts, confcust, feed, - jobcust, cglob, fixed_date): + # def __init__(self, loop, logger, connector_name, globopts, confcust, feed, + # jobcust, cglob, fixed_date): + # self.event_loop = loop + # self.logger = logger + # self.connector_name = connector_name + # self.globopts = globopts + # self.confcust = confcust + # self.feed = feed + # self.jobcust = jobcust + # self.cglob = cglob + # self.fixed_date = fixed_date + + ######################################################## + + def __init__(self, config, loop, jobcust): + self.config = config self.event_loop = loop - self.logger = logger - self.connector_name = connector_name - self.globopts = globopts - self.confcust = confcust - self.feed = feed self.jobcust = jobcust - self.cglob = cglob - self.fixed_date = fixed_date + + self.logger = config.get_logger() + self.connector_name = self.config.get_connector_name() + self.globopts, self.pass_extensions, self.cglob = self.config.get_globopts_n_pass_ext() + self.confcust = self.config.get_confcust(self.globopts) + self.feed = self.config.vaporrpi_data(self.confcust) + self.fixed_date = self.config.get_fixed_date() + async def fetch_data(self): feed_parts = urlparse(self.feed) From 8a7df5eeee313b5463ed4eb4c7f64eb77306f4c3 Mon Sep 17 00:00:00 2001 From: Dean Hudek Date: Mon, 27 Mar 2023 13:35:32 +0200 Subject: [PATCH 05/11] event loop added as method --- exec/topology-gocdb-connector.py | 16 +++++++++--- exec/weights-vapor-connector.py | 26 +++++++++++++----- modules/singleton_config.py | 45 +++++++++++++++++++++++++++++--- modules/tasks/gocdb_topology.py | 20 +++++++++++--- modules/tasks/vapor_weights.py | 39 ++++++++++++++++++--------- 5 files changed, 115 insertions(+), 31 deletions(-) diff --git a/exec/topology-gocdb-connector.py b/exec/topology-gocdb-connector.py index 02ce4e32..7fccc123 100755 --- a/exec/topology-gocdb-connector.py +++ b/exec/topology-gocdb-connector.py @@ -7,7 +7,7 @@ import asyncio import uvloop -from argo_connectors.singleton_config import ConfigClass +from argo_connectors.singleton_config import ConfigClass#, EventLoopSingleton from argo_connectors.config import Global, CustomerConf from argo_connectors.exceptions import ConnectorError, ConnectorParseError, ConnectorHttpError from argo_connectors.log import Logger @@ -120,10 +120,18 @@ def main(): ############################################################################################### - loop = uvloop.new_event_loop() - asyncio.set_event_loop(loop) + # loop = uvloop.new_event_loop() + # asyncio.set_event_loop(loop) + + # loop = EventLoopSingleton.get_event_loop() + # asyncio.set_event_loop(loop) config = ConfigClass(args) + print("config: ", config) + + loop = config.get_loop() + asyncio.set_event_loop(loop) + fixed_date = config.get_fixed_date() ############################################################################################### @@ -139,7 +147,7 @@ def main(): ############################################################################################### - task = TaskGocdbTopology(config, loop) #TODO: OVAKO TREBA IZGLEDATI + task = TaskGocdbTopology()#config)#, loop) #TODO: OVAKO TREBA IZGLEDATI ############################################################################################### diff --git a/exec/weights-vapor-connector.py b/exec/weights-vapor-connector.py index bb58dec9..231fc43a 100755 --- a/exec/weights-vapor-connector.py +++ b/exec/weights-vapor-connector.py @@ -7,13 +7,13 @@ import asyncio import uvloop -from argo_connectors.singleton_config import ConfigClass +from argo_connectors.singleton_config import ConfigClass#, EventLoopSingleton from argo_connectors.exceptions import ConnectorHttpError, ConnectorParseError from argo_connectors.tasks.vapor_weights import TaskVaporWeights from argo_connectors.tasks.common import write_weights_metricprofile_state as write_state from argo_connectors.log import Logger -#from argo_connectors.config import Global, CustomerConf +from argo_connectors.config import Global, CustomerConf from argo_connectors.utils import date_check globopts = {} @@ -32,7 +32,7 @@ def main(): help='write data for this date', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) + # logger = Logger(os.path.basename(sys.argv[0])) # fixed_date = None # if args.date and date_check(args.date): @@ -54,7 +54,12 @@ def main(): ##################################################################### config = ConfigClass(args) + print("config2: ", config) + loop = config.get_loop() + asyncio.set_event_loop(loop) + + logger = config.get_logger() fixed_date = config.get_fixed_date() globopts, pass_extensions, cglob = config.get_globopts_n_pass_ext() confcust = config.get_confcust(globopts) @@ -65,8 +70,12 @@ def main(): ##################################################################### - loop = uvloop.new_event_loop() - asyncio.set_event_loop(loop) + # loop = uvloop.new_event_loop() + # asyncio.set_event_loop(loop) + + # loop = EventLoopSingleton.get_event_loop() + # asyncio.set_event_loop(loop) + for feed, jobcust in feeds.items(): customers = set(map(lambda jc: confcust.get_custname(jc[1]), jobcust)) @@ -83,7 +92,12 @@ def main(): # confcust, VAPORPI, jobcust, cglob, # fixed_date) - task = TaskVaporWeights(config, loop, jobcust) + ############################################################### + + task = TaskVaporWeights(jobcust) + + ############################################################### + loop.run_until_complete(task.run()) except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc: diff --git a/modules/singleton_config.py b/modules/singleton_config.py index 0fd9ae2a..39fa16b1 100755 --- a/modules/singleton_config.py +++ b/modules/singleton_config.py @@ -38,6 +38,23 @@ def get_bdii_opts(confcust): else: return None + +# class Singleton(object): +# __instance = None + +# def __init__(self): +# if Singleton.__instance != None: +# raise Exception("This class is a singleton!") +# else: +# Singleton.__instance = self + +# @staticmethod +# def get_instance(): +# if Singleton.__instance == None: +# Singleton() +# return Singleton.__instance + + class Singleton(type): _instances = {} @@ -46,12 +63,32 @@ def __call__(cls, *args, **kwargs): cls._instances[cls] = super().__call__(*args, **kwargs) return cls._instances[cls] - + + +# class EventLoopSingleton(metaclass=Singleton): +# _loop = None + +# @classmethod +# def get_event_loop(cls): +# if cls._loop is None: +# cls._loop = uvloop.new_event_loop() +# return cls._loop + + + class ConfigClass(metaclass=Singleton): + _loop = None + def __init__(self, args): self.args = args + @classmethod + def get_loop(cls): + if cls._loop is None: + cls._loop = uvloop.new_event_loop() + return cls._loop + def get_logger(self): logger = Logger(os.path.basename(sys.argv[0])) return logger @@ -65,14 +102,14 @@ def get_fixed_date(self): fixed_date = self.args.date return fixed_date - def get_globopts_n_pass_ext(self): + def get_globopts_n_pass_ext(self): confpath = self.args.gloconf[0] if self.args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() pass_extensions = eval(globopts['GeneralPassExtensions'.lower()]) return globopts, pass_extensions, cglob - def get_confcust(self, globopts): + def get_confcust(self, globopts): confpath = self.args.custconf[0] if self.args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) confcust.parse() @@ -100,7 +137,7 @@ def custname_data(self, confcust): custname = confcust.get_custname() return custname - #logger.customer = custname #TODO: VIDITI DAL MI TREBA KASNIJE + # logger.customer = custname #TODO: VIDITI DAL MI TREBA KASNIJE def get_auth_opts(self, confcust, logger): confpath = self.args.gloconf[0] if self.args.gloconf else None diff --git a/modules/tasks/gocdb_topology.py b/modules/tasks/gocdb_topology.py index a98f5107..e44a701a 100644 --- a/modules/tasks/gocdb_topology.py +++ b/modules/tasks/gocdb_topology.py @@ -8,7 +8,7 @@ from concurrent.futures import ProcessPoolExecutor from functools import partial -from argo_connectors.singleton_config import ConfigClass +from argo_connectors.singleton_config import ConfigClass#, EventLoopSingleton from argo_connectors.parse.gocdb_topology import ParseServiceGroups, ParseServiceEndpoints, ParseSites from argo_connectors.parse.gocdb_contacts import ParseServiceEndpointContacts, ParseSitesWithContacts, ParseServiceGroupWithContacts from argo_connectors.exceptions import ConnectorError, ConnectorParseError, ConnectorHttpError @@ -205,9 +205,20 @@ class TaskGocdbTopology(TaskParseContacts, TaskParseTopology): # self.notification_flag = notiflag - def __init__(self, config, loop): - self.config = config - self.loop = loop + def __init__(self):#, config):#, loop): + #self.config = config + #self.loop = loop + + # self.loop = EventLoopSingleton.get_event_loop() + # asyncio.set_event_loop(self.loop) + + self.config = ConfigClass() + print("self.config: ", self.config) + + + self.loop = self.config.get_loop() + asyncio.set_event_loop(self.loop) + self.logger = self.config.get_logger() self.connector_name = self.config.get_connector_name() @@ -226,6 +237,7 @@ def __init__(self, config, loop): self.SERVICE_ENDPOINTS_PI, self.SERVICE_GROUPS_PI, self.SITES_PI = self.config.service_data(self.confcust) + TaskParseTopology.__init__(self, self.logger, self.custname, self.uidservendp, self.pass_extensions, self.notiflag) super(TaskGocdbTopology, self).__init__(self.logger) diff --git a/modules/tasks/vapor_weights.py b/modules/tasks/vapor_weights.py index 1f6380e8..a5f1a73d 100644 --- a/modules/tasks/vapor_weights.py +++ b/modules/tasks/vapor_weights.py @@ -1,7 +1,9 @@ import os from urllib.parse import urlparse +import asyncio +from argo_connectors.singleton_config import ConfigClass#, EventLoopSingleton from argo_connectors.io.http import SessionWithRetry from argo_connectors.io.webapi import WebAPI from argo_connectors.parse.vapor import ParseWeights @@ -11,24 +13,35 @@ class TaskVaporWeights(object): # def __init__(self, loop, logger, connector_name, globopts, confcust, feed, # jobcust, cglob, fixed_date): - # self.event_loop = loop - # self.logger = logger - # self.connector_name = connector_name - # self.globopts = globopts - # self.confcust = confcust - # self.feed = feed - # self.jobcust = jobcust - # self.cglob = cglob - # self.fixed_date = fixed_date + # self.event_loop = loop + # self.logger = logger + # self.connector_name = connector_name + # self.globopts = globopts + # self.confcust = confcust + # self.feed = feed + # self.jobcust = jobcust + # self.cglob = cglob + # self.fixed_date = fixed_date ######################################################## - def __init__(self, config, loop, jobcust): - self.config = config - self.event_loop = loop + def __init__(self, jobcust): + #self.config = config + + self.config = ConfigClass() + print("self.config2: ", self.config) + # self.event_loop = loop self.jobcust = jobcust - self.logger = config.get_logger() + # self.loop = EventLoopSingleton.get_event_loop() + # asyncio.set_event_loop(self.loop) + + + + self.loop = self.config.get_loop() + asyncio.set_event_loop(self.loop) + + self.logger = self.config.get_logger() self.connector_name = self.config.get_connector_name() self.globopts, self.pass_extensions, self.cglob = self.config.get_globopts_n_pass_ext() self.confcust = self.config.get_confcust(self.globopts) From ab265f3fe2f69ccff19814b440d019aa24352265 Mon Sep 17 00:00:00 2001 From: Dean Hudek Date: Mon, 27 Mar 2023 16:10:16 +0200 Subject: [PATCH 06/11] metric-profile, service-types n downtimes connectors refactored, use methods from configclass --- exec/downtimes-gocdb-connector.py | 85 ++++++++++++++++---------- exec/metricprofile-webapi-connector.py | 56 ++++++++++++----- exec/service-types-gocdb-connector.py | 83 ++++++++++++++++--------- exec/topology-gocdb-connector.py | 11 +--- exec/weights-vapor-connector.py | 12 +--- modules/singleton_config.py | 27 +++++--- modules/tasks/gocdb_downtimes.py | 51 ++++++++++++---- modules/tasks/gocdb_servicetypes.py | 46 ++++++++++---- modules/tasks/gocdb_topology.py | 17 +----- modules/tasks/vapor_weights.py | 14 +---- modules/tasks/webapi_metricprofile.py | 38 +++++++++--- 11 files changed, 273 insertions(+), 167 deletions(-) diff --git a/exec/downtimes-gocdb-connector.py b/exec/downtimes-gocdb-connector.py index f6f3dc41..da3664a3 100755 --- a/exec/downtimes-gocdb-connector.py +++ b/exec/downtimes-gocdb-connector.py @@ -8,6 +8,7 @@ import asyncio import uvloop +from argo_connectors.singleton_config import ConfigClass from argo_connectors.exceptions import ConnectorHttpError, ConnectorParseError from argo_connectors.log import Logger from argo_connectors.tasks.gocdb_downtimes import TaskGocdbDowntimes @@ -42,31 +43,31 @@ def main(): help='path to global configuration file', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) - confpath = args.gloconf[0] if args.gloconf else None - cglob = Global(sys.argv[0], confpath) - globopts = cglob.parse() - - confpath = args.custconf[0] if args.custconf else None - confcust = CustomerConf(sys.argv[0], confpath) - confcust.parse() - confcust.make_dirstruct() - confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) - - logger.customer = confcust.get_custname() - - auth_custopts = confcust.get_authopts() - auth_opts = cglob.merge_opts(auth_custopts, 'authentication') - auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') - if not auth_complete: - logger.error('%s options incomplete, missing %s' % - ('authentication', ' '.join(missing))) - raise SystemExit(1) + # logger = Logger(os.path.basename(sys.argv[0])) + # confpath = args.gloconf[0] if args.gloconf else None + # cglob = Global(sys.argv[0], confpath) + # globopts = cglob.parse() - if len(args.date) == 0: - raise SystemExit(1) + # confpath = args.custconf[0] if args.custconf else None + # confcust = CustomerConf(sys.argv[0], confpath) + # confcust.parse() + # confcust.make_dirstruct() + # confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) + + # logger.customer = confcust.get_custname() + + # auth_custopts = confcust.get_authopts() + # auth_opts = cglob.merge_opts(auth_custopts, 'authentication') + # auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') + # if not auth_complete: + # logger.error('%s options incomplete, missing %s' % + # ('authentication', ' '.join(missing))) + # raise SystemExit(1) - # calculate start and end times + # if len(args.date) == 0: + # raise SystemExit(1) + + #calculate start and end times try: start = datetime.datetime.strptime(args.date[0], '%Y-%m-%d') end = datetime.datetime.strptime(args.date[0], '%Y-%m-%d') @@ -78,20 +79,42 @@ def main(): logger.error(exc) raise SystemExit(1) - downtime_feed = confcust.get_downfeed() + # downtime_feed = confcust.get_downfeed() + + # uidservtype = confcust.get_uidserviceendpoints() + # webapi_opts = get_webapi_opts(cglob, confcust) + + # loop = uvloop.new_event_loop() + # asyncio.set_event_loop(loop) + - uidservtype = confcust.get_uidserviceendpoints() - webapi_opts = get_webapi_opts(cglob, confcust) + ########################################################################## - loop = uvloop.new_event_loop() + config = ConfigClass(args) + + globopts, _, _ = config.get_globopts_n_pass_ext() + confcust = config.get_confcust(globopts) + + loop = config.get_loop() asyncio.set_event_loop(loop) + ########################################################################## + try: cust = list(confcust.get_customers())[0] - task = TaskGocdbDowntimes(loop, logger, sys.argv[0], globopts, - auth_opts, webapi_opts, confcust, - confcust.get_custname(cust), downtime_feed, start, - end, uidservtype, args.date[0], timestamp) + cust = confcust.get_custname(cust) + + # task = TaskGocdbDowntimes(loop, logger, sys.argv[0], globopts, + # auth_opts, webapi_opts, confcust, + # confcust.get_custname(cust), downtime_feed, start, + # end, uidservtype, args.date[0], timestamp) + + ################################################################################### + + task = TaskGocdbDowntimes(cust, start, end, timestamp) + + ################################################################################### + loop.run_until_complete(task.run()) except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc: diff --git a/exec/metricprofile-webapi-connector.py b/exec/metricprofile-webapi-connector.py index 33f43542..421db4f8 100755 --- a/exec/metricprofile-webapi-connector.py +++ b/exec/metricprofile-webapi-connector.py @@ -7,6 +7,7 @@ import asyncio import uvloop +from argo_connectors.singleton_config import ConfigClass from argo_connectors.config import CustomerConf, Global from argo_connectors.log import Logger from argo_connectors.tasks.webapi_metricprofile import TaskWebApiMetricProfile @@ -26,30 +27,53 @@ def main(): parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) + # logger = Logger(os.path.basename(sys.argv[0])) - fixed_date = None - if args.date and date_check(args.date): - fixed_date = args.date + # fixed_date = None + # if args.date and date_check(args.date): + # fixed_date = args.date - confpath = args.gloconf[0] if args.gloconf else None - cglob = Global(sys.argv[0], confpath) - globopts = cglob.parse() + # confpath = args.gloconf[0] if args.gloconf else None + # cglob = Global(sys.argv[0], confpath) + # globopts = cglob.parse() - confpath = args.custconf[0] if args.custconf else None - confcust = CustomerConf(sys.argv[0], confpath) - confcust.parse() - confcust.make_dirstruct() - confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) + # confpath = args.custconf[0] if args.custconf else None + # confcust = CustomerConf(sys.argv[0], confpath) + # confcust.parse() + # confcust.make_dirstruct() + # confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) - loop = uvloop.new_event_loop() + # loop = uvloop.new_event_loop() + # asyncio.set_event_loop(loop) + + + ############################################################################ + + config = ConfigClass(args) + + globopts, _, _ = config.get_globopts_n_pass_ext() + confcust = config.get_confcust(globopts) + loop = config.get_loop() asyncio.set_event_loop(loop) + ############################################################################ + + + for cust in confcust.get_customers(): try: - task = TaskWebApiMetricProfile( - loop, logger, sys.argv[0], globopts, cglob, confcust, cust, fixed_date - ) + # task = TaskWebApiMetricProfile( + # loop, logger, sys.argv[0], globopts, cglob, confcust, cust, fixed_date + # ) + + ########################################################################### + + task = TaskWebApiMetricProfile(cust) + + ########################################################################### + + + loop.run_until_complete(task.run()) except (KeyboardInterrupt) as exc: diff --git a/exec/service-types-gocdb-connector.py b/exec/service-types-gocdb-connector.py index 2977afae..0f4d2b6f 100755 --- a/exec/service-types-gocdb-connector.py +++ b/exec/service-types-gocdb-connector.py @@ -8,6 +8,7 @@ import asyncio import uvloop +from argo_connectors.singleton_config import ConfigClass from argo_connectors.exceptions import ConnectorHttpError, ConnectorParseError from argo_connectors.log import Logger from argo_connectors.tasks.gocdb_servicetypes import TaskGocdbServiceTypes @@ -39,41 +40,63 @@ def main(): parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() - fixed_date = None - if args.date and date_check(args.date): - fixed_date = args.date - - logger = Logger(os.path.basename(sys.argv[0])) - confpath = args.gloconf[0] if args.gloconf else None - cglob = Global(sys.argv[0], confpath) - globopts = cglob.parse() - - confpath = args.custconf[0] if args.custconf else None - confcust = CustomerConf(sys.argv[0], confpath) - confcust.parse() - confcust.make_dirstruct() - confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) - feed = confcust.get_servicesfeed() - custname = confcust.get_custname() - logger.customer = confcust.get_custname() - - auth_custopts = confcust.get_authopts() - auth_opts = cglob.merge_opts(auth_custopts, 'authentication') - auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') - if not auth_complete: - logger.error('%s options incomplete, missing %s' % ('authentication', ' '.join(missing))) - raise SystemExit(1) + # fixed_date = None + # if args.date and date_check(args.date): + # fixed_date = args.date + + # logger = Logger(os.path.basename(sys.argv[0])) + # confpath = args.gloconf[0] if args.gloconf else None + # cglob = Global(sys.argv[0], confpath) + # globopts = cglob.parse() + + # confpath = args.custconf[0] if args.custconf else None + # confcust = CustomerConf(sys.argv[0], confpath) + # confcust.parse() + # confcust.make_dirstruct() + # confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) + # feed = confcust.get_servicesfeed() + # custname = confcust.get_custname() + # logger.customer = confcust.get_custname() + + # auth_custopts = confcust.get_authopts() + # auth_opts = cglob.merge_opts(auth_custopts, 'authentication') + # auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') + # if not auth_complete: + # logger.error('%s options incomplete, missing %s' % ('authentication', ' '.join(missing))) + # raise SystemExit(1) - webapi_opts = get_webapi_opts(cglob, confcust) + # webapi_opts = get_webapi_opts(cglob, confcust) - loop = uvloop.new_event_loop() + # loop = uvloop.new_event_loop() + # asyncio.set_event_loop(loop) + + ########################################################################## + + config = ConfigClass(args) + + loop = config.get_loop() asyncio.set_event_loop(loop) + fixed_date = config.get_fixed_date() + globopts, _, _ = config.get_globopts_n_pass_ext() + confcust = config.get_confcust(globopts) + + ########################################################################## + + + try: - task = TaskGocdbServiceTypes( - loop, logger, sys.argv[0], globopts, auth_opts, webapi_opts, - confcust, custname, feed, fixed_date, args.initsync - ) + # task = TaskGocdbServiceTypes( + # loop, logger, sys.argv[0], globopts, auth_opts, webapi_opts, + # confcust, custname, feed, fixed_date, args.initsync + # ) + + ##################################################################### + + task = TaskGocdbServiceTypes() + + ##################################################################### + loop.run_until_complete(task.run()) except (KeyboardInterrupt) as exc: diff --git a/exec/topology-gocdb-connector.py b/exec/topology-gocdb-connector.py index 7fccc123..070e66b1 100755 --- a/exec/topology-gocdb-connector.py +++ b/exec/topology-gocdb-connector.py @@ -7,7 +7,7 @@ import asyncio import uvloop -from argo_connectors.singleton_config import ConfigClass#, EventLoopSingleton +from argo_connectors.singleton_config import ConfigClass from argo_connectors.config import Global, CustomerConf from argo_connectors.exceptions import ConnectorError, ConnectorParseError, ConnectorHttpError from argo_connectors.log import Logger @@ -120,14 +120,7 @@ def main(): ############################################################################################### - # loop = uvloop.new_event_loop() - # asyncio.set_event_loop(loop) - - # loop = EventLoopSingleton.get_event_loop() - # asyncio.set_event_loop(loop) - config = ConfigClass(args) - print("config: ", config) loop = config.get_loop() asyncio.set_event_loop(loop) @@ -147,7 +140,7 @@ def main(): ############################################################################################### - task = TaskGocdbTopology()#config)#, loop) #TODO: OVAKO TREBA IZGLEDATI + task = TaskGocdbTopology() #TODO: OVAKO TREBA IZGLEDATI ############################################################################################### diff --git a/exec/weights-vapor-connector.py b/exec/weights-vapor-connector.py index 231fc43a..2d44ee40 100755 --- a/exec/weights-vapor-connector.py +++ b/exec/weights-vapor-connector.py @@ -54,14 +54,12 @@ def main(): ##################################################################### config = ConfigClass(args) - print("config2: ", config) - loop = config.get_loop() asyncio.set_event_loop(loop) logger = config.get_logger() fixed_date = config.get_fixed_date() - globopts, pass_extensions, cglob = config.get_globopts_n_pass_ext() + globopts, _, _ = config.get_globopts_n_pass_ext() confcust = config.get_confcust(globopts) VAPORPI = config.vaporrpi_data(confcust) feeds = config.get_feeds(confcust, VAPORPI) @@ -69,14 +67,6 @@ def main(): ##################################################################### - - # loop = uvloop.new_event_loop() - # asyncio.set_event_loop(loop) - - # loop = EventLoopSingleton.get_event_loop() - # asyncio.set_event_loop(loop) - - for feed, jobcust in feeds.items(): customers = set(map(lambda jc: confcust.get_custname(jc[1]), jobcust)) customers = customers.pop() if len( diff --git a/modules/singleton_config.py b/modules/singleton_config.py index 39fa16b1..c599657f 100755 --- a/modules/singleton_config.py +++ b/modules/singleton_config.py @@ -82,12 +82,14 @@ class ConfigClass(metaclass=Singleton): def __init__(self, args): self.args = args + self._setup_loop() - @classmethod - def get_loop(cls): - if cls._loop is None: - cls._loop = uvloop.new_event_loop() - return cls._loop + def _setup_loop(self): + if self._loop is None: + self._loop = uvloop.new_event_loop() + + def get_loop(self): + return self._loop def get_logger(self): logger = Logger(os.path.basename(sys.argv[0])) @@ -137,8 +139,6 @@ def custname_data(self, confcust): custname = confcust.get_custname() return custname - # logger.customer = custname #TODO: VIDITI DAL MI TREBA KASNIJE - def get_auth_opts(self, confcust, logger): confpath = self.args.gloconf[0] if self.args.gloconf else None cglob = Global(sys.argv[0], confpath) @@ -190,3 +190,16 @@ def vaporrpi_data(self, confcust): def get_feeds(self, confcust, VAPORPI): feeds = confcust.get_mapfeedjobs(sys.argv[0], deffeed=VAPORPI) return feeds + + def get_feed(self, confcust): + feed = confcust.get_servicesfeed() + return feed + + def get_initsync(self): + return self.args.initsync + + def get_downtime_feed(self, confcust): + return confcust.get_downfeed() + + def get_target_date(self): + return self.args.date[0] \ No newline at end of file diff --git a/modules/tasks/gocdb_downtimes.py b/modules/tasks/gocdb_downtimes.py index 273ad029..53fe061a 100644 --- a/modules/tasks/gocdb_downtimes.py +++ b/modules/tasks/gocdb_downtimes.py @@ -1,7 +1,9 @@ import os +import asyncio from urllib.parse import urlparse +from argo_connectors.singleton_config import ConfigClass from argo_connectors.io.http import SessionWithRetry from argo_connectors.parse.gocdb_downtimes import ParseDowntimes from argo_connectors.io.webapi import WebAPI @@ -9,24 +11,47 @@ class TaskGocdbDowntimes(object): - def __init__(self, loop, logger, connector_name, globopts, auth_opts, - webapi_opts, confcust, custname, feed, start, end, - uidservtype, targetdate, timestamp): - self.event_loop = loop - self.logger = logger - self.connector_name = connector_name - self.globopts = globopts - self.auth_opts = auth_opts - self.webapi_opts = webapi_opts - self.confcust = confcust + # def __init__(self, loop, logger, connector_name, globopts, auth_opts, + # webapi_opts, confcust, custname, feed, start, end, + # uidservtype, targetdate, timestamp): + # self.event_loop = loop + # self.logger = logger + # self.connector_name = connector_name + # self.globopts = globopts + # self.auth_opts = auth_opts + # self.webapi_opts = webapi_opts + # self.confcust = confcust + # self.custname = custname + # print("self.custname: ", self.custname) + # self.feed = feed + # self.start = start + # self.end = end + # self.uidservtype = uidservtype + # self.targetdate = targetdate + # self.timestamp = timestamp + + ################################################################################### + + def __init__(self, custname, start, end, timestamp): self.custname = custname - self.feed = feed self.start = start self.end = end - self.uidservtype = uidservtype - self.targetdate = targetdate self.timestamp = timestamp + self.config = ConfigClass() + self.loop = self.config.get_loop() + asyncio.set_event_loop(self.loop) + self.logger = self.config.get_logger() + self.connector_name = self.config.get_connector_name() + self.globopts, self.pass_extensions, self.cglob = self.config.get_globopts_n_pass_ext() + self.confcust = self.config.get_confcust(self.globopts) + self.auth_opts = self.config.get_auth_opts(self.confcust, self.logger) + self.webapi_opts = self.config.get_webapi_opts_data(self.confcust) + self.custname = self.config.custname_data(self.confcust) + self.feed = self.config.get_downtime_feed(self.confcust) + self.uidservtype = self.config.uidservendp_data(self.confcust) + self.targetdate = self.config.get_target_date() + async def fetch_data(self): feed_parts = urlparse(self.feed) start_fmt = self.start.strftime("%Y-%m-%d") diff --git a/modules/tasks/gocdb_servicetypes.py b/modules/tasks/gocdb_servicetypes.py index 6d7d8f86..63420fe3 100644 --- a/modules/tasks/gocdb_servicetypes.py +++ b/modules/tasks/gocdb_servicetypes.py @@ -3,6 +3,7 @@ from urllib.parse import urlparse +from argo_connectors.singleton_config import ConfigClass from argo_connectors.io.http import SessionWithRetry from argo_connectors.parse.gocdb_servicetypes import ParseGocdbServiceTypes from argo_connectors.parse.webapi_servicetypes import ParseWebApiServiceTypes @@ -20,19 +21,38 @@ def contains_exception(list): class TaskGocdbServiceTypes(object): - def __init__(self, loop, logger, connector_name, globopts, auth_opts, - webapi_opts, confcust, custname, feed, timestamp, initsync): - self.logger = logger - self.loop = loop - self.connector_name = connector_name - self.globopts = globopts - self.auth_opts = auth_opts - self.webapi_opts = webapi_opts - self.confcust = confcust - self.custname = custname - self.feed = feed - self.timestamp = timestamp - self.initsync = initsync + # def __init__(self, loop, logger, connector_name, globopts, auth_opts, + # webapi_opts, confcust, custname, feed, timestamp, initsync): + # self.logger = logger + # self.loop = loop + # self.connector_name = connector_name + # self.globopts = globopts + # self.auth_opts = auth_opts + # self.webapi_opts = webapi_opts + # self.confcust = confcust + # self.custname = custname + # self.feed = feed + # self.timestamp = timestamp + # self.initsync = initsync + + ############################################################################ + + def __init__(self): + self.config = ConfigClass() + self.loop = self.config.get_loop() + asyncio.set_event_loop(self.loop) + self.logger = self.config.get_logger() + self.connector_name = self.config.get_connector_name() + self.globopts, _, _ = self.config.get_globopts_n_pass_ext() + self.confcust = self.config.get_confcust(self.globopts) + self.auth_opts = self.config.get_auth_opts(self.confcust, self.logger) + self.webapi_opts = self.config.get_webapi_opts_data(self.confcust) + self.custname = self.config.custname_data(self.confcust) + self.feed = self.config.get_feed(self.confcust) + self.timestamp = self.config.get_fixed_date() + self.initsync = self.config.get_initsync() + + async def fetch_data(self): feed_parts = urlparse(self.feed) diff --git a/modules/tasks/gocdb_topology.py b/modules/tasks/gocdb_topology.py index e44a701a..63d32749 100644 --- a/modules/tasks/gocdb_topology.py +++ b/modules/tasks/gocdb_topology.py @@ -157,16 +157,13 @@ def parse_servicegroups(logger, custname, uidservendp, pass_extensions, class TaskParseContacts(object): def __init__(self, logger): self.logger = logger - #print("self.logger: ", self.logger) def parse_siteswith_contacts(self, res): contacts = ParseSitesWithContacts(self.logger, res) - #print("contacts1: ", contacts) return contacts.get_contacts() def parse_servicegroups_contacts(self, res): contacts = ParseServiceGroupWithContacts(self.logger, res) - #print("contacts2: ", contacts) return contacts.get_contacts() def parse_serviceendpoints_contacts(self, res): @@ -205,21 +202,10 @@ class TaskGocdbTopology(TaskParseContacts, TaskParseTopology): # self.notification_flag = notiflag - def __init__(self):#, config):#, loop): - #self.config = config - #self.loop = loop - - # self.loop = EventLoopSingleton.get_event_loop() - # asyncio.set_event_loop(self.loop) - + def __init__(self): self.config = ConfigClass() - print("self.config: ", self.config) - - self.loop = self.config.get_loop() asyncio.set_event_loop(self.loop) - - self.logger = self.config.get_logger() self.connector_name = self.config.get_connector_name() self.fixed_date = self.config.get_fixed_date() @@ -237,7 +223,6 @@ def __init__(self):#, config):#, loop): self.SERVICE_ENDPOINTS_PI, self.SERVICE_GROUPS_PI, self.SITES_PI = self.config.service_data(self.confcust) - TaskParseTopology.__init__(self, self.logger, self.custname, self.uidservendp, self.pass_extensions, self.notiflag) super(TaskGocdbTopology, self).__init__(self.logger) diff --git a/modules/tasks/vapor_weights.py b/modules/tasks/vapor_weights.py index a5f1a73d..7bea877a 100644 --- a/modules/tasks/vapor_weights.py +++ b/modules/tasks/vapor_weights.py @@ -26,21 +26,11 @@ class TaskVaporWeights(object): ######################################################## def __init__(self, jobcust): - #self.config = config - - self.config = ConfigClass() - print("self.config2: ", self.config) - # self.event_loop = loop self.jobcust = jobcust - - # self.loop = EventLoopSingleton.get_event_loop() - # asyncio.set_event_loop(self.loop) - - - + + self.config = ConfigClass() self.loop = self.config.get_loop() asyncio.set_event_loop(self.loop) - self.logger = self.config.get_logger() self.connector_name = self.config.get_connector_name() self.globopts, self.pass_extensions, self.cglob = self.config.get_globopts_n_pass_ext() diff --git a/modules/tasks/webapi_metricprofile.py b/modules/tasks/webapi_metricprofile.py index d90da9d4..9fbc62f2 100644 --- a/modules/tasks/webapi_metricprofile.py +++ b/modules/tasks/webapi_metricprofile.py @@ -1,5 +1,7 @@ import os +import asyncio +from argo_connectors.singleton_config import ConfigClass from argo_connectors.exceptions import ConnectorHttpError, ConnectorParseError from argo_connectors.io.http import SessionWithRetry from argo_connectors.tasks.common import write_weights_metricprofile_state as write_state, write_metricprofile_json as write_json @@ -9,16 +11,34 @@ class TaskWebApiMetricProfile(object): - def __init__(self, loop, logger, connector_name, globopts, cglob, confcust, - cust, fixed_date): - self.loop = loop - self.logger = logger - self.connector_name = connector_name - self.globopts = globopts + # def __init__(self, loop, logger, connector_name, globopts, cglob, confcust, + # cust, fixed_date): + # self.loop = loop + # self.logger = logger + # self.connector_name = connector_name + # self.globopts = globopts + # self.cust = cust + # self.confcust = confcust + # self.cglob = cglob + # self.fixed_date = fixed_date + + #################################################################################### + + def __init__(self, cust): self.cust = cust - self.confcust = confcust - self.cglob = cglob - self.fixed_date = fixed_date + + self.config = ConfigClass() + self.loop = self.config.get_loop() + asyncio.set_event_loop(self.loop) + self.logger = self.config.get_logger() + self.connector_name = self.config.get_connector_name() + self.globopts, _, self.cglob = self.config.get_globopts_n_pass_ext() + self.confcust = self.config.get_confcust(self.globopts) + self.fixed_date = self.config.get_fixed_date() + + + #################################################################################### + async def fetch_data(self, host, token): session = SessionWithRetry(self.logger, From 912423ad40f0599ad27734d4e7b2e88a706a0101 Mon Sep 17 00:00:00 2001 From: Dean Hudek Date: Tue, 28 Mar 2023 11:43:16 +0200 Subject: [PATCH 07/11] topology-csv, service-types-csv, downtimes-csv refactored --- exec/downtimes-csv-connector.py | 90 ++++++++++++++-------- exec/service-types-csv-connector.py | 89 ++++++++++++++-------- exec/topology-csv-connector.py | 113 +++++++++++++++++----------- exec/topology-gocdb-connector.py | 44 +++++------ modules/singleton_config.py | 44 +++-------- modules/tasks/flat_downtimes.py | 48 ++++++++---- modules/tasks/flat_servicetypes.py | 49 ++++++++---- modules/tasks/flat_topology.py | 52 +++++++++---- modules/tasks/gocdb_downtimes.py | 2 +- modules/tasks/gocdb_servicetypes.py | 2 +- modules/tasks/gocdb_topology.py | 2 +- 11 files changed, 326 insertions(+), 209 deletions(-) diff --git a/exec/downtimes-csv-connector.py b/exec/downtimes-csv-connector.py index 154e9480..551d5b86 100755 --- a/exec/downtimes-csv-connector.py +++ b/exec/downtimes-csv-connector.py @@ -8,6 +8,7 @@ import asyncio import uvloop +from argo_connectors.singleton_config import ConfigClass from argo_connectors.exceptions import ConnectorHttpError, ConnectorParseError from argo_connectors.log import Logger from argo_connectors.tasks.flat_downtimes import TaskCsvDowntimes @@ -19,14 +20,14 @@ globopts = {} -def get_webapi_opts(cglob, confcust): - webapi_custopts = confcust.get_webapiopts() - webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') - webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') - if not webapi_complete: - logger.error('Customer:%s %s options incomplete, missing %s' % (logger.customer, 'webapi', ' '.join(missopt))) - raise SystemExit(1) - return webapi_opts +# def get_webapi_opts(cglob, confcust): +# webapi_custopts = confcust.get_webapiopts() +# webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') +# webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') +# if not webapi_complete: +# logger.error('Customer:%s %s options incomplete, missing %s' % (logger.customer, 'webapi', ' '.join(missopt))) +# raise SystemExit(1) +# return webapi_opts def main(): @@ -37,22 +38,22 @@ def main(): parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) - confpath = args.gloconf[0] if args.gloconf else None - cglob = Global(sys.argv[0], confpath) - globopts = cglob.parse() - - confpath = args.custconf[0] if args.custconf else None - confcust = CustomerConf(sys.argv[0], confpath) - confcust.parse() - confcust.make_dirstruct() - confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) - feed = confcust.get_downfeed() - logger.customer = confcust.get_custname() - - if len(args.date) == 0: - print(parser.print_help()) - raise SystemExit(1) + # logger = Logger(os.path.basename(sys.argv[0])) + # confpath = args.gloconf[0] if args.gloconf else None + # cglob = Global(sys.argv[0], confpath) + # globopts = cglob.parse() + + # confpath = args.custconf[0] if args.custconf else None + # confcust = CustomerConf(sys.argv[0], confpath) + # confcust.parse() + # confcust.make_dirstruct() + # confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) + # feed = confcust.get_downfeed() + # logger.customer = confcust.get_custname() + + # if len(args.date) == 0: + # print(parser.print_help()) + # raise SystemExit(1) # calculate start and end times try: @@ -64,21 +65,46 @@ def main(): logger.error(exc) raise SystemExit(1) - uidservtype = confcust.get_uidserviceendpoints() + # uidservtype = confcust.get_uidserviceendpoints() - webapi_opts = get_webapi_opts(cglob, confcust) + # webapi_opts = get_webapi_opts(cglob, confcust) - loop = uvloop.new_event_loop() + # loop = uvloop.new_event_loop() + # asyncio.set_event_loop(loop) + + ################################################################################### + + config = ConfigClass(args) + + globopts, _, _ = config.get_globopts_n_pass_ext() + confcust = config.get_confcust(globopts) + + loop = config.get_loop() asyncio.set_event_loop(loop) + + ################################################################################### + try: cust = list(confcust.get_customers())[0] - task = TaskCsvDowntimes(loop, logger, sys.argv[0], globopts, - webapi_opts, confcust, - confcust.get_custname(cust), feed, - current_date, uidservtype, args.date[0], - timestamp) + cust = confcust.get_custname(cust) + + + # task = TaskCsvDowntimes(loop, logger, sys.argv[0], globopts, + # webapi_opts, confcust, + # confcust.get_custname(cust), feed, + # current_date, uidservtype, args.date[0], + # timestamp) + + ######################################################################### + + task = TaskCsvDowntimes(cust, current_date, timestamp) + + ######################################################################### + + loop.run_until_complete(task.run()) + except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc: logger.error(repr(exc)) diff --git a/exec/service-types-csv-connector.py b/exec/service-types-csv-connector.py index 078045da..a80acaa4 100755 --- a/exec/service-types-csv-connector.py +++ b/exec/service-types-csv-connector.py @@ -8,6 +8,7 @@ import asyncio import uvloop +from argo_connectors.singleton_config import ConfigClass from argo_connectors.exceptions import ConnectorHttpError, ConnectorParseError from argo_connectors.log import Logger from argo_connectors.tasks.flat_servicetypes import TaskFlatServiceTypes @@ -39,42 +40,66 @@ def main(): parser.add_argument('--initial', dest='initsync', help='initial sync of service types', action='store_true', default=False, required=False) args = parser.parse_args() - fixed_date = None - if args.date and date_check(args.date): - fixed_date = args.date - - logger = Logger(os.path.basename(sys.argv[0])) - confpath = args.gloconf[0] if args.gloconf else None - cglob = Global(sys.argv[0], confpath) - globopts = cglob.parse() - - confpath = args.custconf[0] if args.custconf else None - confcust = CustomerConf(sys.argv[0], confpath) - confcust.parse() - confcust.make_dirstruct() - confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) - feed = confcust.get_servicesfeed() - custname = confcust.get_custname() - logger.customer = confcust.get_custname() - - auth_custopts = confcust.get_authopts() - auth_opts = cglob.merge_opts(auth_custopts, 'authentication') - auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') - if not auth_complete: - logger.error('%s options incomplete, missing %s' % ('authentication', ' '.join(missing))) - raise SystemExit(1) + # fixed_date = None + # if args.date and date_check(args.date): + # fixed_date = args.date + + # logger = Logger(os.path.basename(sys.argv[0])) + # confpath = args.gloconf[0] if args.gloconf else None + # cglob = Global(sys.argv[0], confpath) + # globopts = cglob.parse() + + # confpath = args.custconf[0] if args.custconf else None + # confcust = CustomerConf(sys.argv[0], confpath) + # confcust.parse() + # confcust.make_dirstruct() + # confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) + # feed = confcust.get_servicesfeed() + # custname = confcust.get_custname() + # logger.customer = confcust.get_custname() + + # auth_custopts = confcust.get_authopts() + # auth_opts = cglob.merge_opts(auth_custopts, 'authentication') + # auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') + # if not auth_complete: + # logger.error('%s options incomplete, missing %s' % ('authentication', ' '.join(missing))) + # raise SystemExit(1) + + # webapi_opts = get_webapi_opts(cglob, confcust) + + # loop = uvloop.new_event_loop() + # asyncio.set_event_loop(loop) + + ####################################################################### + + config = ConfigClass(args) + + loop = config.get_loop() + asyncio.set_event_loop(loop) + + globopts, _, _ = config.get_globopts_n_pass_ext() + confcust = config.get_confcust(globopts) + fixed_date = config.get_fixed_date() + + ####################################################################### + - webapi_opts = get_webapi_opts(cglob, confcust) - loop = uvloop.new_event_loop() - asyncio.set_event_loop(loop) try: - task = TaskFlatServiceTypes( - loop, logger, sys.argv[0], globopts, auth_opts, webapi_opts, - confcust, custname, feed, fixed_date, is_csv=True, - initsync=args.initsync - ) + # task = TaskFlatServiceTypes( + # loop, logger, sys.argv[0], globopts, auth_opts, webapi_opts, + # confcust, custname, feed, fixed_date, is_csv=True, + # initsync=args.initsync + # ) + + ####################################################################### + + task = TaskFlatServiceTypes(initsync=args.initsync) + + ####################################################################### + + loop.run_until_complete(task.run()) except (KeyboardInterrupt) as exc: diff --git a/exec/topology-csv-connector.py b/exec/topology-csv-connector.py index 082b528d..537d1bef 100755 --- a/exec/topology-csv-connector.py +++ b/exec/topology-csv-connector.py @@ -7,6 +7,7 @@ import asyncio import uvloop +from argo_connectors.singleton_config import ConfigClass from argo_connectors.config import Global, CustomerConf from argo_connectors.exceptions import ConnectorHttpError, ConnectorParseError from argo_connectors.log import Logger @@ -22,14 +23,14 @@ isok = True -def get_webapi_opts(cglob, confcust): - webapi_custopts = confcust.get_webapiopts() - webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') - webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') - if not webapi_complete: - logger.error('Customer:%s %s options incomplete, missing %s' % (logger.customer, 'webapi', ' '.join(missopt))) - raise SystemExit(1) - return webapi_opts +# def get_webapi_opts(cglob, confcust): +# webapi_custopts = confcust.get_webapiopts() +# webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') +# webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') +# if not webapi_complete: +# logger.error('Customer:%s %s options incomplete, missing %s' % (logger.customer, 'webapi', ' '.join(missopt))) +# raise SystemExit(1) +# return webapi_opts def main(): @@ -41,45 +42,69 @@ def main(): parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) - - fixed_date = None - if args.date and date_check(args.date): - fixed_date = args.date - - confpath = args.gloconf[0] if args.gloconf else None - cglob = Global(sys.argv[0], confpath) - globopts = cglob.parse() - - confpath = args.custconf[0] if args.custconf else None - confcust = CustomerConf(sys.argv[0], confpath) - confcust.parse() - confcust.make_dirstruct() - confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) - topofeed = confcust.get_topofeed() - uidservendp = confcust.get_uidserviceendpoints() - topofetchtype = confcust.get_topofetchtype()[0] - custname = confcust.get_custname() - logger.customer = custname - - webapi_opts = get_webapi_opts(cglob, confcust) - - auth_custopts = confcust.get_authopts() - auth_opts = cglob.merge_opts(auth_custopts, 'authentication') - auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') - if not auth_complete: - logger.error('%s options incomplete, missing %s' % ('authentication', ' '.join(missing))) - raise SystemExit(1) - - loop = uvloop.new_event_loop() + + + # logger = Logger(os.path.basename(sys.argv[0])) + + # fixed_date = None + # if args.date and date_check(args.date): + # fixed_date = args.date + + # confpath = args.gloconf[0] if args.gloconf else None + # cglob = Global(sys.argv[0], confpath) + # globopts = cglob.parse() + + # confpath = args.custconf[0] if args.custconf else None + # confcust = CustomerConf(sys.argv[0], confpath) + # confcust.parse() + # confcust.make_dirstruct() + # confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) + # topofeed = confcust.get_topofeed() + # uidservendp = confcust.get_uidserviceendpoints() + # topofetchtype = confcust.get_topofetchtype()[0] + # custname = confcust.get_custname() + # logger.customer = custname + + # webapi_opts = get_webapi_opts(cglob, confcust) + + # auth_custopts = confcust.get_authopts() + # auth_opts = cglob.merge_opts(auth_custopts, 'authentication') + # auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') + # if not auth_complete: + # logger.error('%s options incomplete, missing %s' % ('authentication', ' '.join(missing))) + # raise SystemExit(1) + + # loop = uvloop.new_event_loop() + # asyncio.set_event_loop(loop) + +#################################################################################### + + + config = ConfigClass(args) + + loop = config.get_loop() asyncio.set_event_loop(loop) + fixed_date = config.get_fixed_date() + + +#################################################################################### + + try: - task = TaskFlatTopology( - loop, logger, sys.argv[0], globopts, webapi_opts, confcust, - custname, topofeed, topofetchtype, fixed_date, uidservendp, - is_csv=True - ) + # task = TaskFlatTopology( + # loop, logger, sys.argv[0], globopts, webapi_opts, confcust, + # custname, topofeed, topofetchtype, fixed_date, uidservendp, + # is_csv=True + # ) + + #################################################################################### + + task = TaskFlatTopology() + + #################################################################################### + + loop.run_until_complete(task.run()) except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc: diff --git a/exec/topology-gocdb-connector.py b/exec/topology-gocdb-connector.py index 070e66b1..7525cfcd 100755 --- a/exec/topology-gocdb-connector.py +++ b/exec/topology-gocdb-connector.py @@ -22,28 +22,28 @@ # GOCDB explicitly says &scope='' for all scopes -def get_webapi_opts(cglob, confcust): - webapi_custopts = confcust.get_webapiopts() - webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') - webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') - if not webapi_complete: - logger.error('Customer:%s %s options incomplete, missing %s' % - (logger.customer, 'webapi', ' '.join(missopt))) - raise SystemExit(1) - return webapi_opts - - -def get_bdii_opts(confcust): - bdii_custopts = confcust._get_cust_options('BDIIOpts') - if bdii_custopts: - bdii_complete, missing = confcust.is_complete_bdii(bdii_custopts) - if not bdii_complete: - logger.error('%s options incomplete, missing %s' % - ('bdii', ' '.join(missing))) - raise SystemExit(1) - return bdii_custopts - else: - return None +# def get_webapi_opts(cglob, confcust): +# webapi_custopts = confcust.get_webapiopts() +# webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') +# webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') +# if not webapi_complete: +# logger.error('Customer:%s %s options incomplete, missing %s' % +# (logger.customer, 'webapi', ' '.join(missopt))) +# raise SystemExit(1) +# return webapi_opts + + +# def get_bdii_opts(confcust): +# bdii_custopts = confcust._get_cust_options('BDIIOpts') +# if bdii_custopts: +# bdii_complete, missing = confcust.is_complete_bdii(bdii_custopts) +# if not bdii_complete: +# logger.error('%s options incomplete, missing %s' % +# ('bdii', ' '.join(missing))) +# raise SystemExit(1) +# return bdii_custopts +# else: +# return None def main(): diff --git a/modules/singleton_config.py b/modules/singleton_config.py index c599657f..49d1cc13 100755 --- a/modules/singleton_config.py +++ b/modules/singleton_config.py @@ -15,13 +15,14 @@ isok = True -def get_webapi_opts(cglob, confcust): +def get_webapi_opts(cglob, confcust, custname): webapi_custopts = confcust.get_webapiopts() webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') + print("custname: ", custname) if not webapi_complete: logger.error('Customer:%s %s options incomplete, missing %s' % - (logger.customer, 'webapi', ' '.join(missopt))) + (custname, 'webapi', ' '.join(missopt))) raise SystemExit(1) return webapi_opts @@ -39,22 +40,6 @@ def get_bdii_opts(confcust): return None -# class Singleton(object): -# __instance = None - -# def __init__(self): -# if Singleton.__instance != None: -# raise Exception("This class is a singleton!") -# else: -# Singleton.__instance = self - -# @staticmethod -# def get_instance(): -# if Singleton.__instance == None: -# Singleton() -# return Singleton.__instance - - class Singleton(type): _instances = {} @@ -64,19 +49,6 @@ def __call__(cls, *args, **kwargs): return cls._instances[cls] - -# class EventLoopSingleton(metaclass=Singleton): -# _loop = None - -# @classmethod -# def get_event_loop(cls): -# if cls._loop is None: -# cls._loop = uvloop.new_event_loop() -# return cls._loop - - - - class ConfigClass(metaclass=Singleton): _loop = None @@ -155,10 +127,10 @@ def bdii_opts_data(self, confcust): bdii_opts = get_bdii_opts(confcust) return bdii_opts - def get_webapi_opts_data(self, confcust): + def get_webapi_opts_data(self, confcust, connector_name): confpath = self.args.gloconf[0] if self.args.gloconf else None cglob = Global(sys.argv[0], confpath) - webapi_opts = get_webapi_opts(cglob, confcust) + webapi_opts = get_webapi_opts(cglob, confcust, connector_name) return webapi_opts def notiflag_data(self, confcust): @@ -202,4 +174,10 @@ def get_downtime_feed(self, confcust): return confcust.get_downfeed() def get_target_date(self): + return self.args.date[0] + + def is_csv(self): + return True + + def get_targetdate(self): return self.args.date[0] \ No newline at end of file diff --git a/modules/tasks/flat_downtimes.py b/modules/tasks/flat_downtimes.py index aa60970a..3fb6a6a5 100644 --- a/modules/tasks/flat_downtimes.py +++ b/modules/tasks/flat_downtimes.py @@ -1,7 +1,9 @@ import os +import asyncio from urllib.parse import urlparse +from argo_connectors.singleton_config import ConfigClass from argo_connectors.exceptions import ConnectorHttpError, ConnectorParseError from argo_connectors.io.http import SessionWithRetry from argo_connectors.io.webapi import WebAPI @@ -10,22 +12,42 @@ class TaskCsvDowntimes(object): - def __init__(self, loop, logger, connector_name, globopts, webapi_opts, - confcust, custname, feed, current_date, - uidservtype, targetdate, timestamp): - self.event_loop = loop - self.logger = logger - self.connector_name = connector_name - self.globopts = globopts - self.webapi_opts = webapi_opts - self.confcust = confcust - self.custname = custname - self.feed = feed + # def __init__(self, loop, logger, connector_name, globopts, webapi_opts, + # confcust, custname, feed, current_date, + # uidservtype, targetdate, timestamp): + # self.event_loop = loop + + # self.logger = logger + + # self.connector_name = connector_name + + # self.globopts = globopts + + # self.webapi_opts = webapi_opts + + # self.confcust = confcust + + # self.custname = custname + + # self.feed = feed + + # self.current_date = current_date + + # self.uidservtype = uidservtype + + # self.targetdate = targetdate + # self.timestamp = timestamp + + + ########################################################################### + + def __init__(self, cust, current_date, timestamp): + self.custname = cust self.current_date = current_date - self.uidservtype = uidservtype - self.targetdate = targetdate self.timestamp = timestamp + self.config = ConfigClass() + self.loop = self.config.get_loop() + asyncio.set_event_loop(self.loop) + self.logger = self.config.get_logger() + self.connector_name = self.config.get_connector_name() + self.globopts, self.pass_extensions, self.cglob = self.config.get_globopts_n_pass_ext() + self.confcust = self.config.get_confcust(self.globopts) + self.webapi_opts = self.config.get_webapi_opts_data(self.confcust, self.custname) + self.feed = self.config.get_downtime_feed(self.confcust) + self.uidservtype = self.config.uidservendp_data(self.confcust) + self.targetdate = self.config.get_targetdate() + + async def fetch_data(self): session = SessionWithRetry(self.logger, os.path.basename(self.connector_name), diff --git a/modules/tasks/flat_servicetypes.py b/modules/tasks/flat_servicetypes.py index 2e2b6805..43480943 100644 --- a/modules/tasks/flat_servicetypes.py +++ b/modules/tasks/flat_servicetypes.py @@ -3,6 +3,7 @@ from urllib.parse import urlparse +from argo_connectors.singleton_config import ConfigClass from argo_connectors.io.http import SessionWithRetry from argo_connectors.parse.flat_servicetypes import ParseFlatServiceTypes from argo_connectors.parse.webapi_servicetypes import ParseWebApiServiceTypes @@ -20,21 +21,41 @@ def contains_exception(list): class TaskFlatServiceTypes(object): - def __init__(self, loop, logger, connector_name, globopts, auth_opts, - webapi_opts, confcust, custname, feed, timestamp, - is_csv=False, initsync=False): - self.logger = logger - self.loop = loop - self.connector_name = connector_name - self.auth_opts = auth_opts - self.globopts = globopts - self.webapi_opts = webapi_opts - self.confcust = confcust - self.custname = custname - self.feed = feed - self.timestamp = timestamp - self.is_csv = is_csv + # def __init__(self, loop, logger, connector_name, globopts, auth_opts, + # webapi_opts, confcust, custname, feed, timestamp, + # is_csv=False, initsync=False): + # self.logger = logger + # self.loop = loop + # self.connector_name = connector_name + # self.auth_opts = auth_opts + # self.globopts = globopts + # self.webapi_opts = webapi_opts + # self.confcust = confcust + # self.custname = custname + # self.feed = feed + # self.timestamp = timestamp + # self.is_csv = is_csv + # self.initsync = initsync + # print("self.initsync: ", self.initsync) + + ######################################################################### + + def __init__(self, initsync=False): self.initsync = initsync + self.config = ConfigClass() + self.loop = self.config.get_loop() + asyncio.set_event_loop(self.loop) + self.logger = self.config.get_logger() + self.connector_name = self.config.get_connector_name() + self.globopts, self.pass_extensions, self.cglob = self.config.get_globopts_n_pass_ext() + self.confcust = self.config.get_confcust(self.globopts) + self.custname = self.config.custname_data(self.confcust) + self.webapi_opts = self.config.get_webapi_opts_data(self.confcust, self.custname) + self.auth_opts = self.config.get_auth_opts(self.confcust, self.logger) + self.feed = self.config.get_feed(self.confcust) + self.timestamp = self.config.get_fixed_date() + self.is_csv = self.config.is_csv() + async def fetch_data(self): feed_parts = urlparse(self.feed) diff --git a/modules/tasks/flat_topology.py b/modules/tasks/flat_topology.py index 965e2d68..c3a00dee 100644 --- a/modules/tasks/flat_topology.py +++ b/modules/tasks/flat_topology.py @@ -3,6 +3,7 @@ from urllib.parse import urlparse +from argo_connectors.singleton_config import ConfigClass from argo_connectors.io.http import SessionWithRetry from argo_connectors.parse.flat_topology import ParseFlatEndpoints from argo_connectors.parse.flat_contacts import ParseContacts @@ -12,21 +13,40 @@ class TaskFlatTopology(object): - def __init__(self, loop, logger, connector_name, globopts, webapi_opts, - confcust, custname, topofeed, fetchtype, fixed_date, - uidservendp, is_csv=False): - self.event_loop = loop - self.logger = logger - self.connector_name = connector_name - self.globopts = globopts - self.webapi_opts = webapi_opts - self.confcust = confcust - self.custname = custname - self.topofeed = topofeed - self.fetchtype = fetchtype - self.fixed_date = fixed_date - self.uidservendp = uidservendp - self.is_csv = is_csv + # def __init__(self, loop, logger, connector_name, globopts, webapi_opts, + # confcust, custname, topofeed, fetchtype, fixed_date, + # uidservendp, is_csv=False): + # self.event_loop = loop + # self.logger = logger + # self.connector_name = connector_name + # self.globopts = globopts + # self.webapi_opts = webapi_opts + # self.confcust = confcust + # self.custname = custname + # self.topofeed = topofeed + # self.fetchtype = fetchtype + # self.fixed_date = fixed_date + # self.uidservendp = uidservendp + # self.is_csv = is_csv + + ############################################################################ + + def __init__(self): + self.config = ConfigClass() + self.event_loop = self.config.get_loop() + asyncio.set_event_loop(self.event_loop) + self.logger = self.config.get_logger() + self.connector_name = self.config.get_connector_name() + self.globopts, _, _= self.config.get_globopts_n_pass_ext() + self.confcust = self.config.get_confcust(self.globopts) + self.custname = self.config.custname_data(self.confcust) + self.webapi_opts = self.config.get_webapi_opts_data(self.confcust, self.custname) + self.topofeed = self.config.topofeed_data(self.confcust) + self.fetchtype = self.config.topofetchtype_data(self.confcust)[0] + self.fixed_date = self.config.get_fixed_date() + self.uidservendp = self.config.uidservendp_data(self.confcust) + self.is_csv = self.config.is_csv() + def _is_feed(self, feed): data = urlparse(feed) @@ -84,7 +104,7 @@ async def run(self): js = json.load(fp) group_groups, group_endpoints = self.parse_source_topo(js) except IOError as exc: - self.logger.error('Customer:%s : Problem opening %s - %s' % (self.logger.customer, self.topofeed, repr(exc))) + self.logger.error('Customer:%s : Problem opening %s - %s' % (self.custname, self.topofeed, repr(exc))) await write_state(self.connector_name, self.globopts, self.confcust, self.fixed_date, True) diff --git a/modules/tasks/gocdb_downtimes.py b/modules/tasks/gocdb_downtimes.py index 53fe061a..6819ef56 100644 --- a/modules/tasks/gocdb_downtimes.py +++ b/modules/tasks/gocdb_downtimes.py @@ -46,8 +46,8 @@ def __init__(self, custname, start, end, timestamp): self.globopts, self.pass_extensions, self.cglob = self.config.get_globopts_n_pass_ext() self.confcust = self.config.get_confcust(self.globopts) self.auth_opts = self.config.get_auth_opts(self.confcust, self.logger) - self.webapi_opts = self.config.get_webapi_opts_data(self.confcust) self.custname = self.config.custname_data(self.confcust) + self.webapi_opts = self.config.get_webapi_opts_data(self.confcust, self.custname) self.feed = self.config.get_downtime_feed(self.confcust) self.uidservtype = self.config.uidservendp_data(self.confcust) self.targetdate = self.config.get_target_date() diff --git a/modules/tasks/gocdb_servicetypes.py b/modules/tasks/gocdb_servicetypes.py index 63420fe3..0ad2b8b0 100644 --- a/modules/tasks/gocdb_servicetypes.py +++ b/modules/tasks/gocdb_servicetypes.py @@ -46,8 +46,8 @@ def __init__(self): self.globopts, _, _ = self.config.get_globopts_n_pass_ext() self.confcust = self.config.get_confcust(self.globopts) self.auth_opts = self.config.get_auth_opts(self.confcust, self.logger) - self.webapi_opts = self.config.get_webapi_opts_data(self.confcust) self.custname = self.config.custname_data(self.confcust) + self.webapi_opts = self.config.get_webapi_opts_data(self.confcust, self.custname) self.feed = self.config.get_feed(self.confcust) self.timestamp = self.config.get_fixed_date() self.initsync = self.config.get_initsync() diff --git a/modules/tasks/gocdb_topology.py b/modules/tasks/gocdb_topology.py index 63d32749..49a9b7eb 100644 --- a/modules/tasks/gocdb_topology.py +++ b/modules/tasks/gocdb_topology.py @@ -218,7 +218,7 @@ def __init__(self): self.custname = self.config.custname_data(self.confcust) self.auth_opts = self.config.get_auth_opts(self.confcust, self.logger) self.bdii_opts = self.config.bdii_opts_data(self.confcust) - self.webapi_opts = self.config.get_webapi_opts_data(self.confcust) + self.webapi_opts = self.config.get_webapi_opts_data(self.confcust, self.custname) self.notiflag = self.config.notiflag_data(self.confcust) self.SERVICE_ENDPOINTS_PI, self.SERVICE_GROUPS_PI, self.SITES_PI = self.config.service_data(self.confcust) From 28743ab8328d3e597985a0d6871c284e7cacae8a Mon Sep 17 00:00:00 2001 From: Dean Hudek Date: Tue, 28 Mar 2023 13:49:04 +0200 Subject: [PATCH 08/11] agora refactored, bug fixes --- exec/downtimes-csv-connector.py | 3 +- exec/downtimes-gocdb-connector.py | 1 + exec/metricprofile-webapi-connector.py | 2 + exec/service-types-csv-connector.py | 8 ++- exec/service-types-gocdb-connector.py | 9 +-- exec/topology-agora-connector.py | 86 ++++++++++++++++---------- exec/topology-csv-connector.py | 6 +- exec/topology-gocdb-connector.py | 42 +++---------- exec/weights-vapor-connector.py | 8 ++- modules/singleton_config.py | 7 +-- modules/tasks/agora_topology.py | 44 +++++++++---- modules/tasks/gocdb_topology.py | 3 +- 12 files changed, 124 insertions(+), 95 deletions(-) diff --git a/exec/downtimes-csv-connector.py b/exec/downtimes-csv-connector.py index 551d5b86..69f2ee4a 100755 --- a/exec/downtimes-csv-connector.py +++ b/exec/downtimes-csv-connector.py @@ -78,7 +78,8 @@ def main(): globopts, _, _ = config.get_globopts_n_pass_ext() confcust = config.get_confcust(globopts) - + logger = config.get_logger() + loop = config.get_loop() asyncio.set_event_loop(loop) diff --git a/exec/downtimes-gocdb-connector.py b/exec/downtimes-gocdb-connector.py index da3664a3..9a8c8b79 100755 --- a/exec/downtimes-gocdb-connector.py +++ b/exec/downtimes-gocdb-connector.py @@ -94,6 +94,7 @@ def main(): globopts, _, _ = config.get_globopts_n_pass_ext() confcust = config.get_confcust(globopts) + logger = config.get_logger() loop = config.get_loop() asyncio.set_event_loop(loop) diff --git a/exec/metricprofile-webapi-connector.py b/exec/metricprofile-webapi-connector.py index 421db4f8..f3997dfd 100755 --- a/exec/metricprofile-webapi-connector.py +++ b/exec/metricprofile-webapi-connector.py @@ -53,6 +53,8 @@ def main(): globopts, _, _ = config.get_globopts_n_pass_ext() confcust = config.get_confcust(globopts) + logger = config.get_logger() + loop = config.get_loop() asyncio.set_event_loop(loop) diff --git a/exec/service-types-csv-connector.py b/exec/service-types-csv-connector.py index a80acaa4..61de1baa 100755 --- a/exec/service-types-csv-connector.py +++ b/exec/service-types-csv-connector.py @@ -74,12 +74,14 @@ def main(): config = ConfigClass(args) - loop = config.get_loop() - asyncio.set_event_loop(loop) - globopts, _, _ = config.get_globopts_n_pass_ext() confcust = config.get_confcust(globopts) + logger = config.get_logger() fixed_date = config.get_fixed_date() + + loop = config.get_loop() + asyncio.set_event_loop(loop) + ####################################################################### diff --git a/exec/service-types-gocdb-connector.py b/exec/service-types-gocdb-connector.py index 0f4d2b6f..7f1ccd3e 100755 --- a/exec/service-types-gocdb-connector.py +++ b/exec/service-types-gocdb-connector.py @@ -74,13 +74,14 @@ def main(): config = ConfigClass(args) + globopts, _, _ = config.get_globopts_n_pass_ext() + confcust = config.get_confcust(globopts) + logger = config.get_logger() + fixed_date = config.get_fixed_date() + loop = config.get_loop() asyncio.set_event_loop(loop) - fixed_date = config.get_fixed_date() - globopts, _, _ = config.get_globopts_n_pass_ext() - confcust = config.get_confcust(globopts) - ########################################################################## diff --git a/exec/topology-agora-connector.py b/exec/topology-agora-connector.py index 53f04ac7..1e3a0f0d 100755 --- a/exec/topology-agora-connector.py +++ b/exec/topology-agora-connector.py @@ -7,6 +7,7 @@ import uvloop import asyncio +from argo_connectors.singleton_config import ConfigClass from argo_connectors.exceptions import ConnectorError, ConnectorHttpError, ConnectorParseError from argo_connectors.log import Logger from argo_connectors.config import Global, CustomerConf @@ -20,14 +21,14 @@ custname = '' -def get_webapi_opts(cglob, confcust): - webapi_custopts = confcust.get_webapiopts() - webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') - webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') - if not webapi_complete: - logger.error('Customer:%s %s options incomplete, missing %s' % (logger.customer, 'webapi', ' '.join(missopt))) - raise SystemExit(1) - return webapi_opts +# def get_webapi_opts(cglob, confcust): +# webapi_custopts = confcust.get_webapiopts() +# webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') +# webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') +# if not webapi_complete: +# logger.error('Customer:%s %s options incomplete, missing %s' % (logger.customer, 'webapi', ' '.join(missopt))) +# raise SystemExit(1) +# return webapi_opts def main(): @@ -39,35 +40,56 @@ def main(): parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) - fixed_date = None - if args.date and date_check(args.date): - fixed_date = args.date + # logger = Logger(os.path.basename(sys.argv[0])) + # fixed_date = None + # if args.date and date_check(args.date): + # fixed_date = args.date - confpath = args.gloconf[0] if args.gloconf else None - cglob = Global(sys.argv[0], confpath) - globopts = cglob.parse() + # confpath = args.gloconf[0] if args.gloconf else None + # cglob = Global(sys.argv[0], confpath) + # globopts = cglob.parse() - confpath = args.custconf[0] if args.custconf else None - confcust = CustomerConf(sys.argv[0], confpath) - confcust.parse() - confcust.make_dirstruct() - confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) - global custname - custname = confcust.get_custname() - fetchtype = confcust.get_topofetchtype()[0] - webapi_opts = get_webapi_opts(cglob, confcust) - logger.customer = custname - uidservendp = confcust.get_uidserviceendpoints() - - loop = uvloop.new_event_loop() + # confpath = args.custconf[0] if args.custconf else None + # confcust = CustomerConf(sys.argv[0], confpath) + # confcust.parse() + # confcust.make_dirstruct() + # confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) + # global custname + # custname = confcust.get_custname() + # fetchtype = confcust.get_topofetchtype()[0] + # webapi_opts = get_webapi_opts(cglob, confcust) + # logger.customer = custname + # uidservendp = confcust.get_uidserviceendpoints() + + # loop = uvloop.new_event_loop() + # asyncio.set_event_loop(loop) + + ############################################################################### + + config = ConfigClass(args) + + globopts, _, _ = config.get_globopts_n_pass_ext() + confcust = config.get_confcust(globopts) + logger = config.get_logger() + fixed_date = config.get_fixed_date() + + loop = config.get_loop() asyncio.set_event_loop(loop) + ############################################################################### + try: - task = TaskProviderTopology( - loop, logger, sys.argv[0], globopts, webapi_opts, confcust, - uidservendp, fetchtype, fixed_date - ) + # task = TaskProviderTopology( + # loop, logger, sys.argv[0], globopts, webapi_opts, confcust, + # uidservendp, fetchtype, fixed_date + # ) + + ################################################################################# + + task = TaskProviderTopology() + + ################################################################################# + loop.run_until_complete(task.run()) except (ConnectorError, ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc: diff --git a/exec/topology-csv-connector.py b/exec/topology-csv-connector.py index 537d1bef..1bd7dd52 100755 --- a/exec/topology-csv-connector.py +++ b/exec/topology-csv-connector.py @@ -81,13 +81,15 @@ def main(): config = ConfigClass(args) - + + globopts, _, _ = config.get_globopts_n_pass_ext() + confcust = config.get_confcust(globopts) + logger = config.get_logger() loop = config.get_loop() asyncio.set_event_loop(loop) fixed_date = config.get_fixed_date() - #################################################################################### diff --git a/exec/topology-gocdb-connector.py b/exec/topology-gocdb-connector.py index 7525cfcd..500f23ec 100755 --- a/exec/topology-gocdb-connector.py +++ b/exec/topology-gocdb-connector.py @@ -15,10 +15,10 @@ from argo_connectors.tasks.gocdb_topology import TaskGocdbTopology from argo_connectors.utils import date_check -logger = None -globopts = {} -custname = '' -isok = True +# logger = None +# globopts = {} +# custname = '' +# isok = True # GOCDB explicitly says &scope='' for all scopes @@ -121,11 +121,15 @@ def main(): ############################################################################################### config = ConfigClass(args) - + + globopts, _, _ = config.get_globopts_n_pass_ext() + confcust = config.get_confcust(globopts) + logger = config.get_logger() + fixed_date = config.get_fixed_date() + loop = config.get_loop() asyncio.set_event_loop(loop) - fixed_date = config.get_fixed_date() ############################################################################################### @@ -161,29 +165,3 @@ def main(): if __name__ == '__main__': main() - - - - - - - -# print("loop: ", loop) # -# print("logger: ", logger) # -# print("sys.argv[0]: ", sys.argv[0]) # /usr/libexec/argo-connectors/topology-gocdb-connector.py -# print("SERVICE_ENDPOINTS_PI: ", SERVICE_ENDPOINTS_PI) # https://goc-sdc.argo.grnet.gr//gocdbpi/private/?method=get_service_endpoint&scope= -# print("SERVICE_GROUPS_PI: ", SERVICE_GROUPS_PI) # https://goc-sdc.argo.grnet.gr//gocdbpi/private/?method=get_service_group&scope= -# print("SITES_PI: ", SITES_PI) # https://goc-sdc.argo.grnet.gr//gocdbpi/private/?method=get_site&scope= -# print("globopts: ", globopts) # {'generalwritejson': 'True', 'generalpublishwebapi': 'False', 'generalpassextensions': 'True', 'generalcompressjson': 'False', 'authenticationhostkey': '/etc/grid-security/hostkey.pem', 'authenticationhostcert': '/etc/grid-security/hostcert.pem', 'authenticationcapath': '/etc/grid-security/certificates', 'authenticationcafile': '/etc/pki/tls/certs/ca-bundle.crt', 'authenticationverifyservercert': 'True', 'authenticationuseplainhttpauth': 'False', 'authenticationhttpuser': 'xxxx', 'authenticationhttppass': 'xxxx', 'connectiontimeout': '180', 'connectionretry': '60', 'connectionsleepretry': '60', 'connectionretryrandom': 'True', 'connectionsleeprandomretrymax': '300', 'inputstatesavedir': '/var/lib/argo-connectors/states/', 'inputstatedays': '3', 'webapihost': 'api.devel.argo.grnet.gr', 'outputtopologygroupofendpoints': 'group_endpoints_DATE.json', 'outputtopologygroupofgroups': 'group_groups_DATE.json'} -# print("auth_opts: ", auth_opts) # {'authenticationhostkey': '/etc/grid-security/hostkey.pem', 'authenticationhostcert': '/etc/grid-security/hostcert.pem', 'authenticationcapath': '/etc/grid-security/certificates', 'authenticationcafile': '/etc/pki/tls/certs/ca-bundle.crt', 'authenticationverifyservercert': 'True', 'authenticationuseplainhttpauth': 'False', 'authenticationhttpuser': 'xxxx', 'authenticationhttppass': 'xxxx'} -# print("webapi_opts: ", webapi_opts) # {'webapitoken': '4473153af6c67a650a74d81d367e9e83f70e2b7b', 'webapihost': 'api.devel.argo.grnet.gr'} -# print("bdii_opts: ", bdii_opts) # None -# print("confcust: ", confcust) # -# print("custname: ", custname) # SDC -# print("topofeed: ", topofeed) # https://goc-sdc.argo.grnet.gr/ -# print("topofetchtype: ", topofetchtype) # ['sites', 'servicegroups'] -# print("fixed_date: ", fixed_date) # None -# print("uidservendp: ", uidservendp) # False -# print("pass_extensions: ", pass_extensions) # True -# print("topofeedpaging: ", topofeedpaging) # False -# print("notiflag: ", notiflag) # True \ No newline at end of file diff --git a/exec/weights-vapor-connector.py b/exec/weights-vapor-connector.py index 2d44ee40..443a5a4d 100755 --- a/exec/weights-vapor-connector.py +++ b/exec/weights-vapor-connector.py @@ -54,9 +54,7 @@ def main(): ##################################################################### config = ConfigClass(args) - loop = config.get_loop() - asyncio.set_event_loop(loop) - + logger = config.get_logger() fixed_date = config.get_fixed_date() globopts, _, _ = config.get_globopts_n_pass_ext() @@ -64,6 +62,10 @@ def main(): VAPORPI = config.vaporrpi_data(confcust) feeds = config.get_feeds(confcust, VAPORPI) + loop = config.get_loop() + asyncio.set_event_loop(loop) + + ##################################################################### diff --git a/modules/singleton_config.py b/modules/singleton_config.py index 49d1cc13..cf5f1f39 100755 --- a/modules/singleton_config.py +++ b/modules/singleton_config.py @@ -10,16 +10,15 @@ logger = None -globopts = {} -custname = '' -isok = True +# globopts = {} +# custname = '' +# isok = True def get_webapi_opts(cglob, confcust, custname): webapi_custopts = confcust.get_webapiopts() webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') - print("custname: ", custname) if not webapi_complete: logger.error('Customer:%s %s options incomplete, missing %s' % (custname, 'webapi', ' '.join(missopt))) diff --git a/modules/tasks/agora_topology.py b/modules/tasks/agora_topology.py index f2363899..cf20201d 100644 --- a/modules/tasks/agora_topology.py +++ b/modules/tasks/agora_topology.py @@ -1,6 +1,7 @@ import asyncio from urllib.parse import urlparse +from argo_connectors.singleton_config import ConfigClass from argo_connectors.io.http import SessionWithRetry from argo_connectors.io.webapi import WebAPI from argo_connectors.parse.agora_topology import ParseAgoraTopo @@ -17,17 +18,34 @@ def contains_exception(list): class TaskProviderTopology(object): - def __init__(self, loop, logger, connector_name, globopts, webapi_opts, - confcust, uidservendp, fetchtype, fixed_date): - self.loop = loop - self.logger = logger - self.connector_name = connector_name - self.globopts = globopts - self.webapi_opts = webapi_opts - self.confcust = confcust - self.uidservendp = uidservendp - self.fixed_date = fixed_date - self.fetchtype = fetchtype + # def __init__(self, loop, logger, connector_name, globopts, webapi_opts, + # confcust, uidservendp, fetchtype, fixed_date): + # self.loop = loop + # self.logger = logger + # self.connector_name = connector_name + # self.globopts = globopts + # self.webapi_opts = webapi_opts + # self.confcust = confcust + # self.uidservendp = uidservendp + # self.fixed_date = fixed_date + # self.fetchtype = fetchtype + + ################################################################################# + + def __init__(self): + self.config = ConfigClass() + self.loop = self.config.get_loop() + asyncio.set_event_loop(self.loop) + self.logger = self.config.get_logger() + self.connector_name = self.config.get_connector_name() + self.globopts, self.pass_extensions, self.cglob = self.config.get_globopts_n_pass_ext() + self.confcust = self.config.get_confcust(self.globopts) + self.custname = self.config.custname_data(self.confcust) + self.webapi_opts = self.config.get_webapi_opts_data(self.confcust, self.custname) + self.uidservendp = self.config.uidservendp_data(self.confcust) + self.fixed_date = self.config.get_fixed_date() + self.fetchtype = self.config.topofetchtype_data(self.confcust)[0] + def parse_source_topo(self, resources, providers): @@ -51,7 +69,7 @@ async def send_webapi(self, webapi_opts, data, topotype, fixed_date=None): async def fetch_data(self, feed): remote_topo = urlparse(feed) - session = SessionWithRetry(self.logger, self.logger.customer, self.globopts, handle_session_close=True) + session = SessionWithRetry(self.logger, self.custname, self.globopts, handle_session_close=True) headers = { "Accept": "application/json", } @@ -106,4 +124,4 @@ async def run(self): if eval(self.globopts['GeneralWriteJson'.lower()]): write_json(self.logger, self.globopts, self.confcust, group_providers, group_resources, self.fixed_date) - self.logger.info('Customer:' + self.logger.customer + ' Fetched Endpoints:%d' % (numge) + ' Groups(%s):%d' % (self.fetchtype, numgg)) + self.logger.info('Customer:' + self.custname + ' Fetched Endpoints:%d' % (numge) + ' Groups(%s):%d' % (self.fetchtype, numgg)) diff --git a/modules/tasks/gocdb_topology.py b/modules/tasks/gocdb_topology.py index 49a9b7eb..847599aa 100644 --- a/modules/tasks/gocdb_topology.py +++ b/modules/tasks/gocdb_topology.py @@ -201,7 +201,8 @@ class TaskGocdbTopology(TaskParseContacts, TaskParseTopology): # self.topofeedpaging = topofeedpaging # self.notification_flag = notiflag - + ################################################################################################# + def __init__(self): self.config = ConfigClass() self.loop = self.config.get_loop() From 824a26f05fde44ab72fbcd64ac348c23367d115c Mon Sep 17 00:00:00 2001 From: Dean Hudek Date: Tue, 4 Apr 2023 11:12:26 +0200 Subject: [PATCH 09/11] args parser as method --- exec/topology-gocdb-connector.py | 40 ++++++++++++---------- exec/weights-vapor-connector.py | 12 ++++--- modules/io/jsonwrite.py | 13 +++++-- modules/singleton_config.py | 58 ++++++++++++++++++-------------- modules/tasks/common.py | 28 ++++++++++++--- modules/tasks/gocdb_topology.py | 19 +++++++---- modules/tasks/vapor_weights.py | 13 +++---- 7 files changed, 115 insertions(+), 68 deletions(-) diff --git a/exec/topology-gocdb-connector.py b/exec/topology-gocdb-connector.py index 500f23ec..af068183 100755 --- a/exec/topology-gocdb-connector.py +++ b/exec/topology-gocdb-connector.py @@ -47,17 +47,20 @@ def main(): - global logger, globopts, confcust - parser = argparse.ArgumentParser(description="""Fetch entities (ServiceGroups, Sites, Endpoints) - from GOCDB for every customer and job listed in customer.conf and write them - in an appropriate place""") - parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', - help='path to customer configuration file', type=str, required=False) - parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', - help='path to global configuration file', type=str, required=False) - parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', - help='write data for this date', type=str, required=False) - args = parser.parse_args() + # #global logger, globopts, confcust + # parser = argparse.ArgumentParser(description="""Fetch entities (ServiceGroups, Sites, Endpoints) + # from GOCDB for every customer and job listed in customer.conf and write them + # in an appropriate place""") + + # parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', + # help='path to customer configuration file', type=str, required=False) + # parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', + # help='path to global configuration file', type=str, required=False) + # parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', + # help='write data for this date', type=str, required=False) + #args = parser.parse_args() # TODO: NASTAVITI DA PREBACIM ARGPARSE U METODE DA IH MOGU KORISTITI U SVAKOJ KLASI !!!!!!!!!!!!! + + #print("args: ", args) # logger = Logger(os.path.basename(sys.argv[0])) @@ -120,13 +123,14 @@ def main(): ############################################################################################### - config = ConfigClass(args) - - globopts, _, _ = config.get_globopts_n_pass_ext() - confcust = config.get_confcust(globopts) + config = ConfigClass() + args = config.parse_args() # DODAN ARGS + cglob = config.get_cglob(args) + globopts = config.get_globopts(cglob) # + confcust = config.get_confcust(globopts, args) # logger = config.get_logger() - fixed_date = config.get_fixed_date() - + fixed_date = config.get_fixed_date(args) # + loop = config.get_loop() asyncio.set_event_loop(loop) @@ -144,7 +148,7 @@ def main(): ############################################################################################### - task = TaskGocdbTopology() #TODO: OVAKO TREBA IZGLEDATI + task = TaskGocdbTopology() ############################################################################################### diff --git a/exec/weights-vapor-connector.py b/exec/weights-vapor-connector.py index 443a5a4d..39afb60c 100755 --- a/exec/weights-vapor-connector.py +++ b/exec/weights-vapor-connector.py @@ -30,7 +30,7 @@ def main(): help='path to global configuration file', type=str, required=False) parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) - args = parser.parse_args() + #args = parser.parse_args() # logger = Logger(os.path.basename(sys.argv[0])) @@ -53,12 +53,14 @@ def main(): ##################################################################### - config = ConfigClass(args) + config = ConfigClass()#args) + args = config.parse_args() logger = config.get_logger() - fixed_date = config.get_fixed_date() - globopts, _, _ = config.get_globopts_n_pass_ext() - confcust = config.get_confcust(globopts) + fixed_date = config.get_fixed_date(args) + cglob = config.get_cglob(args) + globopts = config.get_globopts(cglob) + confcust = config.get_confcust(globopts, args) VAPORPI = config.vaporrpi_data(confcust) feeds = config.get_feeds(confcust, VAPORPI) diff --git a/modules/io/jsonwrite.py b/modules/io/jsonwrite.py index 757ac9ab..172d9b07 100644 --- a/modules/io/jsonwrite.py +++ b/modules/io/jsonwrite.py @@ -2,11 +2,20 @@ import gzip +from argo_connectors.singleton_config import ConfigClass + + class JsonWriter(object): - def __init__(self, data, filename, compress_json): + def __init__(self, data, filename):#, compress_json): self.data = data self.filename = filename - self.compress_json = compress_json + #self.compress_json = compress_json + + self.config = ConfigClass() + self.args = self.config.parse_args() + self.cglob = self.config.get_cglob(self.args) + self.globopts = self.config.get_globopts(self.cglob) + self.compress_json = eval(self.globopts['GeneralCompressJson'.lower()]) def write_json(self): try: diff --git a/modules/singleton_config.py b/modules/singleton_config.py index cf5f1f39..44b88ae9 100755 --- a/modules/singleton_config.py +++ b/modules/singleton_config.py @@ -1,8 +1,8 @@ import os import sys -import asyncio import uvloop +import argparse from argo_connectors.log import Logger from argo_connectors.config import Global, CustomerConf @@ -51,10 +51,20 @@ def __call__(cls, *args, **kwargs): class ConfigClass(metaclass=Singleton): _loop = None - def __init__(self, args): - self.args = args + def __init__(self): self._setup_loop() + def parse_args(self): + parser = argparse.ArgumentParser() + parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', + help='path to customer configuration file', type=str, required=False) + parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', + help='path to global configuration file', type=str, required=False) + parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', + help='write data for this date', type=str, required=False) + args = parser.parse_args() + return args + def _setup_loop(self): if self._loop is None: self._loop = uvloop.new_event_loop() @@ -69,21 +79,23 @@ def get_logger(self): def get_connector_name(self): return sys.argv[0] - def get_fixed_date(self): + def get_fixed_date(self, args): fixed_date = None - if self.args.date and date_check(self.args.date): - fixed_date = self.args.date + if args.date and date_check(args.date): + fixed_date = args.date return fixed_date - def get_globopts_n_pass_ext(self): - confpath = self.args.gloconf[0] if self.args.gloconf else None + def get_cglob(self, args): # + confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) - globopts = cglob.parse() - pass_extensions = eval(globopts['GeneralPassExtensions'.lower()]) - return globopts, pass_extensions, cglob + return cglob - def get_confcust(self, globopts): - confpath = self.args.custconf[0] if self.args.custconf else None + def get_globopts(self, cglob): # + globopts = cglob.parse() + return globopts + + def get_confcust(self, globopts, args): + confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) confcust.parse() confcust.make_dirstruct() @@ -110,9 +122,7 @@ def custname_data(self, confcust): custname = confcust.get_custname() return custname - def get_auth_opts(self, confcust, logger): - confpath = self.args.gloconf[0] if self.args.gloconf else None - cglob = Global(sys.argv[0], confpath) + def get_auth_opts(self, confcust, cglob, logger): # auth_custopts = confcust.get_authopts() auth_opts = cglob.merge_opts(auth_custopts, 'authentication') auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') @@ -126,9 +136,7 @@ def bdii_opts_data(self, confcust): bdii_opts = get_bdii_opts(confcust) return bdii_opts - def get_webapi_opts_data(self, confcust, connector_name): - confpath = self.args.gloconf[0] if self.args.gloconf else None - cglob = Global(sys.argv[0], confpath) + def get_webapi_opts_data(self, confcust, cglob, connector_name): # webapi_opts = get_webapi_opts(cglob, confcust, connector_name) return webapi_opts @@ -166,17 +174,17 @@ def get_feed(self, confcust): feed = confcust.get_servicesfeed() return feed - def get_initsync(self): - return self.args.initsync + def get_initsync(self, args): + return args.initsync def get_downtime_feed(self, confcust): return confcust.get_downfeed() - def get_target_date(self): - return self.args.date[0] + def get_target_date(self, args): + return args.date[0] def is_csv(self): return True - def get_targetdate(self): - return self.args.date[0] \ No newline at end of file + def get_targetdate(self, args): + return args.date[0] \ No newline at end of file diff --git a/modules/tasks/common.py b/modules/tasks/common.py index 795da608..501190af 100644 --- a/modules/tasks/common.py +++ b/modules/tasks/common.py @@ -2,6 +2,17 @@ from argo_connectors.utils import filename_date, datestamp, date_check from argo_connectors.io.jsonwrite import JsonWriter +from argo_connectors.singleton_config import ConfigClass + + +config = ConfigClass() +args = config.parse_args() +logger = config.get_logger() +cglob = config.get_cglob(args) +globopts = config.get_globopts(cglob) +confcust = config.get_confcust(globopts, args) +fixed_date = config.get_fixed_date(args) + async def write_state(connector_name, globopts, confcust, fixed_date, state): cust = list(confcust.get_customers())[0] @@ -54,6 +65,7 @@ def write_downtimes_json(logger, globopts, confcust, dts, timestamp): logger.error('Customer:{} {}'.format(logger.customer, repr(excep))) raise SystemExit(1) +################################################################################################################## def write_weights_json(logger, globopts, cust, job, confcust, fixed_date, weights): jobdir = confcust.get_fulldir(cust, job) @@ -64,7 +76,7 @@ def write_weights_json(logger, globopts, cust, job, confcust, fixed_date, weight filename = filename_date( logger, globopts['OutputWeights'.lower()], jobdir) - json_writer = JsonWriter(weights, filename, globopts['generalcompressjson']) + json_writer = JsonWriter(weights, filename)#, globopts['generalcompressjson']) ret, excep = json_writer.write_json() if not ret: logger.error('Customer:%s Job:%s %s' % @@ -72,15 +84,20 @@ def write_weights_json(logger, globopts, cust, job, confcust, fixed_date, weight raise SystemExit(1) -def write_topo_json(logger, globopts, confcust, group_groups, group_endpoints, fixed_date): +################################################################################################################## + +#def write_topo_json(logger, globopts, confcust, group_groups, group_endpoints, fixed_date): +def write_topo_json(group_groups, group_endpoints): custdir = confcust.get_custdir() if fixed_date: filename = filename_date(logger, globopts['OutputTopologyGroupOfGroups'.lower( )], custdir, fixed_date.replace('-', '_')) else: filename = filename_date( - logger, globopts['OutputTopologyGroupOfGroups'.lower()], custdir) - json_writer = JsonWriter(group_groups, filename, globopts['generalcompressjson']) + logger, globopts['OutputTopologyGroupOfGroups'.lower()], custdir) + json_writer = JsonWriter(group_groups, filename)#, globopts['generalcompressjson']) + + ret, excep = json_writer.write_json() if not ret: logger.error('Customer:%s : %s' % (logger.customer, repr(excep))) @@ -92,7 +109,8 @@ def write_topo_json(logger, globopts, confcust, group_groups, group_endpoints, f else: filename = filename_date( logger, globopts['OutputTopologyGroupOfEndpoints'.lower()], custdir) - json_writer = JsonWriter(group_endpoints, filename, globopts['generalcompressjson']) + json_writer = JsonWriter(group_endpoints, filename)#, globopts['generalcompressjson']) + ret, excep = json_writer.write_json() if not ret: logger.error('Customer:%s : %s' % (logger.customer, repr(excep))) diff --git a/modules/tasks/gocdb_topology.py b/modules/tasks/gocdb_topology.py index 847599aa..f3521ed9 100644 --- a/modules/tasks/gocdb_topology.py +++ b/modules/tasks/gocdb_topology.py @@ -207,19 +207,22 @@ def __init__(self): self.config = ConfigClass() self.loop = self.config.get_loop() asyncio.set_event_loop(self.loop) + self.args = self.config.parse_args() self.logger = self.config.get_logger() self.connector_name = self.config.get_connector_name() - self.fixed_date = self.config.get_fixed_date() - self.globopts, self.pass_extensions, self.cglob = self.config.get_globopts_n_pass_ext() - self.confcust = self.config.get_confcust(self.globopts) + self.fixed_date = self.config.get_fixed_date(self.args) + self.cglob = self.config.get_cglob(self.args) + self.globopts= self.config.get_globopts(self.cglob) + self.pass_extensions = eval(self.globopts['GeneralPassExtensions'.lower()]) + self.confcust = self.config.get_confcust(self.globopts, self.args) self.topofeed = self.config.topofeed_data(self.confcust) self.topofeedpaging = self.config.topofeedpaging_data(self.confcust) self.uidservendp = self.config.uidservendp_data(self.confcust) self.topofetchtype = self.config.topofetchtype_data(self.confcust) self.custname = self.config.custname_data(self.confcust) - self.auth_opts = self.config.get_auth_opts(self.confcust, self.logger) + self.auth_opts = self.config.get_auth_opts(self.confcust, self.cglob, self.logger) self.bdii_opts = self.config.bdii_opts_data(self.confcust) - self.webapi_opts = self.config.get_webapi_opts_data(self.confcust, self.custname) + self.webapi_opts = self.config.get_webapi_opts_data(self.confcust, self.cglob, self.custname) self.notiflag = self.config.notiflag_data(self.confcust) self.SERVICE_ENDPOINTS_PI, self.SERVICE_GROUPS_PI, self.SITES_PI = self.config.service_data(self.confcust) @@ -438,8 +441,10 @@ async def run(self): ) if eval(self.globopts['GeneralWriteJson'.lower()]): - write_json(self.logger, self.globopts, self.confcust, - group_groups, group_endpoints, self.fixed_date) + #write_json(self.logger, self.globopts, self.confcust, + # group_groups, group_endpoints, self.fixed_date) + write_json(group_groups, group_endpoints) + self.logger.info('Customer:' + self.custname + ' Type:%s ' % (','.join( self.topofetchtype)) + 'Fetched Endpoints:%d' % (numge) + ' Groups:%d' % (numgg)) diff --git a/modules/tasks/vapor_weights.py b/modules/tasks/vapor_weights.py index 7bea877a..5080d273 100644 --- a/modules/tasks/vapor_weights.py +++ b/modules/tasks/vapor_weights.py @@ -3,7 +3,7 @@ from urllib.parse import urlparse import asyncio -from argo_connectors.singleton_config import ConfigClass#, EventLoopSingleton +from argo_connectors.singleton_config import ConfigClass from argo_connectors.io.http import SessionWithRetry from argo_connectors.io.webapi import WebAPI from argo_connectors.parse.vapor import ParseWeights @@ -27,16 +27,17 @@ class TaskVaporWeights(object): def __init__(self, jobcust): self.jobcust = jobcust - self.config = ConfigClass() + self.args = self.config.parse_args() self.loop = self.config.get_loop() - asyncio.set_event_loop(self.loop) + asyncio.set_event_loop(self.loop) self.logger = self.config.get_logger() self.connector_name = self.config.get_connector_name() - self.globopts, self.pass_extensions, self.cglob = self.config.get_globopts_n_pass_ext() - self.confcust = self.config.get_confcust(self.globopts) + self.cglob = self.config.get_cglob(self.args) + self.globopts = self.config.get_globopts(self.cglob) + self.confcust = self.config.get_confcust(self.globopts, self.args) self.feed = self.config.vaporrpi_data(self.confcust) - self.fixed_date = self.config.get_fixed_date() + self.fixed_date = self.config.get_fixed_date(self.args) async def fetch_data(self): From bc4885cd1645544260776c5b20ccb9c44c9e406a Mon Sep 17 00:00:00 2001 From: Dean Hudek Date: Mon, 5 Jun 2023 14:30:53 +0200 Subject: [PATCH 10/11] args as method to metricprofile --- exec/downtimes-gocdb-connector.py | 47 -------------------------- exec/metricprofile-webapi-connector.py | 5 +-- exec/service-types-gocdb-connector.py | 8 +++-- modules/tasks/common.py | 7 ++-- modules/tasks/gocdb_servicetypes.py | 14 ++++---- modules/tasks/webapi_metricprofile.py | 27 ++++----------- 6 files changed, 27 insertions(+), 81 deletions(-) diff --git a/exec/downtimes-gocdb-connector.py b/exec/downtimes-gocdb-connector.py index 9a8c8b79..3f6eb95a 100755 --- a/exec/downtimes-gocdb-connector.py +++ b/exec/downtimes-gocdb-connector.py @@ -43,31 +43,6 @@ def main(): help='path to global configuration file', type=str, required=False) args = parser.parse_args() - # logger = Logger(os.path.basename(sys.argv[0])) - # confpath = args.gloconf[0] if args.gloconf else None - # cglob = Global(sys.argv[0], confpath) - # globopts = cglob.parse() - - # confpath = args.custconf[0] if args.custconf else None - # confcust = CustomerConf(sys.argv[0], confpath) - # confcust.parse() - # confcust.make_dirstruct() - # confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) - - # logger.customer = confcust.get_custname() - - # auth_custopts = confcust.get_authopts() - # auth_opts = cglob.merge_opts(auth_custopts, 'authentication') - # auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') - # if not auth_complete: - # logger.error('%s options incomplete, missing %s' % - # ('authentication', ' '.join(missing))) - # raise SystemExit(1) - - # if len(args.date) == 0: - # raise SystemExit(1) - - #calculate start and end times try: start = datetime.datetime.strptime(args.date[0], '%Y-%m-%d') end = datetime.datetime.strptime(args.date[0], '%Y-%m-%d') @@ -79,17 +54,6 @@ def main(): logger.error(exc) raise SystemExit(1) - # downtime_feed = confcust.get_downfeed() - - # uidservtype = confcust.get_uidserviceendpoints() - # webapi_opts = get_webapi_opts(cglob, confcust) - - # loop = uvloop.new_event_loop() - # asyncio.set_event_loop(loop) - - - ########################################################################## - config = ConfigClass(args) globopts, _, _ = config.get_globopts_n_pass_ext() @@ -99,23 +63,12 @@ def main(): loop = config.get_loop() asyncio.set_event_loop(loop) - ########################################################################## - try: cust = list(confcust.get_customers())[0] cust = confcust.get_custname(cust) - # task = TaskGocdbDowntimes(loop, logger, sys.argv[0], globopts, - # auth_opts, webapi_opts, confcust, - # confcust.get_custname(cust), downtime_feed, start, - # end, uidservtype, args.date[0], timestamp) - - ################################################################################### - task = TaskGocdbDowntimes(cust, start, end, timestamp) - ################################################################################### - loop.run_until_complete(task.run()) except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc: diff --git a/exec/metricprofile-webapi-connector.py b/exec/metricprofile-webapi-connector.py index f3997dfd..caa4245e 100755 --- a/exec/metricprofile-webapi-connector.py +++ b/exec/metricprofile-webapi-connector.py @@ -51,8 +51,9 @@ def main(): config = ConfigClass(args) - globopts, _, _ = config.get_globopts_n_pass_ext() - confcust = config.get_confcust(globopts) + cglob = config.get_cglob(args) + globopts = config.get_globopts(cglob) + confcust = config.get_confcust(globopts, args) logger = config.get_logger() loop = config.get_loop() diff --git a/exec/service-types-gocdb-connector.py b/exec/service-types-gocdb-connector.py index 7f1ccd3e..f9183207 100755 --- a/exec/service-types-gocdb-connector.py +++ b/exec/service-types-gocdb-connector.py @@ -74,10 +74,12 @@ def main(): config = ConfigClass(args) - globopts, _, _ = config.get_globopts_n_pass_ext() - confcust = config.get_confcust(globopts) + args = config.parse_args() + cglob = config.get_cglob(args) + globopts = config.get_globopts(cglob) + confcust = config.get_confcust(globopts, args) logger = config.get_logger() - fixed_date = config.get_fixed_date() + fixed_date = config.get_fixed_date(args) loop = config.get_loop() asyncio.set_event_loop(loop) diff --git a/modules/tasks/common.py b/modules/tasks/common.py index 501190af..38c22409 100644 --- a/modules/tasks/common.py +++ b/modules/tasks/common.py @@ -39,6 +39,8 @@ async def write_weights_metricprofile_state(connector_name, globopts, cust, job, globopts['InputStateDays'.lower()]) +################################################################################################################## + def write_metricprofile_json(logger, globopts, cust, job, confcust, fixed_date, fetched_profiles): jobdir = confcust.get_fulldir(cust, job) if fixed_date: @@ -47,19 +49,20 @@ def write_metricprofile_json(logger, globopts, cust, job, confcust, fixed_date, else: filename = filename_date( logger, globopts['OutputMetricProfile'.lower()], jobdir) - json_writer = JsonWriter(fetched_profiles, filename, globopts['generalcompressjson']) + json_writer = JsonWriter(fetched_profiles, filename)#, globopts['generalcompressjson']) ret, excep = json_writer.write_json() if not ret: logger.error('Customer:%s Job:%s %s' % (logger.customer, logger.job, repr(excep))) raise SystemExit(1) +################################################################################################################## def write_downtimes_json(logger, globopts, confcust, dts, timestamp): custdir = confcust.get_custdir() filename = filename_date( logger, globopts['OutputDowntimes'.lower()], custdir, stamp=timestamp) - json_writer = JsonWriter(dts, filename, globopts['generalcompressjson']) + json_writer = JsonWriter(dts, filename)#, globopts['generalcompressjson']) ret, excep = json_writer.write_json() if not ret: logger.error('Customer:{} {}'.format(logger.customer, repr(excep))) diff --git a/modules/tasks/gocdb_servicetypes.py b/modules/tasks/gocdb_servicetypes.py index 0ad2b8b0..53133ff8 100644 --- a/modules/tasks/gocdb_servicetypes.py +++ b/modules/tasks/gocdb_servicetypes.py @@ -39,18 +39,20 @@ class TaskGocdbServiceTypes(object): def __init__(self): self.config = ConfigClass() + self.args = self.config.parse_args() self.loop = self.config.get_loop() asyncio.set_event_loop(self.loop) self.logger = self.config.get_logger() self.connector_name = self.config.get_connector_name() - self.globopts, _, _ = self.config.get_globopts_n_pass_ext() - self.confcust = self.config.get_confcust(self.globopts) - self.auth_opts = self.config.get_auth_opts(self.confcust, self.logger) + self.cglob = self.config.get_cglob(self.args) + self.globopts= self.config.get_globopts(self.cglob) + self.confcust = self.config.get_confcust(self.globopts, self.args) + self.auth_opts = self.config.get_auth_opts(self.confcust, self.cglob, self.logger) self.custname = self.config.custname_data(self.confcust) - self.webapi_opts = self.config.get_webapi_opts_data(self.confcust, self.custname) + self.webapi_opts = self.config.get_webapi_opts_data(self.confcust, self.cglob, self.custname) self.feed = self.config.get_feed(self.confcust) - self.timestamp = self.config.get_fixed_date() - self.initsync = self.config.get_initsync() + self.timestamp = self.config.get_fixed_date(self.args) + self.initsync = self.config.get_initsync(self.args) diff --git a/modules/tasks/webapi_metricprofile.py b/modules/tasks/webapi_metricprofile.py index 9fbc62f2..86f77df1 100644 --- a/modules/tasks/webapi_metricprofile.py +++ b/modules/tasks/webapi_metricprofile.py @@ -11,34 +11,19 @@ class TaskWebApiMetricProfile(object): - # def __init__(self, loop, logger, connector_name, globopts, cglob, confcust, - # cust, fixed_date): - # self.loop = loop - # self.logger = logger - # self.connector_name = connector_name - # self.globopts = globopts - # self.cust = cust - # self.confcust = confcust - # self.cglob = cglob - # self.fixed_date = fixed_date - - #################################################################################### - def __init__(self, cust): + self.config = ConfigClass() self.cust = cust - + self.args = self.config.parse_args() self.config = ConfigClass() self.loop = self.config.get_loop() asyncio.set_event_loop(self.loop) self.logger = self.config.get_logger() self.connector_name = self.config.get_connector_name() - self.globopts, _, self.cglob = self.config.get_globopts_n_pass_ext() - self.confcust = self.config.get_confcust(self.globopts) - self.fixed_date = self.config.get_fixed_date() - - - #################################################################################### - + self.cglob = self.config.get_cglob(self.args) + self.globopts= self.config.get_globopts(self.cglob) + self.confcust = self.config.get_confcust(self.globopts, self.args) + self.fixed_date = self.config.get_fixed_date(self.args) async def fetch_data(self, host, token): session = SessionWithRetry(self.logger, From 2608ddd6f7d7b7220a021ff462ca041dbb94d2fe Mon Sep 17 00:00:00 2001 From: Dean Hudek Date: Mon, 5 Jun 2023 15:15:26 +0200 Subject: [PATCH 11/11] args as method to downtimes-gocdb and service-types-gocdb --- exec/downtimes-gocdb-connector.py | 7 ++++--- exec/service-types-gocdb-connector.py | 17 ++++++++--------- modules/singleton_config.py | 5 ++++- modules/tasks/gocdb_downtimes.py | 14 ++++++++------ 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/exec/downtimes-gocdb-connector.py b/exec/downtimes-gocdb-connector.py index 3f6eb95a..d0a0a0c7 100755 --- a/exec/downtimes-gocdb-connector.py +++ b/exec/downtimes-gocdb-connector.py @@ -55,9 +55,10 @@ def main(): raise SystemExit(1) config = ConfigClass(args) - - globopts, _, _ = config.get_globopts_n_pass_ext() - confcust = config.get_confcust(globopts) + args = config.parse_args() + cglob = config.get_cglob(args) + globopts = config.get_globopts(cglob) + confcust = config.get_confcust(globopts, args) logger = config.get_logger() loop = config.get_loop() diff --git a/exec/service-types-gocdb-connector.py b/exec/service-types-gocdb-connector.py index f9183207..1902f961 100755 --- a/exec/service-types-gocdb-connector.py +++ b/exec/service-types-gocdb-connector.py @@ -32,13 +32,13 @@ def get_webapi_opts(cglob, confcust): def main(): - global logger, globopts - parser = argparse.ArgumentParser(description='Fetch service types from GOCDB') - parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False) - parser.add_argument('--initial', dest='initsync', help='initial sync of service types', action='store_true', default=False, required=False) - parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) - parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) - args = parser.parse_args() + # global logger, globopts + # parser = argparse.ArgumentParser(description='Fetch service types from GOCDB') + # parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False) + # parser.add_argument('--initial', dest='initsync', help='initial sync of service types', action='store_true', default=False, required=False) + # parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) + # parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) + #args = parser.parse_args() # fixed_date = None # if args.date and date_check(args.date): @@ -72,8 +72,7 @@ def main(): ########################################################################## - config = ConfigClass(args) - + config = ConfigClass() args = config.parse_args() cglob = config.get_cglob(args) globopts = config.get_globopts(cglob) diff --git a/modules/singleton_config.py b/modules/singleton_config.py index 44b88ae9..5ce65cad 100755 --- a/modules/singleton_config.py +++ b/modules/singleton_config.py @@ -175,7 +175,10 @@ def get_feed(self, confcust): return feed def get_initsync(self, args): - return args.initsync + if hasattr(args, 'initsync') and args.initsync: + return True + else: + return False def get_downtime_feed(self, confcust): return confcust.get_downfeed() diff --git a/modules/tasks/gocdb_downtimes.py b/modules/tasks/gocdb_downtimes.py index 6819ef56..9e9ca7d9 100644 --- a/modules/tasks/gocdb_downtimes.py +++ b/modules/tasks/gocdb_downtimes.py @@ -33,24 +33,26 @@ class TaskGocdbDowntimes(object): ################################################################################### def __init__(self, custname, start, end, timestamp): + self.config = ConfigClass() self.custname = custname self.start = start self.end = end self.timestamp = timestamp - + self.args = self.config.parse_args() self.config = ConfigClass() self.loop = self.config.get_loop() asyncio.set_event_loop(self.loop) self.logger = self.config.get_logger() self.connector_name = self.config.get_connector_name() - self.globopts, self.pass_extensions, self.cglob = self.config.get_globopts_n_pass_ext() - self.confcust = self.config.get_confcust(self.globopts) - self.auth_opts = self.config.get_auth_opts(self.confcust, self.logger) + self.cglob = self.config.get_cglob(self.args) + self.globopts= self.config.get_globopts(self.cglob) + self.confcust = self.config.get_confcust(self.globopts, self.args) + self.auth_opts = self.config.get_auth_opts(self.confcust, self.cglob, self.logger) self.custname = self.config.custname_data(self.confcust) - self.webapi_opts = self.config.get_webapi_opts_data(self.confcust, self.custname) + self.webapi_opts = self.config.get_webapi_opts_data(self.confcust, self.cglob, self.custname) self.feed = self.config.get_downtime_feed(self.confcust) self.uidservtype = self.config.uidservendp_data(self.confcust) - self.targetdate = self.config.get_target_date() + self.targetdate = self.config.get_target_date(self.args) async def fetch_data(self): feed_parts = urlparse(self.feed)