ARGO-3525 Singleton with global configuration options

argo-connectors.spec

@@ -20,6 +20,7 @@ Requires: python3-requests
 Requires: python3-typing-extensions
 Requires: python3-uvloop
 Requires: python3-bonsai
+Requires: python3-unidecode
 Requires: python36-lxml
 
 BuildRequires: python3-devel python3-setuptools

exec/downtimes-csv-connector.py

@@ -8,6 +8,7 @@
 import asyncio
 import uvloop
 
+from argo_connectors.singleton_config import ConfigClass
 from argo_connectors.exceptions import ConnectorHttpError, ConnectorParseError
 from argo_connectors.log import Logger
 from argo_connectors.tasks.flat_downtimes import TaskCsvDowntimes
@@ -19,14 +20,14 @@
 globopts = {}
 
 
-def get_webapi_opts(cglob, confcust):
-    webapi_custopts = confcust.get_webapiopts()
-    webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi')
-    webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi')
-    if not webapi_complete:
-        logger.error('Customer:%s %s options incomplete, missing %s' % (logger.customer, 'webapi', ' '.join(missopt)))
-        raise SystemExit(1)
-    return webapi_opts
+# def get_webapi_opts(cglob, confcust):
+#     webapi_custopts = confcust.get_webapiopts()
+#     webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi')
+#     webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi')
+#     if not webapi_complete:
+#         logger.error('Customer:%s %s options incomplete, missing %s' % (logger.customer, 'webapi', ' '.join(missopt)))
+#         raise SystemExit(1)
+#     return webapi_opts
 
 
 def main():
@@ -37,22 +38,22 @@ def main():
     parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False)
     args = parser.parse_args()
 
-    logger = Logger(os.path.basename(sys.argv[0]))
-    confpath = args.gloconf[0] if args.gloconf else None
-    cglob = Global(sys.argv[0], confpath)
-    globopts = cglob.parse()
-
-    confpath = args.custconf[0] if args.custconf else None
-    confcust = CustomerConf(sys.argv[0], confpath)
-    confcust.parse()
-    confcust.make_dirstruct()
-    confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
-    feed = confcust.get_downfeed()
-    logger.customer = confcust.get_custname()
-
-    if len(args.date) == 0:
-        print(parser.print_help())
-        raise SystemExit(1)
+    # logger = Logger(os.path.basename(sys.argv[0]))
+    # confpath = args.gloconf[0] if args.gloconf else None
+    # cglob = Global(sys.argv[0], confpath)
+    # globopts = cglob.parse()
+
+    # confpath = args.custconf[0] if args.custconf else None
+    # confcust = CustomerConf(sys.argv[0], confpath)
+    # confcust.parse()
+    # confcust.make_dirstruct()
+    # confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
+    # feed = confcust.get_downfeed()
+    # logger.customer = confcust.get_custname()
+
+    # if len(args.date) == 0:
+    #     print(parser.print_help())
+    #     raise SystemExit(1)
 
     # calculate start and end times
     try:
@@ -64,21 +65,47 @@ def main():
         logger.error(exc)
         raise SystemExit(1)
 
-    uidservtype = confcust.get_uidserviceendpoints()
+    # uidservtype = confcust.get_uidserviceendpoints()
 
-    webapi_opts = get_webapi_opts(cglob, confcust)
+    # webapi_opts = get_webapi_opts(cglob, confcust)
 
-    loop = uvloop.new_event_loop()
+    # loop = uvloop.new_event_loop()
+    # asyncio.set_event_loop(loop)
+
+    ###################################################################################
+
+    config = ConfigClass(args)
+
+    globopts, _, _ = config.get_globopts_n_pass_ext()
+    confcust = config.get_confcust(globopts)
+    logger = config.get_logger()
+
+    loop = config.get_loop()
     asyncio.set_event_loop(loop)
 
+
+    ###################################################################################
+
     try:
         cust = list(confcust.get_customers())[0]
-        task = TaskCsvDowntimes(loop, logger, sys.argv[0], globopts,
-                                webapi_opts, confcust,
-                                confcust.get_custname(cust), feed,
-                                current_date, uidservtype, args.date[0],
-                                timestamp)
+        cust = confcust.get_custname(cust)
+
+
+        # task = TaskCsvDowntimes(loop, logger, sys.argv[0], globopts,
+        #                         webapi_opts, confcust,
+        #                         confcust.get_custname(cust), feed,
+        #                         current_date, uidservtype, args.date[0],
+        #                         timestamp)
+
+        #########################################################################
+
+        task = TaskCsvDowntimes(cust, current_date, timestamp)
+
+        #########################################################################
+
+
         loop.run_until_complete(task.run())
+
 
     except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc:
         logger.error(repr(exc))

exec/downtimes-gocdb-connector.py

@@ -8,6 +8,7 @@
 import asyncio
 import uvloop
 
+from argo_connectors.singleton_config import ConfigClass
 from argo_connectors.exceptions import ConnectorHttpError, ConnectorParseError
 from argo_connectors.log import Logger
 from argo_connectors.tasks.gocdb_downtimes import TaskGocdbDowntimes
@@ -42,31 +43,6 @@ def main():
                         help='path to global configuration file', type=str, required=False)
     args = parser.parse_args()
 
-    logger = Logger(os.path.basename(sys.argv[0]))
-    confpath = args.gloconf[0] if args.gloconf else None
-    cglob = Global(sys.argv[0], confpath)
-    globopts = cglob.parse()
-
-    confpath = args.custconf[0] if args.custconf else None
-    confcust = CustomerConf(sys.argv[0], confpath)
-    confcust.parse()
-    confcust.make_dirstruct()
-    confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
-
-    logger.customer = confcust.get_custname()
-
-    auth_custopts = confcust.get_authopts()
-    auth_opts = cglob.merge_opts(auth_custopts, 'authentication')
-    auth_complete, missing = cglob.is_complete(auth_opts, 'authentication')
-    if not auth_complete:
-        logger.error('%s options incomplete, missing %s' %
-                     ('authentication', ' '.join(missing)))
-        raise SystemExit(1)
-
-    if len(args.date) == 0:
-        raise SystemExit(1)
-
-    # calculate start and end times
     try:
         start = datetime.datetime.strptime(args.date[0], '%Y-%m-%d')
         end = datetime.datetime.strptime(args.date[0], '%Y-%m-%d')
@@ -78,20 +54,22 @@ def main():
         logger.error(exc)
         raise SystemExit(1)
 
-    downtime_feed = confcust.get_downfeed()
+    config = ConfigClass(args)
+    args = config.parse_args()
+    cglob = config.get_cglob(args)
+    globopts = config.get_globopts(cglob)
+    confcust = config.get_confcust(globopts, args)
+    logger = config.get_logger()
 
-    uidservtype = confcust.get_uidserviceendpoints()
-    webapi_opts = get_webapi_opts(cglob, confcust)
-
-    loop = uvloop.new_event_loop()
+    loop = config.get_loop()
     asyncio.set_event_loop(loop)
 
     try:
         cust = list(confcust.get_customers())[0]
-        task = TaskGocdbDowntimes(loop, logger, sys.argv[0], globopts,
-                                  auth_opts, webapi_opts, confcust,
-                                  confcust.get_custname(cust), downtime_feed, start,
-                                  end, uidservtype, args.date[0], timestamp)
+        cust = confcust.get_custname(cust)
+
+        task = TaskGocdbDowntimes(cust, start, end, timestamp)
+
         loop.run_until_complete(task.run())
 
     except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc:

exec/metricprofile-webapi-connector.py

@@ -7,6 +7,7 @@
 import asyncio
 import uvloop
 
+from argo_connectors.singleton_config import ConfigClass
 from argo_connectors.config import CustomerConf, Global
 from argo_connectors.log import Logger
 from argo_connectors.tasks.webapi_metricprofile import TaskWebApiMetricProfile
@@ -26,30 +27,56 @@ def main():
     parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False)
     args = parser.parse_args()
 
-    logger = Logger(os.path.basename(sys.argv[0]))
+    # logger = Logger(os.path.basename(sys.argv[0]))
 
-    fixed_date = None
-    if args.date and date_check(args.date):
-        fixed_date = args.date
+    # fixed_date = None
+    # if args.date and date_check(args.date):
+    #     fixed_date = args.date
 
-    confpath = args.gloconf[0] if args.gloconf else None
-    cglob = Global(sys.argv[0], confpath)
-    globopts = cglob.parse()
+    # confpath = args.gloconf[0] if args.gloconf else None
+    # cglob = Global(sys.argv[0], confpath)
+    # globopts = cglob.parse()
 
-    confpath = args.custconf[0] if args.custconf else None
-    confcust = CustomerConf(sys.argv[0], confpath)
-    confcust.parse()
-    confcust.make_dirstruct()
-    confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
+    # confpath = args.custconf[0] if args.custconf else None
+    # confcust = CustomerConf(sys.argv[0], confpath)
+    # confcust.parse()
+    # confcust.make_dirstruct()
+    # confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
 
-    loop = uvloop.new_event_loop()
+    # loop = uvloop.new_event_loop()
+    # asyncio.set_event_loop(loop)
+
+
+    ############################################################################
+
+    config = ConfigClass(args)
+
+    cglob = config.get_cglob(args)
+    globopts = config.get_globopts(cglob)
+    confcust = config.get_confcust(globopts, args)
+    logger = config.get_logger()
+
+    loop = config.get_loop()
     asyncio.set_event_loop(loop)
 
+    ############################################################################
+
+
+
     for cust in confcust.get_customers():
         try:
-            task = TaskWebApiMetricProfile(
-                loop, logger, sys.argv[0], globopts, cglob, confcust, cust, fixed_date
-            )
+            # task = TaskWebApiMetricProfile(
+            #     loop, logger, sys.argv[0], globopts, cglob, confcust, cust, fixed_date
+            # )
+
+            ###########################################################################
+
+            task = TaskWebApiMetricProfile(cust)
+
+            ###########################################################################
+
+
+
             loop.run_until_complete(task.run())
 
         except (KeyboardInterrupt) as exc:

exec/service-types-csv-connector.py

@@ -8,6 +8,7 @@
 import asyncio
 import uvloop
 
+from argo_connectors.singleton_config import ConfigClass
 from argo_connectors.exceptions import ConnectorHttpError, ConnectorParseError
 from argo_connectors.log import Logger
 from argo_connectors.tasks.flat_servicetypes import TaskFlatServiceTypes
@@ -39,42 +40,68 @@ def main():
     parser.add_argument('--initial', dest='initsync', help='initial sync of service types', action='store_true', default=False, required=False)
     args = parser.parse_args()
 
-    fixed_date = None
-    if args.date and date_check(args.date):
-        fixed_date = args.date
-
-    logger = Logger(os.path.basename(sys.argv[0]))
-    confpath = args.gloconf[0] if args.gloconf else None
-    cglob = Global(sys.argv[0], confpath)
-    globopts = cglob.parse()
-
-    confpath = args.custconf[0] if args.custconf else None
-    confcust = CustomerConf(sys.argv[0], confpath)
-    confcust.parse()
-    confcust.make_dirstruct()
-    confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
-    feed = confcust.get_servicesfeed()
-    custname = confcust.get_custname()
-    logger.customer = confcust.get_custname()
-
-    auth_custopts = confcust.get_authopts()
-    auth_opts = cglob.merge_opts(auth_custopts, 'authentication')
-    auth_complete, missing = cglob.is_complete(auth_opts, 'authentication')
-    if not auth_complete:
-        logger.error('%s options incomplete, missing %s' % ('authentication', ' '.join(missing)))
-        raise SystemExit(1)
+    # fixed_date = None
+    # if args.date and date_check(args.date):
+    #     fixed_date = args.date
+
+    # logger = Logger(os.path.basename(sys.argv[0]))
+    # confpath = args.gloconf[0] if args.gloconf else None
+    # cglob = Global(sys.argv[0], confpath)
+    # globopts = cglob.parse()
+
+    # confpath = args.custconf[0] if args.custconf else None
+    # confcust = CustomerConf(sys.argv[0], confpath)
+    # confcust.parse()
+    # confcust.make_dirstruct()
+    # confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
+    # feed = confcust.get_servicesfeed()
+    # custname = confcust.get_custname()
+    # logger.customer = confcust.get_custname()
+
+    # auth_custopts = confcust.get_authopts()
+    # auth_opts = cglob.merge_opts(auth_custopts, 'authentication')
+    # auth_complete, missing = cglob.is_complete(auth_opts, 'authentication')
+    # if not auth_complete:
+    #     logger.error('%s options incomplete, missing %s' % ('authentication', ' '.join(missing)))
+    #     raise SystemExit(1)
+
+    # webapi_opts = get_webapi_opts(cglob, confcust)
+
+    # loop = uvloop.new_event_loop()
+    # asyncio.set_event_loop(loop)
+
+    #######################################################################
+
+    config = ConfigClass(args)
+
+    globopts, _, _ = config.get_globopts_n_pass_ext()
+    confcust = config.get_confcust(globopts)
+    logger = config.get_logger()
+    fixed_date = config.get_fixed_date()
+
+    loop = config.get_loop()
+    asyncio.set_event_loop(loop)
+
+
+    #######################################################################
+
 
-    webapi_opts = get_webapi_opts(cglob, confcust)
 
-    loop = uvloop.new_event_loop()
-    asyncio.set_event_loop(loop)
 
     try:
-        task = TaskFlatServiceTypes(
-            loop, logger, sys.argv[0], globopts, auth_opts, webapi_opts,
-            confcust, custname, feed, fixed_date, is_csv=True,
-            initsync=args.initsync
-        )
+        # task = TaskFlatServiceTypes(
+        #     loop, logger, sys.argv[0], globopts, auth_opts, webapi_opts,
+        #     confcust, custname, feed, fixed_date, is_csv=True,
+        #     initsync=args.initsync
+        # )
+
+        #######################################################################
+
+        task = TaskFlatServiceTypes(initsync=args.initsync)
+
+        #######################################################################
+
+
         loop.run_until_complete(task.run())
 
     except (KeyboardInterrupt) as exc: