From 91d0003f592201c110fcb2a7a119c6baf90c7928 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Wed, 6 Jul 2022 12:22:45 +0200 Subject: [PATCH 01/26] add sample downtime csv --- tests/sample-downtimes.csv | 42 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 tests/sample-downtimes.csv diff --git a/tests/sample-downtimes.csv b/tests/sample-downtimes.csv new file mode 100644 index 00000000..78d6a886 --- /dev/null +++ b/tests/sample-downtimes.csv @@ -0,0 +1,42 @@ +unique_id,url,start_time,end_time,Severity,Description +neanias_5,https://atmo-flud.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_7,https://atmo-stress.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_9,https://caesar.neanias.eu:443,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_10,http://caesar-api.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_14,https://bathyprocessing.neanias.eu/hub/login,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_15,https://uw-map.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_16,https://uw-map.neanias.eu/tos,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_17,https://uw-map.neanias.eu/privacy,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_18,https://uw-mos.neanias.eu/api/try,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_20,https://ai-gateway.neanias.eu/hub/login,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_21,https://accounting.neanias.eu/api/api/accounting-service/version-info/current,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_25,https://logging.neanias.eu/api/status,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_26,https://minio.ml-models.neanias.eu/minio/health/live,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_27,https://notification.neanias.eu/api/notification/version-info/current,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_28,https://pms.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_30,https://vis-gateway.neanias.eu/hub/login,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +,,,,, +neanias_5,https://atmo-flud.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_7,https://atmo-stress.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_9,https://caesar.neanias.eu:443,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_10,http://caesar-api.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_14,https://bathyprocessing.neanias.eu/hub/login,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_15,https://uw-map.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_16,https://uw-map.neanias.eu/tos,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_17,https://uw-map.neanias.eu/privacy,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_18,https://uw-mos.neanias.eu/api/try,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_20,https://ai-gateway.neanias.eu/hub/login,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_21,https://accounting.neanias.eu/api/api/accounting-service/version-info/current,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_25,https://logging.neanias.eu/api/status,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_26,https://minio.ml-models.neanias.eu/minio/health/live,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_27,https://notification.neanias.eu/api/notification/version-info/current,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_28,https://pms.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_30,https://vis-gateway.neanias.eu/hub/login,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_2,https://files.neanias.eu,2/28/2022 13:00,2/28/2022 15:00,OUTAGE,Data sharing service maintenance +neanias_1,https://files.dev.neanias.eu,2/28/2022 9:30,2/28/2022 11:00,OUTAGE,Data sharing service maintenance +neanias_1,https://files.dev.neanias.eu,2/7/2022 8:00,2/17/2022 19:00,OUTAGE,GARR Cloud garr-pa1 region maintenance +neanias_2,https://files.neanias.eu,2/7/2022 8:00,2/17/2022 19:00,OUTAGE,GARR Cloud garr-pa1 region maintenance +neanias_4,https://atmo-4cast.neanias.eu/api/forecaststatus/1,2/7/2022 8:00,2/17/2022 19:00,OUTAGE,GARR Cloud garr-pa1 region maintenance +neanias_6,http://atmo-seism.neanias.eu/hub/login,2/7/2022 8:00,2/17/2022 19:00,OUTAGE,GARR Cloud garr-pa1 region maintenance +neanias_11,http://vlkb.neanias.eu:8080/vlkb/tap/availability,2/7/2022 8:00,2/17/2022 19:00,OUTAGE,GARR Cloud garr-pa1 region maintenance +neanias_29,https://vd-maps.neanias.eu,2/7/2022 8:00,2/17/2022 19:00,OUTAGE,GARR Cloud garr-pa1 region maintenance \ No newline at end of file From 8b97bad6ef3fb6bb8a1b410312ac17420df59afd Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Wed, 6 Jul 2022 13:07:04 +0200 Subject: [PATCH 02/26] add initial exec, parse and task for downtimes csv --- exec/downtimes-csv-connector.py | 94 +++++++++++++++++++++++++++++++++ modules/parse/flat_downtimes.py | 12 +++++ modules/tasks/flat_downtimes.py | 76 ++++++++++++++++++++++++++ 3 files changed, 182 insertions(+) create mode 100755 exec/downtimes-csv-connector.py create mode 100644 modules/parse/flat_downtimes.py create mode 100644 modules/tasks/flat_downtimes.py diff --git a/exec/downtimes-csv-connector.py b/exec/downtimes-csv-connector.py new file mode 100755 index 00000000..4441def8 --- /dev/null +++ b/exec/downtimes-csv-connector.py @@ -0,0 +1,94 @@ +#!/usr/bin/python3 + +import argparse +import datetime +import os +import sys + +import asyncio +import uvloop + +from argo_connectors.exceptions import ConnectorHttpError, ConnectorParseError +from argo_connectors.log import Logger +from argo_connectors.tasks.gocdb_downtimes import TaskGocdbDowntimes +from argo_connectors.tasks.common import write_state + +from argo_connectors.config import Global, CustomerConf + +logger = None +globopts = {} + +DOWNTIMEPI = '/gocdbpi/private/?method=get_downtime' + + +def get_webapi_opts(cglob, confcust): + webapi_custopts = confcust.get_webapiopts() + webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') + webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') + if not webapi_complete: + logger.error('Customer:%s %s options incomplete, missing %s' % (logger.customer, 'webapi', ' '.join(missopt))) + raise SystemExit(1) + return webapi_opts + + +def main(): + global logger, globopts + parser = argparse.ArgumentParser(description='Fetch downtimes from CSV for given date') + parser.add_argument('-d', dest='date', nargs=1, metavar='YEAR-MONTH-DAY', required=True) + parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False) + parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) + args = parser.parse_args() + + logger = Logger(os.path.basename(sys.argv[0])) + confpath = args.gloconf[0] if args.gloconf else None + cglob = Global(sys.argv[0], confpath) + globopts = cglob.parse() + + confpath = args.custconf[0] if args.custconf else None + confcust = CustomerConf(sys.argv[0], confpath) + confcust.parse() + confcust.make_dirstruct() + confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) + feed = confcust.get_topofeed() + logger.customer = confcust.get_custname() + + if len(args.date) == 0: + print(parser.print_help()) + raise SystemExit(1) + + # calculate start and end times + try: + start = datetime.datetime.strptime(args.date[0], '%Y-%m-%d') + end = datetime.datetime.strptime(args.date[0], '%Y-%m-%d') + timestamp = start.strftime('%Y_%m_%d') + start = start.replace(hour=0, minute=0, second=0) + end = end.replace(hour=23, minute=59, second=59) + + except ValueError as exc: + logger.error(exc) + raise SystemExit(1) + + uidservtype = confcust.get_uidserviceendpoints() + + webapi_opts = get_webapi_opts(cglob, confcust) + + loop = uvloop.new_event_loop() + asyncio.set_event_loop(loop) + + try: + cust = list(confcust.get_customers())[0] + task = TaskGocdbDowntimes(loop, logger, sys.argv[0], globopts, + webapi_opts, confcust, confcust.get_custname(cust), feed, + DOWNTIMEPI, start, end, uidservtype, args.date[0], timestamp) + loop.run_until_complete(task.run()) + + except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc: + logger.error(repr(exc)) + loop.run_until_complete( + write_state(sys.argv[0], globopts, confcust, timestamp, False) + ) + + loop.close() + +if __name__ == '__main__': + main() diff --git a/modules/parse/flat_downtimes.py b/modules/parse/flat_downtimes.py new file mode 100644 index 00000000..f37cab99 --- /dev/null +++ b/modules/parse/flat_downtimes.py @@ -0,0 +1,12 @@ +import datetime +import xml.dom.minidom + +from xml.parsers.expat import ExpatError +from argo_connectors.utils import module_class_name +from argo_connectors.exceptions import ConnectorParseError +from argo_connectors.parse.base import ParseHelpers + + +class ParseDowntimes(ParseHelpers): + def __init__(self, logger, data, start, end, uid=False): + pass diff --git a/modules/tasks/flat_downtimes.py b/modules/tasks/flat_downtimes.py new file mode 100644 index 00000000..a7352cdd --- /dev/null +++ b/modules/tasks/flat_downtimes.py @@ -0,0 +1,76 @@ +import os + +from urllib.parse import urlparse + +from argo_connectors.io.http import SessionWithRetry +from argo_connectors.parse.flat_downtimes import ParseDowntimes +from argo_connectors.io.webapi import WebAPI +from argo_connectors.tasks.common import write_state, write_downtimes_avro as write_avro + + +class TaskCsvDowntimes(object): + def __init__(self, loop, logger, connector_name, globopts, webapi_opts, + confcust, custname, feed, DOWNTIMEPI, start, end, + uidservtype, targetdate, timestamp): + self.event_loop = loop + self.logger = logger + self.connector_name = connector_name + self.globopts = globopts + self.webapi_opts = webapi_opts + self.confcust = confcust + self.custname = custname + self.feed = feed + self.DOWNTIMEPI = DOWNTIMEPI + self.start = start + self.end = end + self.uidservtype = uidservtype + self.targetdate = targetdate + self.timestamp = timestamp + + async def fetch_data(self): + feed_parts = urlparse(self.feed) + start_fmt = self.start.strftime("%Y-%m-%d") + end_fmt = self.end.strftime("%Y-%m-%d") + session = SessionWithRetry(self.logger, + os.path.basename(self.connector_name), + self.globopts) + res = await session.http_get() + + return res + + def parse_source(self, res): + csv_downtimes = ParseDowntimes(self.logger, res, self.start, self.end, + self.uidservtype) + return csv_downtimes.get_data() + + async def send_webapi(self, dts): + webapi = WebAPI(self.connector_name, self.webapi_opts['webapihost'], + self.webapi_opts['webapitoken'], self.logger, + int(self.globopts['ConnectionRetry'.lower()]), + int(self.globopts['ConnectionTimeout'.lower()]), + int(self.globopts['ConnectionSleepRetry'.lower()]), + date=self.targetdate) + await webapi.send(dts, downtimes_component=True) + + async def run(self): + # we don't have multiple tenant definitions in one + # customer file so we can safely assume one tenant/customer + write_empty = self.confcust.send_empty(self.connector_name) + if not write_empty: + res = await self.fetch_data() + dts = self.parse_source(res) + else: + dts = [] + + await write_state(self.connector_name, self.globopts, self.confcust, self.timestamp, True) + + if eval(self.globopts['GeneralPublishWebAPI'.lower()]): + await self.send_webapi(dts) + + if dts or write_empty: + cust = list(self.confcust.get_customers())[0] + self.logger.info('Customer:%s Fetched Date:%s Endpoints:%d' % + (self.confcust.get_custname(cust), self.targetdate, len(dts))) + + if eval(self.globopts['GeneralWriteAvro'.lower()]): + write_avro(self.logger, self.globopts, self.confcust, dts, self.timestamp) From cb314955067e63ad3cfd6764e1e61bb62fb1afce Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Wed, 6 Jul 2022 14:18:12 +0200 Subject: [PATCH 03/26] initial test_downtfeed --- tests/test_downfeed.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 tests/test_downfeed.py diff --git a/tests/test_downfeed.py b/tests/test_downfeed.py new file mode 100644 index 00000000..53fa1f63 --- /dev/null +++ b/tests/test_downfeed.py @@ -0,0 +1,24 @@ +import unittest + +from argo_connectors.log import Logger +from argo_connectors.parse.gocdb_topology import ParseServiceGroups, ParseServiceEndpoints, ParseSites +from argo_connectors.parse.flat_topology import ParseFlatEndpoints +from argo_connectors.parse.provider_topology import ParseTopo, ParseExtensions, buildmap_id2groupname +from argo_connectors.exceptions import ConnectorParseError +from argo_connectors.mesh.contacts import attach_contacts_topodata + +logger = Logger('test_downfeed.py') +CUSTOMER_NAME = 'CUSTOMERFOO' + + +class ParseCsvDowntimes(unittest.TestCase): + def setUp(self): + with open('tests/sample-downtimes.csv', encoding='utf-8') as feed_file: + downtimes = feed_file.read() + self.maxDiff = None + + def test_parseDowntimes(self): + pass + +if __name__ == '__main__': + unittest.main() From 2bd8a1a6d3e5d27a5be8d350df82a2479738c208 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 7 Jul 2022 15:48:30 +0200 Subject: [PATCH 04/26] adapt start and end for first parse test --- exec/downtimes-csv-connector.py | 10 ++++++---- tests/test_downfeed.py | 14 +++++++++----- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/exec/downtimes-csv-connector.py b/exec/downtimes-csv-connector.py index 4441def8..10dc2f46 100755 --- a/exec/downtimes-csv-connector.py +++ b/exec/downtimes-csv-connector.py @@ -10,7 +10,7 @@ from argo_connectors.exceptions import ConnectorHttpError, ConnectorParseError from argo_connectors.log import Logger -from argo_connectors.tasks.gocdb_downtimes import TaskGocdbDowntimes +from argo_connectors.tasks.flat_downtimes import TaskCsvDowntimes from argo_connectors.tasks.common import write_state from argo_connectors.config import Global, CustomerConf @@ -77,9 +77,11 @@ def main(): try: cust = list(confcust.get_customers())[0] - task = TaskGocdbDowntimes(loop, logger, sys.argv[0], globopts, - webapi_opts, confcust, confcust.get_custname(cust), feed, - DOWNTIMEPI, start, end, uidservtype, args.date[0], timestamp) + task = TaskCsvDowntimes(loop, logger, sys.argv[0], globopts, + webapi_opts, confcust, + confcust.get_custname(cust), feed, DOWNTIMEPI, + start, end, uidservtype, args.date[0], + timestamp) loop.run_until_complete(task.run()) except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc: diff --git a/tests/test_downfeed.py b/tests/test_downfeed.py index 53fa1f63..3687221c 100644 --- a/tests/test_downfeed.py +++ b/tests/test_downfeed.py @@ -1,11 +1,10 @@ import unittest +import datetime + from argo_connectors.log import Logger -from argo_connectors.parse.gocdb_topology import ParseServiceGroups, ParseServiceEndpoints, ParseSites -from argo_connectors.parse.flat_topology import ParseFlatEndpoints -from argo_connectors.parse.provider_topology import ParseTopo, ParseExtensions, buildmap_id2groupname +from argo_connectors.parse.flat_downtimes import ParseDowntimes from argo_connectors.exceptions import ConnectorParseError -from argo_connectors.mesh.contacts import attach_contacts_topodata logger = Logger('test_downfeed.py') CUSTOMER_NAME = 'CUSTOMERFOO' @@ -16,9 +15,14 @@ def setUp(self): with open('tests/sample-downtimes.csv', encoding='utf-8') as feed_file: downtimes = feed_file.read() self.maxDiff = None + date_2_21_2022 = datetime.datetime(2022, 2, 21) + start = date_2_21_2022 + end = date_2_21_2022.replace(hour=23, minute=59, second=59) + self.flat_downtimes = ParseDowntimes(logger, downtimes, start, end, False) def test_parseDowntimes(self): - pass + downtimes = self.flat_downtimes.get_data() + self.assertEqual(downtimes, []) if __name__ == '__main__': unittest.main() From 7a7743509b01b6501f81a3f117f14651ef2a2fdb Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 7 Jul 2022 16:52:41 +0200 Subject: [PATCH 05/26] added service_type column in sample --- tests/sample-downtimes.csv | 84 +++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/tests/sample-downtimes.csv b/tests/sample-downtimes.csv index 78d6a886..84e6e4d0 100644 --- a/tests/sample-downtimes.csv +++ b/tests/sample-downtimes.csv @@ -1,42 +1,42 @@ -unique_id,url,start_time,end_time,Severity,Description -neanias_5,https://atmo-flud.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_7,https://atmo-stress.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_9,https://caesar.neanias.eu:443,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_10,http://caesar-api.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_14,https://bathyprocessing.neanias.eu/hub/login,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_15,https://uw-map.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_16,https://uw-map.neanias.eu/tos,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_17,https://uw-map.neanias.eu/privacy,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_18,https://uw-mos.neanias.eu/api/try,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_20,https://ai-gateway.neanias.eu/hub/login,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_21,https://accounting.neanias.eu/api/api/accounting-service/version-info/current,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_25,https://logging.neanias.eu/api/status,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_26,https://minio.ml-models.neanias.eu/minio/health/live,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_27,https://notification.neanias.eu/api/notification/version-info/current,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_28,https://pms.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_30,https://vis-gateway.neanias.eu/hub/login,2/21/2022 8:00,2/22/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -,,,,, -neanias_5,https://atmo-flud.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_7,https://atmo-stress.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_9,https://caesar.neanias.eu:443,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_10,http://caesar-api.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_14,https://bathyprocessing.neanias.eu/hub/login,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_15,https://uw-map.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_16,https://uw-map.neanias.eu/tos,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_17,https://uw-map.neanias.eu/privacy,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_18,https://uw-mos.neanias.eu/api/try,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_20,https://ai-gateway.neanias.eu/hub/login,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_21,https://accounting.neanias.eu/api/api/accounting-service/version-info/current,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_25,https://logging.neanias.eu/api/status,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_26,https://minio.ml-models.neanias.eu/minio/health/live,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_27,https://notification.neanias.eu/api/notification/version-info/current,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_28,https://pms.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_30,https://vis-gateway.neanias.eu/hub/login,3/1/2022 8:00,3/4/2022 19:00,OUTAGE,GARR Cloud garr-ct1 region maintenance -neanias_2,https://files.neanias.eu,2/28/2022 13:00,2/28/2022 15:00,OUTAGE,Data sharing service maintenance -neanias_1,https://files.dev.neanias.eu,2/28/2022 9:30,2/28/2022 11:00,OUTAGE,Data sharing service maintenance -neanias_1,https://files.dev.neanias.eu,2/7/2022 8:00,2/17/2022 19:00,OUTAGE,GARR Cloud garr-pa1 region maintenance -neanias_2,https://files.neanias.eu,2/7/2022 8:00,2/17/2022 19:00,OUTAGE,GARR Cloud garr-pa1 region maintenance -neanias_4,https://atmo-4cast.neanias.eu/api/forecaststatus/1,2/7/2022 8:00,2/17/2022 19:00,OUTAGE,GARR Cloud garr-pa1 region maintenance -neanias_6,http://atmo-seism.neanias.eu/hub/login,2/7/2022 8:00,2/17/2022 19:00,OUTAGE,GARR Cloud garr-pa1 region maintenance -neanias_11,http://vlkb.neanias.eu:8080/vlkb/tap/availability,2/7/2022 8:00,2/17/2022 19:00,OUTAGE,GARR Cloud garr-pa1 region maintenance -neanias_29,https://vd-maps.neanias.eu,2/7/2022 8:00,2/17/2022 19:00,OUTAGE,GARR Cloud garr-pa1 region maintenance \ No newline at end of file +unique_id,url,start_time,end_time,service_type,Severity,Description +neanias_5,https://atmo-flud.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,Webservice,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_7,https://atmo-stress.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_9,https://caesar.neanias.eu:443,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_10,http://caesar-api.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_14,https://bathyprocessing.neanias.eu/hub/login,2/21/2022 8:00,2/22/2022 19:00,JupyterHub,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_15,https://uw-map.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_16,https://uw-map.neanias.eu/tos,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_17,https://uw-map.neanias.eu/privacy,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_18,https://uw-mos.neanias.eu/api/try,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_20,https://ai-gateway.neanias.eu/hub/login,2/21/2022 8:00,2/22/2022 19:00,JupyterHub,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_21,https://accounting.neanias.eu/api/api/accounting-service/version-info/current,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_25,https://logging.neanias.eu/api/status,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_26,https://minio.ml-models.neanias.eu/minio/health/live,2/21/2022 8:00,2/22/2022 19:00,MinIO,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_27,https://notification.neanias.eu/api/notification/version-info/current,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_28,https://pms.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_30,https://vis-gateway.neanias.eu/hub/login,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +,,,,,, +neanias_5,https://atmo-flud.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_7,https://atmo-stress.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_9,https://caesar.neanias.eu:443,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_10,http://caesar-api.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_14,https://bathyprocessing.neanias.eu/hub/login,3/1/2022 8:00,3/4/2022 19:00,JupyterHub,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_15,https://uw-map.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_16,https://uw-map.neanias.eu/tos,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_17,https://uw-map.neanias.eu/privacy,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_18,https://uw-mos.neanias.eu/api/try,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_20,https://ai-gateway.neanias.eu/hub/login,3/1/2022 8:00,3/4/2022 19:00,JupyterHub,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_21,https://accounting.neanias.eu/api/api/accounting-service/version-info/current,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_25,https://logging.neanias.eu/api/status,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_26,https://minio.ml-models.neanias.eu/minio/health/live,3/1/2022 8:00,3/4/2022 19:00,MinIO,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_27,https://notification.neanias.eu/api/notification/version-info/current,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_28,https://pms.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_30,https://vis-gateway.neanias.eu/hub/login,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_2,https://files.neanias.eu,2/28/2022 13:00,2/28/2022 15:00,nextcloud,OUTAGE,Data sharing service maintenance +neanias_1,https://files.dev.neanias.eu,2/28/2022 9:30,2/28/2022 11:00,nextcloud,OUTAGE,Data sharing service maintenance +neanias_1,https://files.dev.neanias.eu,2/7/2022 8:00,2/17/2022 19:00,nextcloud,OUTAGE,GARR Cloud garr-pa1 region maintenance +neanias_2,https://files.neanias.eu,2/7/2022 8:00,2/17/2022 19:00,nextcloud,OUTAGE,GARR Cloud garr-pa1 region maintenance +neanias_4,https://atmo-4cast.neanias.eu/api/forecaststatus/1,2/7/2022 8:00,2/17/2022 19:00,WebService,OUTAGE,GARR Cloud garr-pa1 region maintenance +neanias_6,http://atmo-seism.neanias.eu/hub/login,2/7/2022 8:00,2/17/2022 19:00,JupyterHub,OUTAGE,GARR Cloud garr-pa1 region maintenance +neanias_11,http://vlkb.neanias.eu:8080/vlkb/tap/availability,2/7/2022 8:00,2/17/2022 19:00,WebService,OUTAGE,GARR Cloud garr-pa1 region maintenance +neanias_29,https://vd-maps.neanias.eu,2/7/2022 8:00,2/17/2022 19:00,WebService,OUTAGE,GARR Cloud garr-pa1 region maintenance \ No newline at end of file From 0caeb58c81df95f4957b4db0aaca95eef6aa804a Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 7 Jul 2022 17:15:28 +0200 Subject: [PATCH 06/26] generate downtimes list from parsed csv --- modules/parse/flat_downtimes.py | 36 +++++++++++++++++++++++++++++++-- modules/tasks/flat_downtimes.py | 3 --- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/modules/parse/flat_downtimes.py b/modules/parse/flat_downtimes.py index f37cab99..28ed49f7 100644 --- a/modules/parse/flat_downtimes.py +++ b/modules/parse/flat_downtimes.py @@ -2,11 +2,43 @@ import xml.dom.minidom from xml.parsers.expat import ExpatError -from argo_connectors.utils import module_class_name + from argo_connectors.exceptions import ConnectorParseError from argo_connectors.parse.base import ParseHelpers +from argo_connectors.utils import construct_fqdn +from argo_connectors.utils import module_class_name class ParseDowntimes(ParseHelpers): def __init__(self, logger, data, start, end, uid=False): - pass + self.data = self.csv_to_json(data) + self.start = start + self.end = end + self.uid = uid + + def get_data(self): + downtimes = list() + + for downtime in self.data: + entry = dict() + + service_id = downtime['unique_id'] + if not service_id: + continue + + hostname = construct_fqdn(downtime['url']) + service_type = downtime['service_type'] + start_time = datetime.datetime.strptime(downtime['start_time'], "%m/%d/%Y %H:%M") + end_time = datetime.datetime.strptime(downtime['end_time'], "%m/%d/%Y %H:%M") + + if self.uid: + entry['hostname'] = '{0}_{1}'.format(hostname, service_id) + else: + entry['hostname'] = hostname + + downtime['service'] = service_type + downtime['start_time'] = start_time.strftime('%Y-%m-%dT%H:%M:00Z') + downtime['end_time'] = end_time.strftime('%Y-%m-%dT%H:%M:00Z') + downtimes.append(downtime) + + return downtimes diff --git a/modules/tasks/flat_downtimes.py b/modules/tasks/flat_downtimes.py index a7352cdd..9c548383 100644 --- a/modules/tasks/flat_downtimes.py +++ b/modules/tasks/flat_downtimes.py @@ -28,9 +28,6 @@ def __init__(self, loop, logger, connector_name, globopts, webapi_opts, self.timestamp = timestamp async def fetch_data(self): - feed_parts = urlparse(self.feed) - start_fmt = self.start.strftime("%Y-%m-%d") - end_fmt = self.end.strftime("%Y-%m-%d") session = SessionWithRetry(self.logger, os.path.basename(self.connector_name), self.globopts) From 65fd50416d260c40e7de910ad29eb7ec1af0e62d Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 7 Jul 2022 17:38:19 +0200 Subject: [PATCH 07/26] test for failed downtimes parse --- modules/parse/flat_downtimes.py | 1 + tests/test_downfeed.py | 29 +++++++++++++++++++++-------- tests/test_servicetypefeed.py | 1 + 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/modules/parse/flat_downtimes.py b/modules/parse/flat_downtimes.py index 28ed49f7..3a2ff19a 100644 --- a/modules/parse/flat_downtimes.py +++ b/modules/parse/flat_downtimes.py @@ -11,6 +11,7 @@ class ParseDowntimes(ParseHelpers): def __init__(self, logger, data, start, end, uid=False): + self.logger = logger self.data = self.csv_to_json(data) self.start = start self.end = end diff --git a/tests/test_downfeed.py b/tests/test_downfeed.py index 3687221c..3434e880 100644 --- a/tests/test_downfeed.py +++ b/tests/test_downfeed.py @@ -1,28 +1,41 @@ -import unittest - import datetime +import mock +import unittest from argo_connectors.log import Logger from argo_connectors.parse.flat_downtimes import ParseDowntimes from argo_connectors.exceptions import ConnectorParseError -logger = Logger('test_downfeed.py') CUSTOMER_NAME = 'CUSTOMERFOO' class ParseCsvDowntimes(unittest.TestCase): def setUp(self): with open('tests/sample-downtimes.csv', encoding='utf-8') as feed_file: - downtimes = feed_file.read() + self.downtimes = feed_file.read() self.maxDiff = None + logger = mock.Mock() + logger.customer = CUSTOMER_NAME + self.logger = logger + + def test_parseDowntimes(self): date_2_21_2022 = datetime.datetime(2022, 2, 21) start = date_2_21_2022 end = date_2_21_2022.replace(hour=23, minute=59, second=59) - self.flat_downtimes = ParseDowntimes(logger, downtimes, start, end, False) - - def test_parseDowntimes(self): + self.flat_downtimes = ParseDowntimes(self.logger, self.downtimes, start, end, False) downtimes = self.flat_downtimes.get_data() - self.assertEqual(downtimes, []) + + def test_failedParseDowntimes(self): + date_2_21_2022 = datetime.datetime(2022, 2, 21) + start = date_2_21_2022 + end = date_2_21_2022.replace(hour=23, minute=59, second=59) + with self.assertRaises(ConnectorParseError) as cm: + self.flat_downtimes = ParseDowntimes(self.logger, 'DUMMY DATA', start, end, False) + downtimes = self.flat_downtimes.get_data() + + excep = cm.exception + self.assertTrue('CSV feed' in excep.msg) + self.assertTrue(CUSTOMER_NAME in excep.msg) if __name__ == '__main__': unittest.main() diff --git a/tests/test_servicetypefeed.py b/tests/test_servicetypefeed.py index e2c56574..520e0b8e 100644 --- a/tests/test_servicetypefeed.py +++ b/tests/test_servicetypefeed.py @@ -10,6 +10,7 @@ CUSTOMER_NAME = 'CUSTOMERFOO' + class ParseGocdb(unittest.TestCase): def setUp(self): with open('tests/sample-service_types_gocdb.xml', encoding='utf-8') as feed_file: From d58dce7d6bf5bf457657ff8cb21489ba3d69652c Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Jul 2022 12:25:36 +0200 Subject: [PATCH 08/26] take into account only OUTAGE scheduled downtimes --- modules/parse/flat_downtimes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/parse/flat_downtimes.py b/modules/parse/flat_downtimes.py index 3a2ff19a..70facd3f 100644 --- a/modules/parse/flat_downtimes.py +++ b/modules/parse/flat_downtimes.py @@ -31,6 +31,7 @@ def get_data(self): service_type = downtime['service_type'] start_time = datetime.datetime.strptime(downtime['start_time'], "%m/%d/%Y %H:%M") end_time = datetime.datetime.strptime(downtime['end_time'], "%m/%d/%Y %H:%M") + classification = downtime['Severity'] if self.uid: entry['hostname'] = '{0}_{1}'.format(hostname, service_id) @@ -40,6 +41,8 @@ def get_data(self): downtime['service'] = service_type downtime['start_time'] = start_time.strftime('%Y-%m-%dT%H:%M:00Z') downtime['end_time'] = end_time.strftime('%Y-%m-%dT%H:%M:00Z') - downtimes.append(downtime) + + if classification == 'OUTAGE': + downtimes.append(downtime) return downtimes From fcfc6936cd5a98f1e5ffc0f185a921356ba4aa5a Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Jul 2022 13:24:10 +0200 Subject: [PATCH 09/26] refactored class init with only current_date arg --- modules/parse/flat_downtimes.py | 14 +++++++++++--- tests/test_downfeed.py | 8 ++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/modules/parse/flat_downtimes.py b/modules/parse/flat_downtimes.py index 70facd3f..dc4ec372 100644 --- a/modules/parse/flat_downtimes.py +++ b/modules/parse/flat_downtimes.py @@ -10,11 +10,11 @@ class ParseDowntimes(ParseHelpers): - def __init__(self, logger, data, start, end, uid=False): + def __init__(self, logger, data, current_date, uid=False): self.logger = logger self.data = self.csv_to_json(data) - self.start = start - self.end = end + self.start = current_date + self.end = current_date.replace(hour=23, minute=59, second=59) self.uid = uid def get_data(self): @@ -38,6 +38,14 @@ def get_data(self): else: entry['hostname'] = hostname + start_date = start_time.replace(hour=0, minute=0, second=0) + end_date = end_time.replace(hour=0, minute=0, second=0) + if self.start >= start_date and self.start <= end_date: + if start_time < self.start: + start_time = self.start + if end_time > self.end: + end_time = self.end + downtime['service'] = service_type downtime['start_time'] = start_time.strftime('%Y-%m-%dT%H:%M:00Z') downtime['end_time'] = end_time.strftime('%Y-%m-%dT%H:%M:00Z') diff --git a/tests/test_downfeed.py b/tests/test_downfeed.py index 3434e880..02a9bc49 100644 --- a/tests/test_downfeed.py +++ b/tests/test_downfeed.py @@ -20,17 +20,13 @@ def setUp(self): def test_parseDowntimes(self): date_2_21_2022 = datetime.datetime(2022, 2, 21) - start = date_2_21_2022 - end = date_2_21_2022.replace(hour=23, minute=59, second=59) - self.flat_downtimes = ParseDowntimes(self.logger, self.downtimes, start, end, False) + self.flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_2_21_2022, False) downtimes = self.flat_downtimes.get_data() def test_failedParseDowntimes(self): date_2_21_2022 = datetime.datetime(2022, 2, 21) - start = date_2_21_2022 - end = date_2_21_2022.replace(hour=23, minute=59, second=59) with self.assertRaises(ConnectorParseError) as cm: - self.flat_downtimes = ParseDowntimes(self.logger, 'DUMMY DATA', start, end, False) + self.flat_downtimes = ParseDowntimes(self.logger, 'DUMMY DATA', date_2_21_2022, False) downtimes = self.flat_downtimes.get_data() excep = cm.exception From 546b1f7655c2558ace1deb902cd2d50445082c70 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Jul 2022 13:44:31 +0200 Subject: [PATCH 10/26] refactored discarding of schedules --- modules/parse/flat_downtimes.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/parse/flat_downtimes.py b/modules/parse/flat_downtimes.py index dc4ec372..1e86987f 100644 --- a/modules/parse/flat_downtimes.py +++ b/modules/parse/flat_downtimes.py @@ -24,14 +24,14 @@ def get_data(self): entry = dict() service_id = downtime['unique_id'] - if not service_id: + classification = downtime['Severity'] + if not service_id or classification != 'OUTAGE': continue hostname = construct_fqdn(downtime['url']) service_type = downtime['service_type'] start_time = datetime.datetime.strptime(downtime['start_time'], "%m/%d/%Y %H:%M") end_time = datetime.datetime.strptime(downtime['end_time'], "%m/%d/%Y %H:%M") - classification = downtime['Severity'] if self.uid: entry['hostname'] = '{0}_{1}'.format(hostname, service_id) @@ -50,7 +50,6 @@ def get_data(self): downtime['start_time'] = start_time.strftime('%Y-%m-%dT%H:%M:00Z') downtime['end_time'] = end_time.strftime('%Y-%m-%dT%H:%M:00Z') - if classification == 'OUTAGE': - downtimes.append(downtime) + downtimes.append(downtime) return downtimes From ef5fc31c91884b498371e5bdd8760bf7f5b612f5 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Jul 2022 13:56:45 +0200 Subject: [PATCH 11/26] correct fill of data struct --- modules/parse/flat_downtimes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/parse/flat_downtimes.py b/modules/parse/flat_downtimes.py index 1e86987f..018a65f8 100644 --- a/modules/parse/flat_downtimes.py +++ b/modules/parse/flat_downtimes.py @@ -46,10 +46,10 @@ def get_data(self): if end_time > self.end: end_time = self.end - downtime['service'] = service_type - downtime['start_time'] = start_time.strftime('%Y-%m-%dT%H:%M:00Z') - downtime['end_time'] = end_time.strftime('%Y-%m-%dT%H:%M:00Z') + entry['service'] = service_type + entry['start_time'] = start_time.strftime('%Y-%m-%dT%H:%M:00Z') + entry['end_time'] = end_time.strftime('%Y-%m-%dT%H:%M:00Z') - downtimes.append(downtime) + downtimes.append(entry) return downtimes From 3fdaceb7aad08f83544cab9f4085a26bf7c4f99e Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Jul 2022 14:09:40 +0200 Subject: [PATCH 12/26] test for correct start and end of downtime schedule of particular date --- tests/test_downfeed.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/test_downfeed.py b/tests/test_downfeed.py index 02a9bc49..f3f51752 100644 --- a/tests/test_downfeed.py +++ b/tests/test_downfeed.py @@ -20,14 +20,30 @@ def setUp(self): def test_parseDowntimes(self): date_2_21_2022 = datetime.datetime(2022, 2, 21) - self.flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_2_21_2022, False) - downtimes = self.flat_downtimes.get_data() + flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_2_21_2022, False) + downtimes = flat_downtimes.get_data() + self.assertEqual(len(downtimes), 16) + first_schedule = downtimes[0] + start_time = datetime.datetime.strptime(first_schedule['start_time'], '%Y-%m-%dT%H:%M:00Z') + end_time = datetime.datetime.strptime(first_schedule['end_time'], '%Y-%m-%dT%H:%M:00Z') + self.assertEqual(start_time, datetime.datetime(2022, 2, 21, 8, 0)) + self.assertEqual(end_time, datetime.datetime(2022, 2, 21, 23, 59)) + + date_2_22_2022 = datetime.datetime(2022, 2, 22) + flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_2_22_2022, False) + downtimes = flat_downtimes.get_data() + self.assertEqual(len(downtimes), 16) + first_schedule = downtimes[0] + start_time = datetime.datetime.strptime(first_schedule['start_time'], '%Y-%m-%dT%H:%M:00Z') + end_time = datetime.datetime.strptime(first_schedule['end_time'], '%Y-%m-%dT%H:%M:00Z') + self.assertEqual(start_time, datetime.datetime(2022, 2, 22, 0, 0)) + self.assertEqual(end_time, datetime.datetime(2022, 2, 22, 19, 0)) def test_failedParseDowntimes(self): date_2_21_2022 = datetime.datetime(2022, 2, 21) with self.assertRaises(ConnectorParseError) as cm: - self.flat_downtimes = ParseDowntimes(self.logger, 'DUMMY DATA', date_2_21_2022, False) - downtimes = self.flat_downtimes.get_data() + flat_downtimes = ParseDowntimes(self.logger, 'DUMMY DATA', date_2_21_2022, False) + downtimes = flat_downtimes.get_data() excep = cm.exception self.assertTrue('CSV feed' in excep.msg) From e8bd891a609477ba6ee6814aa2f4549008f6283f Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Jul 2022 14:17:15 +0200 Subject: [PATCH 13/26] test for correct start and end of downtime schedule of another set of days --- tests/test_downfeed.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_downfeed.py b/tests/test_downfeed.py index f3f51752..270cb2bb 100644 --- a/tests/test_downfeed.py +++ b/tests/test_downfeed.py @@ -39,6 +39,36 @@ def test_parseDowntimes(self): self.assertEqual(start_time, datetime.datetime(2022, 2, 22, 0, 0)) self.assertEqual(end_time, datetime.datetime(2022, 2, 22, 19, 0)) + date_3_1_2022 = datetime.datetime(2022, 3, 1) + flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_3_1_2022, False) + downtimes = flat_downtimes.get_data() + self.assertEqual(len(downtimes), 16) + first_schedule = downtimes[0] + start_time = datetime.datetime.strptime(first_schedule['start_time'], '%Y-%m-%dT%H:%M:00Z') + end_time = datetime.datetime.strptime(first_schedule['end_time'], '%Y-%m-%dT%H:%M:00Z') + self.assertEqual(start_time, datetime.datetime(2022, 3, 1, 8, 0)) + self.assertEqual(end_time, datetime.datetime(2022, 3, 1, 23, 59)) + + date_3_2_2022 = datetime.datetime(2022, 3, 2) + flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_3_2_2022, False) + downtimes = flat_downtimes.get_data() + self.assertEqual(len(downtimes), 16) + first_schedule = downtimes[0] + start_time = datetime.datetime.strptime(first_schedule['start_time'], '%Y-%m-%dT%H:%M:00Z') + end_time = datetime.datetime.strptime(first_schedule['end_time'], '%Y-%m-%dT%H:%M:00Z') + self.assertEqual(start_time, datetime.datetime(2022, 3, 2, 0, 0)) + self.assertEqual(end_time, datetime.datetime(2022, 3, 2, 23, 59)) + + date_3_4_2022 = datetime.datetime(2022, 3, 4) + flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_3_4_2022, False) + downtimes = flat_downtimes.get_data() + self.assertEqual(len(downtimes), 16) + first_schedule = downtimes[0] + start_time = datetime.datetime.strptime(first_schedule['start_time'], '%Y-%m-%dT%H:%M:00Z') + end_time = datetime.datetime.strptime(first_schedule['end_time'], '%Y-%m-%dT%H:%M:00Z') + self.assertEqual(start_time, datetime.datetime(2022, 3, 4, 0, 0)) + self.assertEqual(end_time, datetime.datetime(2022, 3, 4, 19, 0)) + def test_failedParseDowntimes(self): date_2_21_2022 = datetime.datetime(2022, 2, 21) with self.assertRaises(ConnectorParseError) as cm: From bcad28aba332b213c80aeec1c4442c280b554ecf Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Jul 2022 15:00:09 +0200 Subject: [PATCH 14/26] parse customer option DowntimesFeed --- modules/config.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/modules/config.py b/modules/config.py index dfffbece..106aa488 100644 --- a/modules/config.py +++ b/modules/config.py @@ -58,6 +58,10 @@ def __init__(self, caller, confpath=None, **kwargs): self._merge_dict(self.shared_secopts, self.conf_topo_schemas, self.conf_topo_output), + 'downtimes-csv-connector.py': + self._merge_dict(self.shared_secopts, + self.conf_downtimes_schemas, + self.conf_downtimes_output), 'downtimes-gocdb-connector.py': self._merge_dict(self.shared_secopts, self.conf_downtimes_schemas, @@ -209,6 +213,7 @@ class CustomerConf(object): 'topology-provider-connector.py': [''], 'metricprofile-webapi-connector.py': ['MetricProfileNamespace'], 'downtimes-gocdb-connector.py': ['DowntimesFeed', 'TopoUIDServiceEndpoints'], + 'downtimes-csv-connector.py': ['DowntimesFeed', 'TopoUIDServiceEndpoints'], 'weights-vapor-connector.py': ['WeightsFeed', 'TopoFetchType'], 'service-types-gocdb-connector.py': ['ServiceTypesFeed'], @@ -266,6 +271,7 @@ def parse(self): topofeedservicegroups = config.get(section, 'TopoFeedServiceGroups', fallback=None) topofeedpaging = config.get(section, 'TopoFeedPaging', fallback='GOCDB') servicetypesfeed = config.get(section, 'ServiceTypesFeed', fallback=None) + downtimesfeed = config.get(section, 'DowntimesFeed', fallback=None) if not custdir.endswith('/'): custdir = '{}/'.format(custdir) @@ -286,17 +292,18 @@ def parse(self): self._cust.update({section: {'Jobs': custjobs, 'OutputDir': custdir, 'Name': custname, - 'TopoFetchType': topofetchtype, - 'TopoFeedPaging': topofeedpaging, - 'TopoScope': toposcope, + 'DowntimesFeed': downtimesfeed, + 'ServiceTypesFeed': servicetypesfeed, 'TopoFeed': topofeed, - 'TopoFeedSites': topofeedsites, - 'TopoFeedServiceGroups': topofeedservicegroups, 'TopoFeedEndpoints': topofeedendpoints, 'TopoFeedEndpointsExtensions': topofeedendpointsextensions, - 'TopoUIDServiceEnpoints': topouidservendpoints, + 'TopoFeedPaging': topofeedpaging, + 'TopoFeedServiceGroups': topofeedservicegroups, + 'TopoFeedSites': topofeedsites, + 'TopoFetchType': topofetchtype, + 'TopoScope': toposcope, 'TopoType': topotype, - 'ServiceTypesFeed': servicetypesfeed + 'TopoUIDServiceEnpoints': topouidservendpoints }}) if optopts: auth, webapi, empty_data, bdii = {}, {}, {}, {} @@ -504,6 +511,9 @@ def _get_cust_options(self, opt): target_option = options[option] return target_option + def get_downfeed(self): + return self._get_cust_options('DowntimesFeed') + def get_topofeed(self): return self._get_cust_options('TopoFeed') From a3c50a0cb35500a2d37f0544f93e87a22257fda6 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Jul 2022 15:01:25 +0200 Subject: [PATCH 15/26] correct task class initialization --- exec/downtimes-csv-connector.py | 14 ++++++-------- modules/tasks/flat_downtimes.py | 12 +++++------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/exec/downtimes-csv-connector.py b/exec/downtimes-csv-connector.py index 10dc2f46..94718e93 100755 --- a/exec/downtimes-csv-connector.py +++ b/exec/downtimes-csv-connector.py @@ -49,7 +49,7 @@ def main(): confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) - feed = confcust.get_topofeed() + feed = confcust.get_downfeed() logger.customer = confcust.get_custname() if len(args.date) == 0: @@ -58,11 +58,9 @@ def main(): # calculate start and end times try: - start = datetime.datetime.strptime(args.date[0], '%Y-%m-%d') - end = datetime.datetime.strptime(args.date[0], '%Y-%m-%d') - timestamp = start.strftime('%Y_%m_%d') - start = start.replace(hour=0, minute=0, second=0) - end = end.replace(hour=23, minute=59, second=59) + current_date = datetime.datetime.strptime(args.date[0], '%Y-%m-%d') + timestamp = current_date.strftime('%Y_%m_%d') + current_date = current_date.replace(hour=0, minute=0, second=0) except ValueError as exc: logger.error(exc) @@ -79,8 +77,8 @@ def main(): cust = list(confcust.get_customers())[0] task = TaskCsvDowntimes(loop, logger, sys.argv[0], globopts, webapi_opts, confcust, - confcust.get_custname(cust), feed, DOWNTIMEPI, - start, end, uidservtype, args.date[0], + confcust.get_custname(cust), feed, + current_date, uidservtype, args.date[0], timestamp) loop.run_until_complete(task.run()) diff --git a/modules/tasks/flat_downtimes.py b/modules/tasks/flat_downtimes.py index 9c548383..d4993f9a 100644 --- a/modules/tasks/flat_downtimes.py +++ b/modules/tasks/flat_downtimes.py @@ -10,7 +10,7 @@ class TaskCsvDowntimes(object): def __init__(self, loop, logger, connector_name, globopts, webapi_opts, - confcust, custname, feed, DOWNTIMEPI, start, end, + confcust, custname, feed, current_date, uidservtype, targetdate, timestamp): self.event_loop = loop self.logger = logger @@ -20,9 +20,7 @@ def __init__(self, loop, logger, connector_name, globopts, webapi_opts, self.confcust = confcust self.custname = custname self.feed = feed - self.DOWNTIMEPI = DOWNTIMEPI - self.start = start - self.end = end + self.current_date = current_date self.uidservtype = uidservtype self.targetdate = targetdate self.timestamp = timestamp @@ -31,13 +29,13 @@ async def fetch_data(self): session = SessionWithRetry(self.logger, os.path.basename(self.connector_name), self.globopts) - res = await session.http_get() + res = await session.http_get(self.feed) return res def parse_source(self, res): - csv_downtimes = ParseDowntimes(self.logger, res, self.start, self.end, - self.uidservtype) + csv_downtimes = ParseDowntimes(self.logger, res, self.current_date, + self.uidservtype) return csv_downtimes.get_data() async def send_webapi(self, dts): From adf86b89797eb885501afd959b9d719d4181c6e7 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Jul 2022 15:03:46 +0200 Subject: [PATCH 16/26] remove unused global var --- exec/downtimes-csv-connector.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/exec/downtimes-csv-connector.py b/exec/downtimes-csv-connector.py index 94718e93..154e9480 100755 --- a/exec/downtimes-csv-connector.py +++ b/exec/downtimes-csv-connector.py @@ -18,8 +18,6 @@ logger = None globopts = {} -DOWNTIMEPI = '/gocdbpi/private/?method=get_downtime' - def get_webapi_opts(cglob, confcust): webapi_custopts = confcust.get_webapiopts() From a99df63e7263e36aa9854a390e5de93cfed775ca Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Jul 2022 15:06:20 +0200 Subject: [PATCH 17/26] proper initialization of WebAPI class with newly added downtimes-csv-connector --- modules/io/webapi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/io/webapi.py b/modules/io/webapi.py index d4e7d272..b23cada8 100644 --- a/modules/io/webapi.py +++ b/modules/io/webapi.py @@ -9,6 +9,7 @@ class WebAPI(object): methods = { + 'downtimes-csv-connector.py': 'downtimes', 'downtimes-gocdb-connector.py': 'downtimes', 'topology-gocdb-connector.py': 'topology', 'topology-csv-connector.py': 'topology', From 6c02790cdbdc9d5de2ca1865685ed6262b614697 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Jul 2022 15:09:09 +0200 Subject: [PATCH 18/26] test id append of hostname field --- tests/test_downfeed.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/test_downfeed.py b/tests/test_downfeed.py index 270cb2bb..1413d363 100644 --- a/tests/test_downfeed.py +++ b/tests/test_downfeed.py @@ -20,50 +20,55 @@ def setUp(self): def test_parseDowntimes(self): date_2_21_2022 = datetime.datetime(2022, 2, 21) - flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_2_21_2022, False) + flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_2_21_2022, True) downtimes = flat_downtimes.get_data() self.assertEqual(len(downtimes), 16) first_schedule = downtimes[0] + self.assertEqual(first_schedule['hostname'], 'atmo-flud.neanias.eu_neanias_5') start_time = datetime.datetime.strptime(first_schedule['start_time'], '%Y-%m-%dT%H:%M:00Z') end_time = datetime.datetime.strptime(first_schedule['end_time'], '%Y-%m-%dT%H:%M:00Z') self.assertEqual(start_time, datetime.datetime(2022, 2, 21, 8, 0)) self.assertEqual(end_time, datetime.datetime(2022, 2, 21, 23, 59)) date_2_22_2022 = datetime.datetime(2022, 2, 22) - flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_2_22_2022, False) + flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_2_22_2022, True) downtimes = flat_downtimes.get_data() self.assertEqual(len(downtimes), 16) first_schedule = downtimes[0] + self.assertEqual(first_schedule['hostname'], 'atmo-flud.neanias.eu_neanias_5') start_time = datetime.datetime.strptime(first_schedule['start_time'], '%Y-%m-%dT%H:%M:00Z') end_time = datetime.datetime.strptime(first_schedule['end_time'], '%Y-%m-%dT%H:%M:00Z') self.assertEqual(start_time, datetime.datetime(2022, 2, 22, 0, 0)) self.assertEqual(end_time, datetime.datetime(2022, 2, 22, 19, 0)) date_3_1_2022 = datetime.datetime(2022, 3, 1) - flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_3_1_2022, False) + flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_3_1_2022, True) downtimes = flat_downtimes.get_data() self.assertEqual(len(downtimes), 16) first_schedule = downtimes[0] + self.assertEqual(first_schedule['hostname'], 'atmo-flud.neanias.eu_neanias_5') start_time = datetime.datetime.strptime(first_schedule['start_time'], '%Y-%m-%dT%H:%M:00Z') end_time = datetime.datetime.strptime(first_schedule['end_time'], '%Y-%m-%dT%H:%M:00Z') self.assertEqual(start_time, datetime.datetime(2022, 3, 1, 8, 0)) self.assertEqual(end_time, datetime.datetime(2022, 3, 1, 23, 59)) date_3_2_2022 = datetime.datetime(2022, 3, 2) - flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_3_2_2022, False) + flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_3_2_2022, True) downtimes = flat_downtimes.get_data() self.assertEqual(len(downtimes), 16) first_schedule = downtimes[0] + self.assertEqual(first_schedule['hostname'], 'atmo-flud.neanias.eu_neanias_5') start_time = datetime.datetime.strptime(first_schedule['start_time'], '%Y-%m-%dT%H:%M:00Z') end_time = datetime.datetime.strptime(first_schedule['end_time'], '%Y-%m-%dT%H:%M:00Z') self.assertEqual(start_time, datetime.datetime(2022, 3, 2, 0, 0)) self.assertEqual(end_time, datetime.datetime(2022, 3, 2, 23, 59)) date_3_4_2022 = datetime.datetime(2022, 3, 4) - flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_3_4_2022, False) + flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_3_4_2022, True) downtimes = flat_downtimes.get_data() self.assertEqual(len(downtimes), 16) first_schedule = downtimes[0] + self.assertEqual(first_schedule['hostname'], 'atmo-flud.neanias.eu_neanias_5') start_time = datetime.datetime.strptime(first_schedule['start_time'], '%Y-%m-%dT%H:%M:00Z') end_time = datetime.datetime.strptime(first_schedule['end_time'], '%Y-%m-%dT%H:%M:00Z') self.assertEqual(start_time, datetime.datetime(2022, 3, 4, 0, 0)) From c1fea947628d24effba17dfe8efc2372109f0a99 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Jul 2022 15:10:48 +0200 Subject: [PATCH 19/26] ensure that new connector is packaged --- setup.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index d6e07be0..472445ee 100644 --- a/setup.py +++ b/setup.py @@ -36,14 +36,17 @@ def get_ver(): 'argo_connectors.parse', 'argo_connectors.mesh', 'argo_connectors.tasks'], data_files=[('/etc/argo-connectors', glob.glob('etc/*.conf.template')), - ('/usr/libexec/argo-connectors', ['exec/downtimes-gocdb-connector.py', - 'exec/metricprofile-webapi-connector.py', - 'exec/topology-provider-connector.py', - 'exec/topology-gocdb-connector.py', - 'exec/service-types-gocdb-connector.py', - 'exec/service-types-json-connector.py', - 'exec/service-types-csv-connector.py', - 'exec/topology-json-connector.py', - 'exec/weights-vapor-connector.py', - 'exec/topology-csv-connector.py']), + ('/usr/libexec/argo-connectors', [ + 'exec/downtimes-csv-connector.py', + 'exec/downtimes-gocdb-connector.py', + 'exec/metricprofile-webapi-connector.py', + 'exec/service-types-csv-connector.py', + 'exec/service-types-gocdb-connector.py', + 'exec/service-types-json-connector.py', + 'exec/topology-csv-connector.py', + 'exec/topology-gocdb-connector.py', + 'exec/topology-json-connector.py', + 'exec/topology-provider-connector.py', + 'exec/weights-vapor-connector.py', + ]), ('/etc/argo-connectors/schemas', glob.glob('etc/schemas/*.avsc'))]) From a1def8f78e379e93c1437d377aec302f0286b38f Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Jul 2022 15:33:27 +0200 Subject: [PATCH 20/26] test TaskCsvDowntimes --- tests/test_asynctasks.py | 70 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/tests/test_asynctasks.py b/tests/test_asynctasks.py index 3518452b..08461190 100644 --- a/tests/test_asynctasks.py +++ b/tests/test_asynctasks.py @@ -7,10 +7,11 @@ import mock from argo_connectors.exceptions import ConnectorError, ConnectorParseError, ConnectorHttpError +from argo_connectors.tasks.flat_downtimes import TaskCsvDowntimes +from argo_connectors.tasks.flat_servicetypes import TaskFlatServiceTypes from argo_connectors.tasks.gocdb_servicetypes import TaskGocdbServiceTypes -from argo_connectors.tasks.provider_topology import TaskProviderTopology from argo_connectors.tasks.gocdb_topology import TaskGocdbTopology -from argo_connectors.tasks.flat_servicetypes import TaskFlatServiceTypes +from argo_connectors.tasks.provider_topology import TaskProviderTopology from argo_connectors.parse.base import ParseHelpers @@ -202,6 +203,71 @@ async def test_StepsFailedRun(self, mock_writestate): self.assertTrue(self.services_gocdb.logger.error.call_args[0][0], repr(ConnectorHttpError('fetch_data failed'))) self.assertFalse(self.services_gocdb.send_webapi.called) +class DowntimesCsv(unittest.TestCase): + def setUp(self): + logger = mock.Mock() + logger.customer = CUSTOMER_NAME + self.loop = asyncio.get_event_loop() + mocked_globopts = dict(generalpublishwebapi='True') + globopts = mocked_globopts + webapiopts = mock.Mock() + authopts = mock.Mock() + confcust = mock.Mock() + confcust.send_empty.return_value = False + custname = CUSTOMER_NAME + feed = 'https://downtimes-csv.com/api/fetch' + timestamp = datetime.datetime.now().strftime('%Y_%m_%d') + current_date = datetime.datetime.now() + self.downtimes_flat = TaskCsvDowntimes( + self.loop, + logger, + 'test_asynctasks_downtimesflat', + globopts, + webapiopts, + confcust, + custname, + feed, + current_date, + True, + current_date, + timestamp + ) + self.maxDiff = None + + @mock.patch('argo_connectors.tasks.flat_downtimes.write_state') + @async_test + async def test_StepsSuccessRun(self, mock_writestate): + self.downtimes_flat.fetch_data = mock.AsyncMock() + self.downtimes_flat.fetch_data.side_effect = ['data_downtimes'] + self.downtimes_flat.send_webapi = mock.AsyncMock() + self.downtimes_flat.parse_source = mock.MagicMock() + await self.downtimes_flat.run() + self.assertTrue(self.downtimes_flat.fetch_data.called) + self.assertTrue(self.downtimes_flat.parse_source.called) + self.downtimes_flat.parse_source.assert_called_with('data_downtimes') + self.assertEqual(mock_writestate.call_args[0][0], 'test_asynctasks_downtimesflat') + self.assertEqual(mock_writestate.call_args[0][3], self.downtimes_flat.timestamp) + self.assertTrue(mock_writestate.call_args[0][4]) + self.assertTrue(self.downtimes_flat.send_webapi.called) + self.assertTrue(self.downtimes_flat.logger.info.called) + + @mock.patch('argo_connectors.tasks.flat_downtimes.write_state') + @async_test + async def test_StepsFailedRun(self, mock_writestate): + self.downtimes_flat.fetch_data = mock.AsyncMock() + self.downtimes_flat.fetch_data.side_effect = [ConnectorHttpError('fetch_data failed')] + self.downtimes_flat.send_webapi = mock.AsyncMock() + self.downtimes_flat.parse_source = mock.MagicMock() + await self.downtimes_flat.run() + self.assertTrue(self.downtimes_flat.fetch_data.called) + self.assertFalse(self.downtimes_flat.parse_source.called) + self.assertEqual(mock_writestate.call_args[0][0], 'test_asynctasks_servicetypesflat') + self.assertEqual(mock_writestate.call_args[0][3], self.downtimes_flat.timestamp) + self.assertFalse(mock_writestate.call_args[0][4]) + self.assertTrue(self.downtimes_flat.logger.error.called) + self.assertTrue(self.downtimes_flat.logger.error.call_args[0][0], repr(ConnectorHttpError('fetch_data failed'))) + self.assertFalse(self.downtimes_flat.send_webapi.called) + class ServiceTypesFlat(unittest.TestCase): def setUp(self): From 4823c85e257517a2519d72878fbf2477ad0b3baa Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Jul 2022 15:38:37 +0200 Subject: [PATCH 21/26] mock list as expected --- tests/test_asynctasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_asynctasks.py b/tests/test_asynctasks.py index 08461190..455f58dd 100644 --- a/tests/test_asynctasks.py +++ b/tests/test_asynctasks.py @@ -214,6 +214,7 @@ def setUp(self): authopts = mock.Mock() confcust = mock.Mock() confcust.send_empty.return_value = False + confcust.get_customers.return_value = ['CUSTOMERFOO', 'CUSTOMERBAR'] custname = CUSTOMER_NAME feed = 'https://downtimes-csv.com/api/fetch' timestamp = datetime.datetime.now().strftime('%Y_%m_%d') From 2532fd8a7e3376d0bdf6a3efa97663270ceddcd8 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Jul 2022 16:02:08 +0200 Subject: [PATCH 22/26] fix task test by proper mocking of additional steps like avro file write --- tests/test_asynctasks.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_asynctasks.py b/tests/test_asynctasks.py index 455f58dd..a12b83b1 100644 --- a/tests/test_asynctasks.py +++ b/tests/test_asynctasks.py @@ -208,13 +208,17 @@ def setUp(self): logger = mock.Mock() logger.customer = CUSTOMER_NAME self.loop = asyncio.get_event_loop() - mocked_globopts = dict(generalpublishwebapi='True') + mocked_globopts = dict(generalpublishwebapi='True', + generalwriteavro='True', + outputdowntimes='downtimes_DATE.avro', + avroschemasdowntimes='downtimes.avsc') globopts = mocked_globopts webapiopts = mock.Mock() authopts = mock.Mock() confcust = mock.Mock() confcust.send_empty.return_value = False confcust.get_customers.return_value = ['CUSTOMERFOO', 'CUSTOMERBAR'] + confcust.get_custdir.return_value = '/some/path' custname = CUSTOMER_NAME feed = 'https://downtimes-csv.com/api/fetch' timestamp = datetime.datetime.now().strftime('%Y_%m_%d') @@ -235,9 +239,10 @@ def setUp(self): ) self.maxDiff = None + @mock.patch('argo_connectors.tasks.flat_downtimes.write_avro') @mock.patch('argo_connectors.tasks.flat_downtimes.write_state') @async_test - async def test_StepsSuccessRun(self, mock_writestate): + async def test_StepsSuccessRun(self, mock_writestate, mock_writeavro): self.downtimes_flat.fetch_data = mock.AsyncMock() self.downtimes_flat.fetch_data.side_effect = ['data_downtimes'] self.downtimes_flat.send_webapi = mock.AsyncMock() @@ -249,6 +254,8 @@ async def test_StepsSuccessRun(self, mock_writestate): self.assertEqual(mock_writestate.call_args[0][0], 'test_asynctasks_downtimesflat') self.assertEqual(mock_writestate.call_args[0][3], self.downtimes_flat.timestamp) self.assertTrue(mock_writestate.call_args[0][4]) + self.assertTrue(mock_writeavro.called, True) + self.assertEqual(mock_writeavro.call_args[0][4], datetime.datetime.now().strftime('%Y_%m_%d')) self.assertTrue(self.downtimes_flat.send_webapi.called) self.assertTrue(self.downtimes_flat.logger.info.called) From f6f48cc3fb9074ce053452f07f57b88e8ed6a503 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Jul 2022 16:12:02 +0200 Subject: [PATCH 23/26] ensure that failed data fetchs are properly handled --- modules/tasks/flat_downtimes.py | 42 +++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/modules/tasks/flat_downtimes.py b/modules/tasks/flat_downtimes.py index d4993f9a..1dd8c326 100644 --- a/modules/tasks/flat_downtimes.py +++ b/modules/tasks/flat_downtimes.py @@ -2,9 +2,10 @@ from urllib.parse import urlparse +from argo_connectors.exceptions import ConnectorHttpError, ConnectorParseError from argo_connectors.io.http import SessionWithRetry -from argo_connectors.parse.flat_downtimes import ParseDowntimes from argo_connectors.io.webapi import WebAPI +from argo_connectors.parse.flat_downtimes import ParseDowntimes from argo_connectors.tasks.common import write_state, write_downtimes_avro as write_avro @@ -48,24 +49,29 @@ async def send_webapi(self, dts): await webapi.send(dts, downtimes_component=True) async def run(self): - # we don't have multiple tenant definitions in one - # customer file so we can safely assume one tenant/customer - write_empty = self.confcust.send_empty(self.connector_name) - if not write_empty: - res = await self.fetch_data() - dts = self.parse_source(res) - else: - dts = [] + try: + write_empty = self.confcust.send_empty(self.connector_name) + if not write_empty: + res = await self.fetch_data() + dts = self.parse_source(res) + else: + dts = [] + + await write_state(self.connector_name, self.globopts, self.confcust, self.timestamp, True) - await write_state(self.connector_name, self.globopts, self.confcust, self.timestamp, True) + if eval(self.globopts['GeneralPublishWebAPI'.lower()]): + await self.send_webapi(dts) - if eval(self.globopts['GeneralPublishWebAPI'.lower()]): - await self.send_webapi(dts) + # we don't have multiple tenant definitions in one + # customer file so we can safely assume one tenant/customer + if dts or write_empty: + cust = list(self.confcust.get_customers())[0] + self.logger.info('Customer:%s Fetched Date:%s Endpoints:%d' % + (self.confcust.get_custname(cust), self.targetdate, len(dts))) - if dts or write_empty: - cust = list(self.confcust.get_customers())[0] - self.logger.info('Customer:%s Fetched Date:%s Endpoints:%d' % - (self.confcust.get_custname(cust), self.targetdate, len(dts))) + if eval(self.globopts['GeneralWriteAvro'.lower()]): + write_avro(self.logger, self.globopts, self.confcust, dts, self.timestamp) - if eval(self.globopts['GeneralWriteAvro'.lower()]): - write_avro(self.logger, self.globopts, self.confcust, dts, self.timestamp) + except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc: + self.logger.error(repr(exc)) + await write_state(self.connector_name, self.globopts, self.confcust, self.timestamp, False) From 2f5b2431365da20122b4f59f7d7b4cae4924a80c Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Fri, 8 Jul 2022 16:13:07 +0200 Subject: [PATCH 24/26] correct mock string --- tests/test_asynctasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_asynctasks.py b/tests/test_asynctasks.py index a12b83b1..415a55ab 100644 --- a/tests/test_asynctasks.py +++ b/tests/test_asynctasks.py @@ -269,7 +269,7 @@ async def test_StepsFailedRun(self, mock_writestate): await self.downtimes_flat.run() self.assertTrue(self.downtimes_flat.fetch_data.called) self.assertFalse(self.downtimes_flat.parse_source.called) - self.assertEqual(mock_writestate.call_args[0][0], 'test_asynctasks_servicetypesflat') + self.assertEqual(mock_writestate.call_args[0][0], 'test_asynctasks_downtimesflat') self.assertEqual(mock_writestate.call_args[0][3], self.downtimes_flat.timestamp) self.assertFalse(mock_writestate.call_args[0][4]) self.assertTrue(self.downtimes_flat.logger.error.called) From 6e4e134e6a385aef69d6b773f97844580f6f0921 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 28 Jul 2022 08:35:43 +0200 Subject: [PATCH 25/26] spec version bump --- argo-connectors.spec | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/argo-connectors.spec b/argo-connectors.spec index 7ef5dc0d..4d66e5fc 100644 --- a/argo-connectors.spec +++ b/argo-connectors.spec @@ -2,7 +2,7 @@ %global __python /usr/bin/python3 Name: argo-connectors -Version: 2.1.0 +Version: 2.2.0 Release: 1%{?dist} Group: EGI/SA4 License: ASL 2.0 diff --git a/setup.py b/setup.py index 472445ee..3a3011ac 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ def get_ver(): setup(name=NAME, version=get_ver(), author='SRCE', - author_email='dvrcic@srce.hr, kzailac@srce.hr', + author_email='dvrcic@srce.hr', description='Components generate input data for ARGO Compute Engine', classifiers=[ "Development Status :: 5 - Production/Stable", From db66e709de36bd23d4c0db440a4be73a5cd72ba6 Mon Sep 17 00:00:00 2001 From: Daniel Vrcic Date: Thu, 28 Jul 2022 08:42:52 +0200 Subject: [PATCH 26/26] updated changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ea43df5..ea17d8e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [2.2.0] - 2022-07-28 + +### Added + +* ARGO-3695 [NEANIAS] Use ARGO for downtimes + ## [2.1.0] - 2022-06-07 ### Added