diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ea43df5..ea17d8e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [2.2.0] - 2022-07-28 + +### Added + +* ARGO-3695 [NEANIAS] Use ARGO for downtimes + ## [2.1.0] - 2022-06-07 ### Added diff --git a/argo-connectors.spec b/argo-connectors.spec index 7ef5dc0d..4d66e5fc 100644 --- a/argo-connectors.spec +++ b/argo-connectors.spec @@ -2,7 +2,7 @@ %global __python /usr/bin/python3 Name: argo-connectors -Version: 2.1.0 +Version: 2.2.0 Release: 1%{?dist} Group: EGI/SA4 License: ASL 2.0 diff --git a/exec/downtimes-csv-connector.py b/exec/downtimes-csv-connector.py new file mode 100755 index 00000000..154e9480 --- /dev/null +++ b/exec/downtimes-csv-connector.py @@ -0,0 +1,92 @@ +#!/usr/bin/python3 + +import argparse +import datetime +import os +import sys + +import asyncio +import uvloop + +from argo_connectors.exceptions import ConnectorHttpError, ConnectorParseError +from argo_connectors.log import Logger +from argo_connectors.tasks.flat_downtimes import TaskCsvDowntimes +from argo_connectors.tasks.common import write_state + +from argo_connectors.config import Global, CustomerConf + +logger = None +globopts = {} + + +def get_webapi_opts(cglob, confcust): + webapi_custopts = confcust.get_webapiopts() + webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') + webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') + if not webapi_complete: + logger.error('Customer:%s %s options incomplete, missing %s' % (logger.customer, 'webapi', ' '.join(missopt))) + raise SystemExit(1) + return webapi_opts + + +def main(): + global logger, globopts + parser = argparse.ArgumentParser(description='Fetch downtimes from CSV for given date') + parser.add_argument('-d', dest='date', nargs=1, metavar='YEAR-MONTH-DAY', required=True) + parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False) + parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) + args = parser.parse_args() + + logger = Logger(os.path.basename(sys.argv[0])) + confpath = args.gloconf[0] if args.gloconf else None + cglob = Global(sys.argv[0], confpath) + globopts = cglob.parse() + + confpath = args.custconf[0] if args.custconf else None + confcust = CustomerConf(sys.argv[0], confpath) + confcust.parse() + confcust.make_dirstruct() + confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) + feed = confcust.get_downfeed() + logger.customer = confcust.get_custname() + + if len(args.date) == 0: + print(parser.print_help()) + raise SystemExit(1) + + # calculate start and end times + try: + current_date = datetime.datetime.strptime(args.date[0], '%Y-%m-%d') + timestamp = current_date.strftime('%Y_%m_%d') + current_date = current_date.replace(hour=0, minute=0, second=0) + + except ValueError as exc: + logger.error(exc) + raise SystemExit(1) + + uidservtype = confcust.get_uidserviceendpoints() + + webapi_opts = get_webapi_opts(cglob, confcust) + + loop = uvloop.new_event_loop() + asyncio.set_event_loop(loop) + + try: + cust = list(confcust.get_customers())[0] + task = TaskCsvDowntimes(loop, logger, sys.argv[0], globopts, + webapi_opts, confcust, + confcust.get_custname(cust), feed, + current_date, uidservtype, args.date[0], + timestamp) + loop.run_until_complete(task.run()) + + except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc: + logger.error(repr(exc)) + loop.run_until_complete( + write_state(sys.argv[0], globopts, confcust, timestamp, False) + ) + + loop.close() + +if __name__ == '__main__': + main() diff --git a/modules/config.py b/modules/config.py index dfffbece..106aa488 100644 --- a/modules/config.py +++ b/modules/config.py @@ -58,6 +58,10 @@ def __init__(self, caller, confpath=None, **kwargs): self._merge_dict(self.shared_secopts, self.conf_topo_schemas, self.conf_topo_output), + 'downtimes-csv-connector.py': + self._merge_dict(self.shared_secopts, + self.conf_downtimes_schemas, + self.conf_downtimes_output), 'downtimes-gocdb-connector.py': self._merge_dict(self.shared_secopts, self.conf_downtimes_schemas, @@ -209,6 +213,7 @@ class CustomerConf(object): 'topology-provider-connector.py': [''], 'metricprofile-webapi-connector.py': ['MetricProfileNamespace'], 'downtimes-gocdb-connector.py': ['DowntimesFeed', 'TopoUIDServiceEndpoints'], + 'downtimes-csv-connector.py': ['DowntimesFeed', 'TopoUIDServiceEndpoints'], 'weights-vapor-connector.py': ['WeightsFeed', 'TopoFetchType'], 'service-types-gocdb-connector.py': ['ServiceTypesFeed'], @@ -266,6 +271,7 @@ def parse(self): topofeedservicegroups = config.get(section, 'TopoFeedServiceGroups', fallback=None) topofeedpaging = config.get(section, 'TopoFeedPaging', fallback='GOCDB') servicetypesfeed = config.get(section, 'ServiceTypesFeed', fallback=None) + downtimesfeed = config.get(section, 'DowntimesFeed', fallback=None) if not custdir.endswith('/'): custdir = '{}/'.format(custdir) @@ -286,17 +292,18 @@ def parse(self): self._cust.update({section: {'Jobs': custjobs, 'OutputDir': custdir, 'Name': custname, - 'TopoFetchType': topofetchtype, - 'TopoFeedPaging': topofeedpaging, - 'TopoScope': toposcope, + 'DowntimesFeed': downtimesfeed, + 'ServiceTypesFeed': servicetypesfeed, 'TopoFeed': topofeed, - 'TopoFeedSites': topofeedsites, - 'TopoFeedServiceGroups': topofeedservicegroups, 'TopoFeedEndpoints': topofeedendpoints, 'TopoFeedEndpointsExtensions': topofeedendpointsextensions, - 'TopoUIDServiceEnpoints': topouidservendpoints, + 'TopoFeedPaging': topofeedpaging, + 'TopoFeedServiceGroups': topofeedservicegroups, + 'TopoFeedSites': topofeedsites, + 'TopoFetchType': topofetchtype, + 'TopoScope': toposcope, 'TopoType': topotype, - 'ServiceTypesFeed': servicetypesfeed + 'TopoUIDServiceEnpoints': topouidservendpoints }}) if optopts: auth, webapi, empty_data, bdii = {}, {}, {}, {} @@ -504,6 +511,9 @@ def _get_cust_options(self, opt): target_option = options[option] return target_option + def get_downfeed(self): + return self._get_cust_options('DowntimesFeed') + def get_topofeed(self): return self._get_cust_options('TopoFeed') diff --git a/modules/io/webapi.py b/modules/io/webapi.py index d4e7d272..b23cada8 100644 --- a/modules/io/webapi.py +++ b/modules/io/webapi.py @@ -9,6 +9,7 @@ class WebAPI(object): methods = { + 'downtimes-csv-connector.py': 'downtimes', 'downtimes-gocdb-connector.py': 'downtimes', 'topology-gocdb-connector.py': 'topology', 'topology-csv-connector.py': 'topology', diff --git a/modules/parse/flat_downtimes.py b/modules/parse/flat_downtimes.py new file mode 100644 index 00000000..018a65f8 --- /dev/null +++ b/modules/parse/flat_downtimes.py @@ -0,0 +1,55 @@ +import datetime +import xml.dom.minidom + +from xml.parsers.expat import ExpatError + +from argo_connectors.exceptions import ConnectorParseError +from argo_connectors.parse.base import ParseHelpers +from argo_connectors.utils import construct_fqdn +from argo_connectors.utils import module_class_name + + +class ParseDowntimes(ParseHelpers): + def __init__(self, logger, data, current_date, uid=False): + self.logger = logger + self.data = self.csv_to_json(data) + self.start = current_date + self.end = current_date.replace(hour=23, minute=59, second=59) + self.uid = uid + + def get_data(self): + downtimes = list() + + for downtime in self.data: + entry = dict() + + service_id = downtime['unique_id'] + classification = downtime['Severity'] + if not service_id or classification != 'OUTAGE': + continue + + hostname = construct_fqdn(downtime['url']) + service_type = downtime['service_type'] + start_time = datetime.datetime.strptime(downtime['start_time'], "%m/%d/%Y %H:%M") + end_time = datetime.datetime.strptime(downtime['end_time'], "%m/%d/%Y %H:%M") + + if self.uid: + entry['hostname'] = '{0}_{1}'.format(hostname, service_id) + else: + entry['hostname'] = hostname + + start_date = start_time.replace(hour=0, minute=0, second=0) + end_date = end_time.replace(hour=0, minute=0, second=0) + if self.start >= start_date and self.start <= end_date: + if start_time < self.start: + start_time = self.start + if end_time > self.end: + end_time = self.end + + entry['service'] = service_type + entry['start_time'] = start_time.strftime('%Y-%m-%dT%H:%M:00Z') + entry['end_time'] = end_time.strftime('%Y-%m-%dT%H:%M:00Z') + + downtimes.append(entry) + + return downtimes diff --git a/modules/tasks/flat_downtimes.py b/modules/tasks/flat_downtimes.py new file mode 100644 index 00000000..1dd8c326 --- /dev/null +++ b/modules/tasks/flat_downtimes.py @@ -0,0 +1,77 @@ +import os + +from urllib.parse import urlparse + +from argo_connectors.exceptions import ConnectorHttpError, ConnectorParseError +from argo_connectors.io.http import SessionWithRetry +from argo_connectors.io.webapi import WebAPI +from argo_connectors.parse.flat_downtimes import ParseDowntimes +from argo_connectors.tasks.common import write_state, write_downtimes_avro as write_avro + + +class TaskCsvDowntimes(object): + def __init__(self, loop, logger, connector_name, globopts, webapi_opts, + confcust, custname, feed, current_date, + uidservtype, targetdate, timestamp): + self.event_loop = loop + self.logger = logger + self.connector_name = connector_name + self.globopts = globopts + self.webapi_opts = webapi_opts + self.confcust = confcust + self.custname = custname + self.feed = feed + self.current_date = current_date + self.uidservtype = uidservtype + self.targetdate = targetdate + self.timestamp = timestamp + + async def fetch_data(self): + session = SessionWithRetry(self.logger, + os.path.basename(self.connector_name), + self.globopts) + res = await session.http_get(self.feed) + + return res + + def parse_source(self, res): + csv_downtimes = ParseDowntimes(self.logger, res, self.current_date, + self.uidservtype) + return csv_downtimes.get_data() + + async def send_webapi(self, dts): + webapi = WebAPI(self.connector_name, self.webapi_opts['webapihost'], + self.webapi_opts['webapitoken'], self.logger, + int(self.globopts['ConnectionRetry'.lower()]), + int(self.globopts['ConnectionTimeout'.lower()]), + int(self.globopts['ConnectionSleepRetry'.lower()]), + date=self.targetdate) + await webapi.send(dts, downtimes_component=True) + + async def run(self): + try: + write_empty = self.confcust.send_empty(self.connector_name) + if not write_empty: + res = await self.fetch_data() + dts = self.parse_source(res) + else: + dts = [] + + await write_state(self.connector_name, self.globopts, self.confcust, self.timestamp, True) + + if eval(self.globopts['GeneralPublishWebAPI'.lower()]): + await self.send_webapi(dts) + + # we don't have multiple tenant definitions in one + # customer file so we can safely assume one tenant/customer + if dts or write_empty: + cust = list(self.confcust.get_customers())[0] + self.logger.info('Customer:%s Fetched Date:%s Endpoints:%d' % + (self.confcust.get_custname(cust), self.targetdate, len(dts))) + + if eval(self.globopts['GeneralWriteAvro'.lower()]): + write_avro(self.logger, self.globopts, self.confcust, dts, self.timestamp) + + except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc: + self.logger.error(repr(exc)) + await write_state(self.connector_name, self.globopts, self.confcust, self.timestamp, False) diff --git a/setup.py b/setup.py index d6e07be0..3a3011ac 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ def get_ver(): setup(name=NAME, version=get_ver(), author='SRCE', - author_email='dvrcic@srce.hr, kzailac@srce.hr', + author_email='dvrcic@srce.hr', description='Components generate input data for ARGO Compute Engine', classifiers=[ "Development Status :: 5 - Production/Stable", @@ -36,14 +36,17 @@ def get_ver(): 'argo_connectors.parse', 'argo_connectors.mesh', 'argo_connectors.tasks'], data_files=[('/etc/argo-connectors', glob.glob('etc/*.conf.template')), - ('/usr/libexec/argo-connectors', ['exec/downtimes-gocdb-connector.py', - 'exec/metricprofile-webapi-connector.py', - 'exec/topology-provider-connector.py', - 'exec/topology-gocdb-connector.py', - 'exec/service-types-gocdb-connector.py', - 'exec/service-types-json-connector.py', - 'exec/service-types-csv-connector.py', - 'exec/topology-json-connector.py', - 'exec/weights-vapor-connector.py', - 'exec/topology-csv-connector.py']), + ('/usr/libexec/argo-connectors', [ + 'exec/downtimes-csv-connector.py', + 'exec/downtimes-gocdb-connector.py', + 'exec/metricprofile-webapi-connector.py', + 'exec/service-types-csv-connector.py', + 'exec/service-types-gocdb-connector.py', + 'exec/service-types-json-connector.py', + 'exec/topology-csv-connector.py', + 'exec/topology-gocdb-connector.py', + 'exec/topology-json-connector.py', + 'exec/topology-provider-connector.py', + 'exec/weights-vapor-connector.py', + ]), ('/etc/argo-connectors/schemas', glob.glob('etc/schemas/*.avsc'))]) diff --git a/tests/sample-downtimes.csv b/tests/sample-downtimes.csv new file mode 100644 index 00000000..84e6e4d0 --- /dev/null +++ b/tests/sample-downtimes.csv @@ -0,0 +1,42 @@ +unique_id,url,start_time,end_time,service_type,Severity,Description +neanias_5,https://atmo-flud.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,Webservice,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_7,https://atmo-stress.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_9,https://caesar.neanias.eu:443,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_10,http://caesar-api.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_14,https://bathyprocessing.neanias.eu/hub/login,2/21/2022 8:00,2/22/2022 19:00,JupyterHub,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_15,https://uw-map.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_16,https://uw-map.neanias.eu/tos,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_17,https://uw-map.neanias.eu/privacy,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_18,https://uw-mos.neanias.eu/api/try,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_20,https://ai-gateway.neanias.eu/hub/login,2/21/2022 8:00,2/22/2022 19:00,JupyterHub,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_21,https://accounting.neanias.eu/api/api/accounting-service/version-info/current,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_25,https://logging.neanias.eu/api/status,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_26,https://minio.ml-models.neanias.eu/minio/health/live,2/21/2022 8:00,2/22/2022 19:00,MinIO,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_27,https://notification.neanias.eu/api/notification/version-info/current,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_28,https://pms.neanias.eu/,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_30,https://vis-gateway.neanias.eu/hub/login,2/21/2022 8:00,2/22/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +,,,,,, +neanias_5,https://atmo-flud.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_7,https://atmo-stress.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_9,https://caesar.neanias.eu:443,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_10,http://caesar-api.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_14,https://bathyprocessing.neanias.eu/hub/login,3/1/2022 8:00,3/4/2022 19:00,JupyterHub,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_15,https://uw-map.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_16,https://uw-map.neanias.eu/tos,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_17,https://uw-map.neanias.eu/privacy,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_18,https://uw-mos.neanias.eu/api/try,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_20,https://ai-gateway.neanias.eu/hub/login,3/1/2022 8:00,3/4/2022 19:00,JupyterHub,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_21,https://accounting.neanias.eu/api/api/accounting-service/version-info/current,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_25,https://logging.neanias.eu/api/status,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_26,https://minio.ml-models.neanias.eu/minio/health/live,3/1/2022 8:00,3/4/2022 19:00,MinIO,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_27,https://notification.neanias.eu/api/notification/version-info/current,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_28,https://pms.neanias.eu/,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_30,https://vis-gateway.neanias.eu/hub/login,3/1/2022 8:00,3/4/2022 19:00,WebService,OUTAGE,GARR Cloud garr-ct1 region maintenance +neanias_2,https://files.neanias.eu,2/28/2022 13:00,2/28/2022 15:00,nextcloud,OUTAGE,Data sharing service maintenance +neanias_1,https://files.dev.neanias.eu,2/28/2022 9:30,2/28/2022 11:00,nextcloud,OUTAGE,Data sharing service maintenance +neanias_1,https://files.dev.neanias.eu,2/7/2022 8:00,2/17/2022 19:00,nextcloud,OUTAGE,GARR Cloud garr-pa1 region maintenance +neanias_2,https://files.neanias.eu,2/7/2022 8:00,2/17/2022 19:00,nextcloud,OUTAGE,GARR Cloud garr-pa1 region maintenance +neanias_4,https://atmo-4cast.neanias.eu/api/forecaststatus/1,2/7/2022 8:00,2/17/2022 19:00,WebService,OUTAGE,GARR Cloud garr-pa1 region maintenance +neanias_6,http://atmo-seism.neanias.eu/hub/login,2/7/2022 8:00,2/17/2022 19:00,JupyterHub,OUTAGE,GARR Cloud garr-pa1 region maintenance +neanias_11,http://vlkb.neanias.eu:8080/vlkb/tap/availability,2/7/2022 8:00,2/17/2022 19:00,WebService,OUTAGE,GARR Cloud garr-pa1 region maintenance +neanias_29,https://vd-maps.neanias.eu,2/7/2022 8:00,2/17/2022 19:00,WebService,OUTAGE,GARR Cloud garr-pa1 region maintenance \ No newline at end of file diff --git a/tests/test_asynctasks.py b/tests/test_asynctasks.py index 3518452b..415a55ab 100644 --- a/tests/test_asynctasks.py +++ b/tests/test_asynctasks.py @@ -7,10 +7,11 @@ import mock from argo_connectors.exceptions import ConnectorError, ConnectorParseError, ConnectorHttpError +from argo_connectors.tasks.flat_downtimes import TaskCsvDowntimes +from argo_connectors.tasks.flat_servicetypes import TaskFlatServiceTypes from argo_connectors.tasks.gocdb_servicetypes import TaskGocdbServiceTypes -from argo_connectors.tasks.provider_topology import TaskProviderTopology from argo_connectors.tasks.gocdb_topology import TaskGocdbTopology -from argo_connectors.tasks.flat_servicetypes import TaskFlatServiceTypes +from argo_connectors.tasks.provider_topology import TaskProviderTopology from argo_connectors.parse.base import ParseHelpers @@ -202,6 +203,79 @@ async def test_StepsFailedRun(self, mock_writestate): self.assertTrue(self.services_gocdb.logger.error.call_args[0][0], repr(ConnectorHttpError('fetch_data failed'))) self.assertFalse(self.services_gocdb.send_webapi.called) +class DowntimesCsv(unittest.TestCase): + def setUp(self): + logger = mock.Mock() + logger.customer = CUSTOMER_NAME + self.loop = asyncio.get_event_loop() + mocked_globopts = dict(generalpublishwebapi='True', + generalwriteavro='True', + outputdowntimes='downtimes_DATE.avro', + avroschemasdowntimes='downtimes.avsc') + globopts = mocked_globopts + webapiopts = mock.Mock() + authopts = mock.Mock() + confcust = mock.Mock() + confcust.send_empty.return_value = False + confcust.get_customers.return_value = ['CUSTOMERFOO', 'CUSTOMERBAR'] + confcust.get_custdir.return_value = '/some/path' + custname = CUSTOMER_NAME + feed = 'https://downtimes-csv.com/api/fetch' + timestamp = datetime.datetime.now().strftime('%Y_%m_%d') + current_date = datetime.datetime.now() + self.downtimes_flat = TaskCsvDowntimes( + self.loop, + logger, + 'test_asynctasks_downtimesflat', + globopts, + webapiopts, + confcust, + custname, + feed, + current_date, + True, + current_date, + timestamp + ) + self.maxDiff = None + + @mock.patch('argo_connectors.tasks.flat_downtimes.write_avro') + @mock.patch('argo_connectors.tasks.flat_downtimes.write_state') + @async_test + async def test_StepsSuccessRun(self, mock_writestate, mock_writeavro): + self.downtimes_flat.fetch_data = mock.AsyncMock() + self.downtimes_flat.fetch_data.side_effect = ['data_downtimes'] + self.downtimes_flat.send_webapi = mock.AsyncMock() + self.downtimes_flat.parse_source = mock.MagicMock() + await self.downtimes_flat.run() + self.assertTrue(self.downtimes_flat.fetch_data.called) + self.assertTrue(self.downtimes_flat.parse_source.called) + self.downtimes_flat.parse_source.assert_called_with('data_downtimes') + self.assertEqual(mock_writestate.call_args[0][0], 'test_asynctasks_downtimesflat') + self.assertEqual(mock_writestate.call_args[0][3], self.downtimes_flat.timestamp) + self.assertTrue(mock_writestate.call_args[0][4]) + self.assertTrue(mock_writeavro.called, True) + self.assertEqual(mock_writeavro.call_args[0][4], datetime.datetime.now().strftime('%Y_%m_%d')) + self.assertTrue(self.downtimes_flat.send_webapi.called) + self.assertTrue(self.downtimes_flat.logger.info.called) + + @mock.patch('argo_connectors.tasks.flat_downtimes.write_state') + @async_test + async def test_StepsFailedRun(self, mock_writestate): + self.downtimes_flat.fetch_data = mock.AsyncMock() + self.downtimes_flat.fetch_data.side_effect = [ConnectorHttpError('fetch_data failed')] + self.downtimes_flat.send_webapi = mock.AsyncMock() + self.downtimes_flat.parse_source = mock.MagicMock() + await self.downtimes_flat.run() + self.assertTrue(self.downtimes_flat.fetch_data.called) + self.assertFalse(self.downtimes_flat.parse_source.called) + self.assertEqual(mock_writestate.call_args[0][0], 'test_asynctasks_downtimesflat') + self.assertEqual(mock_writestate.call_args[0][3], self.downtimes_flat.timestamp) + self.assertFalse(mock_writestate.call_args[0][4]) + self.assertTrue(self.downtimes_flat.logger.error.called) + self.assertTrue(self.downtimes_flat.logger.error.call_args[0][0], repr(ConnectorHttpError('fetch_data failed'))) + self.assertFalse(self.downtimes_flat.send_webapi.called) + class ServiceTypesFlat(unittest.TestCase): def setUp(self): diff --git a/tests/test_downfeed.py b/tests/test_downfeed.py new file mode 100644 index 00000000..1413d363 --- /dev/null +++ b/tests/test_downfeed.py @@ -0,0 +1,88 @@ +import datetime +import mock +import unittest + +from argo_connectors.log import Logger +from argo_connectors.parse.flat_downtimes import ParseDowntimes +from argo_connectors.exceptions import ConnectorParseError + +CUSTOMER_NAME = 'CUSTOMERFOO' + + +class ParseCsvDowntimes(unittest.TestCase): + def setUp(self): + with open('tests/sample-downtimes.csv', encoding='utf-8') as feed_file: + self.downtimes = feed_file.read() + self.maxDiff = None + logger = mock.Mock() + logger.customer = CUSTOMER_NAME + self.logger = logger + + def test_parseDowntimes(self): + date_2_21_2022 = datetime.datetime(2022, 2, 21) + flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_2_21_2022, True) + downtimes = flat_downtimes.get_data() + self.assertEqual(len(downtimes), 16) + first_schedule = downtimes[0] + self.assertEqual(first_schedule['hostname'], 'atmo-flud.neanias.eu_neanias_5') + start_time = datetime.datetime.strptime(first_schedule['start_time'], '%Y-%m-%dT%H:%M:00Z') + end_time = datetime.datetime.strptime(first_schedule['end_time'], '%Y-%m-%dT%H:%M:00Z') + self.assertEqual(start_time, datetime.datetime(2022, 2, 21, 8, 0)) + self.assertEqual(end_time, datetime.datetime(2022, 2, 21, 23, 59)) + + date_2_22_2022 = datetime.datetime(2022, 2, 22) + flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_2_22_2022, True) + downtimes = flat_downtimes.get_data() + self.assertEqual(len(downtimes), 16) + first_schedule = downtimes[0] + self.assertEqual(first_schedule['hostname'], 'atmo-flud.neanias.eu_neanias_5') + start_time = datetime.datetime.strptime(first_schedule['start_time'], '%Y-%m-%dT%H:%M:00Z') + end_time = datetime.datetime.strptime(first_schedule['end_time'], '%Y-%m-%dT%H:%M:00Z') + self.assertEqual(start_time, datetime.datetime(2022, 2, 22, 0, 0)) + self.assertEqual(end_time, datetime.datetime(2022, 2, 22, 19, 0)) + + date_3_1_2022 = datetime.datetime(2022, 3, 1) + flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_3_1_2022, True) + downtimes = flat_downtimes.get_data() + self.assertEqual(len(downtimes), 16) + first_schedule = downtimes[0] + self.assertEqual(first_schedule['hostname'], 'atmo-flud.neanias.eu_neanias_5') + start_time = datetime.datetime.strptime(first_schedule['start_time'], '%Y-%m-%dT%H:%M:00Z') + end_time = datetime.datetime.strptime(first_schedule['end_time'], '%Y-%m-%dT%H:%M:00Z') + self.assertEqual(start_time, datetime.datetime(2022, 3, 1, 8, 0)) + self.assertEqual(end_time, datetime.datetime(2022, 3, 1, 23, 59)) + + date_3_2_2022 = datetime.datetime(2022, 3, 2) + flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_3_2_2022, True) + downtimes = flat_downtimes.get_data() + self.assertEqual(len(downtimes), 16) + first_schedule = downtimes[0] + self.assertEqual(first_schedule['hostname'], 'atmo-flud.neanias.eu_neanias_5') + start_time = datetime.datetime.strptime(first_schedule['start_time'], '%Y-%m-%dT%H:%M:00Z') + end_time = datetime.datetime.strptime(first_schedule['end_time'], '%Y-%m-%dT%H:%M:00Z') + self.assertEqual(start_time, datetime.datetime(2022, 3, 2, 0, 0)) + self.assertEqual(end_time, datetime.datetime(2022, 3, 2, 23, 59)) + + date_3_4_2022 = datetime.datetime(2022, 3, 4) + flat_downtimes = ParseDowntimes(self.logger, self.downtimes, date_3_4_2022, True) + downtimes = flat_downtimes.get_data() + self.assertEqual(len(downtimes), 16) + first_schedule = downtimes[0] + self.assertEqual(first_schedule['hostname'], 'atmo-flud.neanias.eu_neanias_5') + start_time = datetime.datetime.strptime(first_schedule['start_time'], '%Y-%m-%dT%H:%M:00Z') + end_time = datetime.datetime.strptime(first_schedule['end_time'], '%Y-%m-%dT%H:%M:00Z') + self.assertEqual(start_time, datetime.datetime(2022, 3, 4, 0, 0)) + self.assertEqual(end_time, datetime.datetime(2022, 3, 4, 19, 0)) + + def test_failedParseDowntimes(self): + date_2_21_2022 = datetime.datetime(2022, 2, 21) + with self.assertRaises(ConnectorParseError) as cm: + flat_downtimes = ParseDowntimes(self.logger, 'DUMMY DATA', date_2_21_2022, False) + downtimes = flat_downtimes.get_data() + + excep = cm.exception + self.assertTrue('CSV feed' in excep.msg) + self.assertTrue(CUSTOMER_NAME in excep.msg) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_servicetypefeed.py b/tests/test_servicetypefeed.py index e2c56574..520e0b8e 100644 --- a/tests/test_servicetypefeed.py +++ b/tests/test_servicetypefeed.py @@ -10,6 +10,7 @@ CUSTOMER_NAME = 'CUSTOMERFOO' + class ParseGocdb(unittest.TestCase): def setUp(self): with open('tests/sample-service_types_gocdb.xml', encoding='utf-8') as feed_file: