diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..47d8366f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,43 @@ +# Changelog + +## [2.0.0] - 2022-02-10 + +* ARGO-3375 Pass contacts for each topology entity +* ARGO-3427 Fetch HOSTDN service attribute from GOCDB +* ARGO-3428 Fetch SE_PATH from BDII +* ARGO-3448 NEANIAS dulpicates in group topology +* ARGO-3503 Support multiple VO SE_PATHs for host +* ARGO-3521 Retry on empty responses +* ARGO-3522 Introduce optional scope key for topology fetch and BIOMED feed parse +* ARGO-3524 Improve exception handling +* ARGO-3528 Verbose retry log messages +* ARGO-3540 Tests for retry logic + +### Added + +* ARGO-2620 Update connectors configuration templates +* ARGO-2622 Populate group_type field for WEB-API weights +* ARGO-3064 Introduce async topology CSV +* ARGO-3215 Pass EXTENSIONS bucket in new connectors +* ARGO-3295 Add GOCDB URLs +* ARGO-3301 Query BDII and extract SRM port +* ARGO-3335 Retry on LDAP queries +* ARGO-3340 Unit testing of parsing of GOCDB service endpoint +* ARGO-3341 Pass date argument to WEB-API methods + +### Fixed + +* ARGO-2621 Detect missing path separator for tenant directory avros +* ARGO-2660 Basic HTTP authentication not correctly set up +* ARGO-2681 Weights use HTTP PUT to write new daily entries +* ARGO-2771 Weight connector issues investigated +* ARGO-3268 Downtimes DELETE with explicit date passed + +### Changed + +* ARGO-2591 Connectors CentOS7 and Py3 migrate +* ARGO-2596 Push sync_data to WEB-API +* ARGO-2619 Remove Centos 6 RPM building +* ARGO-2656 Downtimes as global tenant resource +* ARGO-2860 Separate connection, parsing and file handling +* ARGO-2861 Switch to async IO operations diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 995ea6c2..00000000 --- a/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM centos:6.9 -MAINTAINER Themis Zamani themiszamani@gmail.com - -RUN yum -y install epel-release -RUN yum -y makecache; yum -y update -RUN yum install -y \ - gcc \ - git \ - libffi \ - libffi-devel \ - modules \ - openssl-devel \ - python \ - python-argparse \ - python-devel \ - python-pip \ - python-requests \ - tar \ - wget -RUN pip install \ - argo_ams_library \ - avro \ - cffi \ - coverage==4.5.4 \ - cryptography==2.1.4 \ - discover \ - httmock \ - mock==2.0.0 \ - pyOpenSSL \ - setuptools \ - unittest2 diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..539e4f3b --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,77 @@ +pipeline { + agent any + options { + checkoutToSubdirectory('argo-egi-connectors') + } + environment { + PROJECT_DIR="argo-egi-connectors" + GIT_COMMIT=sh(script: "cd ${WORKSPACE}/$PROJECT_DIR && git log -1 --format=\"%H\"",returnStdout: true).trim() + GIT_COMMIT_HASH=sh(script: "cd ${WORKSPACE}/$PROJECT_DIR && git log -1 --format=\"%H\" | cut -c1-7",returnStdout: true).trim() + GIT_COMMIT_DATE=sh(script: "date -d \"\$(cd ${WORKSPACE}/$PROJECT_DIR && git show -s --format=%ci ${GIT_COMMIT_HASH})\" \"+%Y%m%d%H%M%S\"",returnStdout: true).trim() + } + stages { + stage ('Build'){ + parallel { + stage ('Build Centos 7 RPM') { + agent { + docker { + image 'argo.registry:5000/epel-7-ams' + args '-u jenkins:jenkins' + } + } + steps { + echo 'Building Rpm...' + withCredentials(bindings: [sshUserPrivateKey(credentialsId: 'jenkins-rpm-repo', usernameVariable: 'REPOUSER', \ + keyFileVariable: 'REPOKEY')]) { + sh "/home/jenkins/build-rpm.sh -w ${WORKSPACE} -b ${BRANCH_NAME} -d centos7 -p ${PROJECT_DIR} -s ${REPOKEY}" + } + archiveArtifacts artifacts: '**/*.rpm', fingerprint: true + } + post { + always { + cleanWs() + } + } + } + stage ('Execute tests') { + agent { + docker { + image 'argo.registry:5000/epel-7-ams' + args '-u jenkins:jenkins' + } + } + steps { + sh ''' + cd $WORKSPACE/$PROJECT_DIR/ + rm -f tests/argo_egi_connectors + ln -s $PWD/modules/ tests/argo_egi_connectors + coverage run -m xmlrunner discover --output-file junit.xml -v tests/ + coverage xml + ''' + cobertura coberturaReportFile: 'coverage.xml' + junit 'junit.xml' + } + } + } + } + } + post { + always { + cleanWs() + } + success { + script{ + if ( env.BRANCH_NAME == 'master' || env.BRANCH_NAME == 'devel' ) { + slackSend( message: ":rocket: New version for <$BUILD_URL|$PROJECT_DIR>:$BRANCH_NAME Job: $JOB_NAME !") + } + } + } + failure { + script{ + if ( env.BRANCH_NAME == 'master' || env.BRANCH_NAME == 'devel' ) { + slackSend( message: ":rain_cloud: Build Failed for <$BUILD_URL|$PROJECT_DIR>:$BRANCH_NAME Job: $JOB_NAME") + } + } + } + } +} diff --git a/Makefile b/Makefile index 8e73a476..f5cf2df7 100644 --- a/Makefile +++ b/Makefile @@ -4,14 +4,14 @@ SPECFILE=${PKGNAME}.spec PKGVERSION=$(shell grep -s '^Version:' $(SPECFILE) | sed -e 's/Version: *//') srpm: dist - rpmbuild -ts --define='dist .el6' ${PKGNAME}-${PKGVERSION}.tar.gz + rpmbuild -ts --define='dist .el7' ${PKGNAME}-${PKGVERSION}.tar.gz rpm: dist rpmbuild -ta ${PKGNAME}-${PKGVERSION}.tar.gz dist: rm -rf dist - python setup.py sdist + python3 setup.py sdist mv dist/${PKGNAME}-${PKGVERSION}.tar.gz . rm -rf dist diff --git a/README.md b/README.md index cab19092..f6f02ba1 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,21 @@ -**argo-egi-connectors** is a bundle of connectors/sync components for various data sources established in EGI infrastructure, most notably GOCDB (topology, downtimes), but there's also support for alternative topology fetching for virtual organizations, weights factors provided within a VAPOR service and POEM profiles information. +`argo-connectors` is bundle of data fetch components that gather insights in +various e-infrastructures for the ARGO monitoring purposes. -Data is synced on a daily basis and all together with a prefiltered raw metric results coming from _argo-egi-consumer_ represents an input for _argo-compute-engine_. Data is written in binary avro formatted files which are grouped into a directory structure that represents set of customers and jobs at compute side. +It primarily deals with hierarchical topology from GOCDB compatible sources, +but also flat topologies defined in simple CSV or JSON format are supported. +Topology lists endpoints with asociated services running of them that needs to +be inspected as well as contact informations that will be used for alerts. In +case of hierarchical topology, endpoints are organized in sites or +servicegroups which themselves are organized in NGIs or projects. -Configuration of connectors is centered around two configuration files: -- `global.conf` - used for defining some common information like connection and authentication parameters, avro schemas and output filenames -- `customer.conf` - used for listing jobs for EGI customer and defining new customers and theirs jobs residing in EGI infrastructure. Customers and jobs have attributes that control the behaviour of related connector and also create implicit directory structure. +Next to topology, `argo-connectors` also gather weights and downtimes data. +Weights are Computation Power figures fetched from VAPOR service for each SITE +that are used for A/R calculation of interested NGI. Downtimes data are time +period scheduled for each endpoint under maintenance that will not be taken +into account during A/R calculation. + +All collected data is transformed in appropriate format and pushed on daily +basis on the corresponding API method of ARGO-WEB-API service. More info: http://argoeu.github.io/guides/sync/ + diff --git a/argo-egi-connectors.spec b/argo-egi-connectors.spec index 4838db31..685f157f 100644 --- a/argo-egi-connectors.spec +++ b/argo-egi-connectors.spec @@ -1,18 +1,29 @@ -Name: argo-egi-connectors -Version: 1.7.4 +# override so that bytecode compiling is called with python3 +%global __python /usr/bin/python3 + +Name: argo-egi-connectors +Version: 2.0.0 Release: 1%{?dist} -Group: EGI/SA4 +Group: EGI/SA4 License: ASL 2.0 -Summary: Components generate input for ARGO Compute Engine -Url: http://argoeu.github.io/guides/sync/ -Vendor: SRCE , SRCE +Summary: Components fetch and transform data that represents input for ARGO Compute Engine +Url: http://argoeu.github.io/guides/sync/ +Vendor: SRCE , SRCE Obsoletes: ar-sync -Prefix: %{_prefix} -Requires: argo-ams-library -Requires: avro -Requires: python-requests -Requires: python2-ndg_httpsclient +Prefix: %{_prefix} + +Requires: python3-aiofiles +Requires: python3-aiohttp +Requires: python3-attrs +Requires: python3-avro +Requires: python3-requests +Requires: python3-typing-extensions +Requires: python3-uvloop +Requires: python3-bonsai + +BuildRequires: python3-devel python3-setuptools + Source0: %{name}-%{version}.tar.gz BuildArch: noarch @@ -23,13 +34,13 @@ Installs the components for syncing ARGO Compute Engine with GOCDB, VAPOR and POEM definitions per day. %prep -%setup -n %{name}-%{version} +%setup -q %build -python setup.py build +%{py3_build} %install -python setup.py install -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES +%{py3_install "--record=INSTALLED_FILES" } install --directory %{buildroot}/%{_sharedstatedir}/argo-connectors/ install --directory %{buildroot}/%{_localstatedir}/log/argo-connectors/ install --directory %{buildroot}/%{_libexecdir}/argo-egi-connectors/ @@ -40,257 +51,12 @@ rm -rf $RPM_BUILD_ROOT %files -f INSTALLED_FILES %config(noreplace) /etc/argo-egi-connectors/* %attr(0755,root,root) %dir %{_libexecdir}/argo-egi-connectors/ -%attr(0755,root,root) %{_libexecdir}/argo-egi-connectors/*.py* +%attr(0755,root,root) %{_libexecdir}/argo-egi-connectors/*.py +%attr(0755,root,root) %{_libexecdir}/argo-egi-connectors/__pycache__/* %attr(0755,root,root) %dir %{_sharedstatedir}/argo-connectors/ %attr(0755,root,root) %dir %{_localstatedir}/log/argo-connectors/ %changelog -* Mon Mar 30 2020 Daniel Vrcic - 1.7.4-1%{dist} -- ARGO-2247 Pass URL from EOSC topology -- ARGO-2225 Support for creating empty weights and downtimes data -- ARGO-2221 Metric profile namespace optional -- ARGO-2210 Introduce topology connector for EOSC-PORTAL -- ARGO-2209 Pass PRIMARY_KEY of GOCDB service endpoint as a unique service endpoint identifier -* Fri Nov 8 2019 Daniel Vrcic - 1.7.3-1%{?dist} -- ARGO-2017 - Token per tenants config option -- ARGO-2013 - Metric profiles WEB-API connector -- ARGO-1549 - New helper tool that can replay avro data on AMS with customizable datestamp -- ARGO-1575 - Switch poem-connector to use new token protected POEM API -* Wed Feb 20 2019 Daniel Vrcic - 1.7.2-1%{?dist} -- ARGO-1674 Use requests library in connectors -* Fri Nov 30 2018 Daniel Vrcic , Katarina Zailac - 1.7.1-1%{?dist} -- ARGO-1428 ServiceGroup topology filtering -- ARGO-1370 Optimize connectors queries to POEM -- ARGO-1269 Refactor poem-connector -- ARGO-1236 Datestamp of AMS msg does not match corresponding avro filename -* Wed May 23 2018 Daniel Vrcic - 1.7.0-1%{?dist} -- ARGO-1093 Support for GOCDB paginated topology API -- ARGO-1080 add support for basic-auth in Connectors -- ARGO-966 Lower state files permissions -* Tue Mar 27 2018 Daniel Vrcic - 1.6.1-1%{?dist} -- selectively use GOCDB paginated API for topology -* Thu Nov 30 2017 Daniel Vrcic - 1.6.0-1%{?dist} -- ARGO-965 Support for packing connectors data in a single AMS message -- ARGO-921 Use ComputationPower instead of HEPSPEC2006 value for weights -- ARGO-906 No explicit exit on connection problem so state file will be written -- ARGO-886 Finer retry logic -- ARGO-872 Tenant and jobname in retries log msgs -- ARGO-853 Connectors retry to fetch data -- ARGO-843 Write/send data as it is data for passed date -- ARGO-842 Connectors dedicated file logger -- ARGO-549 Use of AMS for delivering topology, downtimes, POEM profile and weights -- added unit tests -* Tue Apr 25 2017 Daniel Vrcic - 1.5.9-1%{?dist} -- ARGO-724 Each connector must try to create states directory structure -* Wed Mar 29 2017 Daniel Vrcic - 1.5.8-1%{?dist} -- ARGO-766 Remove SRMv2 service type mapping -* Mon Mar 20 2017 Daniel Vrcic - 1.5.7-1%{?dist} -- ARGO-767 Remove topology-vo connector -- refactored topology filtering -- removed schema migration helper -* Fri Mar 17 2017 Daniel Vrcic - 1.5.6-1%{?dist} -- ARGO-762 Remove inspection logic of HEPSPEC factors for previous days -* Thu Mar 9 2017 Daniel Vrcic - 1.5.5-1%{?dist} -- ARGO-713 Topology connector should be able to pick only particular NGI or site -* Mon Jan 30 2017 Daniel Vrcic - 1.5.4-1%{?dist} -- ARGO-667 filter endpoints whose groups are filtered in groups of groups -* Wed Jan 25 2017 Daniel Vrcic - 1.5.3-3%{?dist} -- prefilter output datestamp with underscores -* Wed Jan 25 2017 Daniel Vrcic - 1.5.3-2%{?dist} -- prefilter datestamp extracted from arg tuple -* Thu Jan 19 2017 Daniel Vrcic - 1.5.3-1%{?dist} -- poem and output files as arguments to prefilter -- refactored filename datestamp creation -* Wed Jan 4 2017 Daniel Vrcic - 1.5.2-1%{?dist} -- ARGO-550 Introduce states that can be checked by Nagios probe -* Thu Nov 24 2016 Daniel Vrcic - 1.5.1-2%{?dist} -- remove code for nagioses defined in obsoleted nagios-roles.conf -- catch JSON parse errors -- catch XML parse errors -* Fri Oct 28 2016 Daniel Vrcic - 1.5.1-1%{?dist} -- ARGO-584 Ensure to catch all exceptions of underlying library -* Sat Sep 24 2016 Themis Zamani - 1.5.0-1%{?dist} -- New RPM package release -* Wed Aug 31 2016 Daniel Vrcic - 1.4.6-2%{?dist} -- make use of VAPOR service for weights -- extended cert verification with CAfile bundle -* Tue Feb 16 2016 Daniel Vrcic - 1.4.6-1%{?dist} -- topology data without mixed int and string values -* Mon Feb 1 2016 Daniel Vrcic - 1.4.5-3%{?dist} -- poem connector optional write data needed for prefilter - https://github.com/ARGOeu/ARGO/issues/184 -* Tue Jan 12 2016 Daniel Vrcic - 1.4.5-2%{?dist} -- weights connector refactored -- README updated - https://github.com/ARGOeu/ARGO/issues/181 -* Sun Jan 10 2016 Daniel Vrcic - 1.4.5-1%{?dist} -- log failed VO and weights connections - https://github.com/ARGOeu/ARGO/issues/179 -- added connection timeout for all connectors -- config files can be passed as arguments to every component - https://github.com/ARGOeu/ARGO/issues/180 -- added connection retry feature forr all connectors -- prefilter explicit input and output -- reorganized prefilter global.conf -- DATE placeholder in global.conf so interpolation can be used -- prefilter poem_sync.out look back option -- remove obsoleted logging -- guide updated -- refactored connection retries -* Thu Oct 15 2015 Daniel Vrcic - 1.4.4-6%{?dist} -- bugfix handling lowercase defined POEM profiles -- remove hardcoded customer name for topology-gocdb-connector - https://github.com/ARGOeu/ARGO/issues/173 -- guide updated with new configuration option for customer -* Thu Oct 8 2015 Daniel Vrcic - 1.4.4-5%{?dist} -- bugfix in case of no downtimes defined for given date - https://github.com/ARGOeu/ARGO/issues/170 -* Wed Oct 7 2015 Daniel Vrcic - 1.4.4-4%{?dist} -- poem-connector urlparse bugfix -* Wed Oct 7 2015 Daniel Vrcic - 1.4.4-3%{?dist} -- grab all distinct scopes for feed -* Tue Oct 6 2015 Daniel Vrcic - 1.4.4-2%{?dist} -- fix initialization of loggers in config parsers -- backward compatible exception messages -* Fri Oct 2 2015 Daniel Vrcic - 1.4.4-1%{?dist} -- filter SRM endpoints too -- refactored use of logging -- connectors can verify server certificate - https://github.com/ARGOeu/ARGO/issues/153 -- report correct number of fetched endpoints even if SRM endpoints were being filtered -- connectors handle help argument and describe basic info and usage - https://github.com/ARGOeu/ARGO/issues/169 -- removed hardcoded scopes and grab them dynamically from config - https://github.com/ARGOeu/ARGO/issues/168 -- report config parser errors via logger -- downtimes connector complain if wrong date specified -- remove notion of default scope -- doc moved to repo -- updated doc with server's cert validate options -* Wed Aug 19 2015 Daniel Vrcic - 1.4.3-3%{?dist} -- fix exception in case of returned HTTP 500 for other connectors -* Sat Aug 15 2015 Daniel Vrcic - 1.4.3-2%{?dist} -- fix poem-connector exception in case of returned HTTP 500 -* Mon Aug 10 2015 Daniel Vrcic - 1.4.3-1%{?dist} -- generate meaningful statistic messages for every connector -- messages are written into syslog - https://github.com/ARGOeu/ARGO/issues/116 -* Wed Jul 15 2015 Daniel Vrcic - 1.4.2-2%{?dist} -- fixed bug with duplicating poem profiles info for prefilter -- fixed bug with SRM service type handling for topology and downtimes connectors -* Tue Jun 23 2015 Daniel Vrcic - 1.4.2-1%{?dist} -- changed internal parser structure to address entities with doubled scope - https://github.com/ARGOeu/ARGO/issues/141 -* Tue Jun 2 2015 Daniel Vrcic - 1.4.1-5%{?dist} -- new path and filename for consumer logs -* Thu May 28 2015 Daniel Vrcic - 1.4.1-4%{?dist} -- migrate.py lower on resources -* Thu May 21 2015 Daniel Vrcic - 1.4.1-3%{?dist} -- migration script to transform old data to new avro schema with map type - https://github.com/ARGOeu/ARGO/issues/134 -* Mon May 18 2015 Daniel Vrcic - 1.4.1-2%{?dist} -- GridPP VO job example -- downtimes filename day timestamp is queried one - https://github.com/ARGOeu/ARGO/issues/133 -* Wed May 6 2015 Daniel Vrcic - 1.4.1-1%{?dist} -- removed VO as an entity in configuration; only customers and set of jobs -- multiple customers in config each with own outputdir -- data feeds for all connectors can be defined per job -- prefilter-egi.py is aware of multiple customers -- avro schemas with generic tags -- case insensitive sections and options -- setup.py with automatic version catch from spec -- new default config - https://github.com/ARGOeu/ARGO/issues/132 -* Fri Apr 17 2015 Daniel Vrcic - 1.4.0-10%{?dist} -- VO jobs are moved under customer's directory -* Wed Apr 8 2015 Daniel Vrcic - 1.4.0-9%{?dist} -- handle group type names with whitespaces -- fixed bug with filtering VO groups across multiple VO jobs -* Fri Apr 3 2015 Daniel Vrcic - 1.4.0-8%{?dist} -- added Dirname optional option for VO config -- correctly renamed avro schemas -* Mon Mar 30 2015 Daniel Vrcic - 1.4.0-7%{?dist} -- added README.md with a basic project info -* Sun Mar 29 2015 Daniel Vrcic - 1.4.0-6%{?dist} -- renamed weights and more configs refactoring -- put scripts back into libexec -* Fri Mar 27 2015 Daniel Vrcic - 1.4.0-5%{?dist} -- minor code cleanups and renamed connectors to reflect the source of data -* Fri Mar 27 2015 Daniel Vrcic - 1.4.0-4%{?dist} -- poem server is defined in its config file, not global one -* Fri Mar 27 2015 Daniel Vrcic - 1.4.0-3%{?dist} -- prefilter-egi.py cleanups and roll back missing file -* Fri Mar 27 2015 Daniel Vrcic - 1.4.0-2%{?dist} -- deleted leftovers -* Fri Mar 27 2015 Daniel Vrcic - 1.4.0-1%{?dist} -- refactor the configuration of connectors/components - https://github.com/ARGOeu/ARGO/issues/114 -- fixed topology connector for VO'es to produce correct GE and GG avro files - https://github.com/ARGOeu/ARGO/issues/121 -- use of distutils for package building -* Tue Feb 17 2015 Daniel Vrcic - 1.3.1-16%{?dist} -- prefilter-avro has fixed configuration -* Thu Feb 12 2015 Daniel Vrcic - 1.3.1-15%{?dist} -- legacy SRM service type handle for downtime syncs -* Tue Feb 10 2015 Daniel Vrcic - 1.3.1-14%{?dist} -- updated .spec with removed configs for a per job prefilter-avro -* Tue Feb 10 2015 Daniel Vrcic - 1.3.1-13%{?dist} -- different internal handle of avro poem-sync so it doesn't contain duplicated entries -- special handle of legacy SRM service type -* Thu Feb 5 2015 Daniel Vrcic - 1.3.1-12%{?dist} -- plaintxt prefilter has fixed configuration -* Tue Feb 3 2015 Daniel Vrcic - 1.3.1-11%{?dist} -- update .spec to deploy new configs -- removed whitespaces at the end of config lines -* Mon Feb 2 2015 Daniel Vrcic - 1.3.1-10%{?dist} -- tools can have config file as their argument -- config files with changed output directory for customer/job -- modified cronjobs for customer and his two jobs -* Thu Jan 29 2015 Daniel Vrcic - 1.3.1-9%{?dist} -- bug fixes for poem-sync and prefilter -- typo in plaintext groups filename -* Mon Jan 19 2015 Daniel Vrcic - 1.3.1-8%{?dist} -- topology-sync: avro schemas updated with tags and filtering by tags values -- poem-sync: avro schema updated with tags -- poem-sync: output profiles per customer and job - https://github.com/ARGOeu/ARGO/issues/85 -* Thu Jan 15 2015 Luko Gjenero - 1.3.1-3%{?dist} -- avro prefiltering -* Wed Dec 17 2014 Daniel Vrcic - 1.3.1-2%{?dist} -- ar-sync is missing avro dependency -- poem-sync is missing data for servers listed in URL -* Thu Nov 27 2014 Luko Gjenero - 1.3.0-0%{?dist} -- Avro format for poem, downtimes, topology and hepspec -* Tue May 13 2014 Paschalis Korosoglou - 1.2.3-1%{?dist} -- Added logging to sync components -* Fri Apr 26 2014 Luko Gjenero - 1.2.2-1%{?dist} -- Updated prefilter -* Tue Mar 18 2014 Paschalis Korosoglou - 1.2.1-1%{?dist} -- Updated daily cronjobs to run within first five minutes of each day -* Thu Jan 30 2014 Paschalis Korosoglou - 1.1.19-1%{?dist} -- Updated daily cronjobs to run within first hour of each day -* Tue Jan 14 2014 Paschalis Korosoglou - 1.1.18-1%{?dist} -- Added daily cronjob for hepspec values -* Thu Nov 28 2013 Luko Gjenero - 1.1.16-3%{?dist} -- Fixed prefilter -* Thu Nov 28 2013 Luko Gjenero - 1.1.16-2%{?dist} -- Fixed prefilter -* Thu Nov 28 2013 Luko Gjenero - 1.1.16-1%{?dist} -- Updated prefilter -* Thu Nov 13 2013 Luko Gjenero - 1.1.15-1%{?dist} -- VO Sync component -* Fri Nov 8 2013 Paschalis Korosoglou - 1.1.0-1%{?dist} -- Inclusion of hepspec sync plus cronjobs -* Mon Nov 4 2013 Paschalis Korosoglou - 1.0.0-6%{?dist} -- Fixes in consumer -* Tue Sep 17 2013 Paschalis Korosoglou - 1.0.0-5%{?dist} -- Fix in prefilter -* Mon Sep 9 2013 Paschalis Korosoglou - 1.0.0-4%{?dist} -- Rebuilt with fixed downtimes issue -* Thu Aug 29 2013 Paschalis Korosoglou - 1.0.0-2%{?dist} -- Minor change in prefilter script -* Thu Aug 1 2013 Luko Gjenero - 1.0.0-1%{?dist} -- Initial release - +* Thu Feb 10 2022 Daniel Vrcic - 2.0.0-1%{dist} +- release of async-enabled connectors with additional CSV and JSON topologies diff --git a/bin/downtimes-gocdb-connector.py b/bin/downtimes-gocdb-connector.py index d1eef48f..571895fa 100755 --- a/bin/downtimes-gocdb-connector.py +++ b/bin/downtimes-gocdb-connector.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # Copyright (c) 2013 GRNET S.A., SRCE, IN2P3 CNRS Computing Centre # @@ -28,114 +28,83 @@ import datetime import os import sys -from urlparse import urlparse +from urllib.parse import urlparse -from argo_egi_connectors import input -from argo_egi_connectors import output +import uvloop +import asyncio + +from argo_egi_connectors.io.http import SessionWithRetry +from argo_egi_connectors.exceptions import ConnectorHttpError, ConnectorParseError +from argo_egi_connectors.io.webapi import WebAPI +from argo_egi_connectors.io.avrowrite import AvroWriter +from argo_egi_connectors.io.statewrite import state_write from argo_egi_connectors.log import Logger from argo_egi_connectors.config import Global, CustomerConf -from argo_egi_connectors.helpers import filename_date, module_class_name +from argo_egi_connectors.utils import filename_date, module_class_name +from argo_egi_connectors.parse.gocdb_downtimes import ParseDowntimes logger = None +globopts = {} DOWNTIMEPI = '/gocdbpi/private/?method=get_downtime' -globopts = {} +async def fetch_data(feed, auth_opts, start, end): + feed_parts = urlparse(feed) + start_fmt = start.strftime("%Y-%m-%d") + end_fmt = end.strftime("%Y-%m-%d") + session = SessionWithRetry(logger, os.path.basename(sys.argv[0]), globopts) + res = await session.http_get( + '{}://{}{}&windowstart={}&windowend={}'.format(feed_parts.scheme, + feed_parts.netloc, + DOWNTIMEPI, start_fmt, + end_fmt) + ) -def getText(nodelist): - rc = [] - for node in nodelist: - if node.nodeType == node.TEXT_NODE: - rc.append(node.data) - return ''.join(rc) + return res -def all_same(elemlist): - prev = None +def parse_source(res, start, end, uidservtype): + gocdb = ParseDowntimes(logger, res, start, end, uidservtype) + return gocdb.get_data() - for el in elemlist: - if prev is None: - prev = el - else: - if el != prev: - return False - else: - return True - - -class GOCDBReader(object): - def __init__(self, feed, auth=None, uid=False): - self._o = urlparse(feed) - self.argDateFormat = "%Y-%m-%d" - self.WSDateFormat = "%Y-%m-%d %H:%M" - self.state = True - self.custauth = auth - self.uid = uid - - def getDowntimes(self, start, end): - filteredDowntimes = list() - - try: - res = input.connection(logger, module_class_name(self), globopts, self._o.scheme, self._o.netloc, - DOWNTIMEPI + '&windowstart=%s&windowend=%s' % (start.strftime(self.argDateFormat), - end.strftime(self.argDateFormat)), - custauth=self.custauth) - if not res: - raise input.ConnectorError() - - doc = input.parse_xml(logger, module_class_name(self), globopts, - res, self._o.scheme + '://' + self._o.netloc + DOWNTIMEPI) - - if not doc: - raise input.ConnectorError() - - except input.ConnectorError: - self.state = False - return [] - else: - downtimes = doc.getElementsByTagName('DOWNTIME') - try: - for downtime in downtimes: - classification = downtime.getAttributeNode('CLASSIFICATION').nodeValue - hostname = getText(downtime.getElementsByTagName('HOSTNAME')[0].childNodes) - serviceType = getText(downtime.getElementsByTagName('SERVICE_TYPE')[0].childNodes) - startStr = getText(downtime.getElementsByTagName('FORMATED_START_DATE')[0].childNodes) - endStr = getText(downtime.getElementsByTagName('FORMATED_END_DATE')[0].childNodes) - severity = getText(downtime.getElementsByTagName('SEVERITY')[0].childNodes) - try: - serviceId = getText(downtime.getElementsByTagName('PRIMARY_KEY')[0].childNodes) - except IndexError: - serviceId = downtime.getAttribute('PRIMARY_KEY') - startTime = datetime.datetime.strptime(startStr, self.WSDateFormat) - endTime = datetime.datetime.strptime(endStr, self.WSDateFormat) - - if (startTime < start): - startTime = start - if (endTime > end): - endTime = end - - if classification == 'SCHEDULED' and severity == 'OUTAGE': - dt = dict() - if self.uid: - dt['hostname'] = '{0}_{1}'.format(hostname, serviceId) - else: - dt['hostname'] = hostname - dt['service'] = serviceType - dt['start_time'] = startTime.strftime('%Y-%m-%d %H:%M').replace(' ', 'T', 1).replace(' ', ':') + ':00Z' - dt['end_time'] = endTime.strftime('%Y-%m-%d %H:%M').replace(' ', 'T', 1).replace(' ', ':') + ':00Z' - filteredDowntimes.append(dt) - - except (KeyError, IndexError, AttributeError, TypeError, AssertionError) as e: - self.state = False - logger.error(module_class_name(self) + 'Customer:%s Job:%s : Error parsing feed %s - %s' % (logger.customer, logger.job, - self._o.scheme + '://' + self._o.netloc + DOWNTIMEPI, - repr(e).replace('\'',''))) - return [] - else: - return filteredDowntimes +def get_webapi_opts(cglob, confcust): + webapi_custopts = confcust.get_webapiopts() + webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') + webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') + if not webapi_complete: + logger.error('Customer:%s %s options incomplete, missing %s' % (logger.customer, 'webapi', ' '.join(missopt))) + raise SystemExit(1) + return webapi_opts + + +async def send_webapi(webapi_opts, date, dts): + webapi = WebAPI(sys.argv[0], webapi_opts['webapihost'], + webapi_opts['webapitoken'], logger, + int(globopts['ConnectionRetry'.lower()]), + int(globopts['ConnectionTimeout'.lower()]), + int(globopts['ConnectionSleepRetry'.lower()]), date=date) + await webapi.send(dts, downtimes_component=True) + + +async def write_state(confcust, timestamp, state): + # safely assume here one customer defined in customer file + cust = list(confcust.get_customers())[0] + statedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust) + await state_write(sys.argv[0], statedir, state, + globopts['InputStateDays'.lower()], timestamp) + + +def write_avro(confcust, dts, timestamp): + custdir = confcust.get_custdir() + filename = filename_date(logger, globopts['OutputDowntimes'.lower()], custdir, stamp=timestamp) + avro = AvroWriter(globopts['AvroSchemasDowntimes'.lower()], filename) + ret, excep = avro.write(dts) + if not ret: + logger.error('Customer:{} {}'.format(logger.customer, repr(excep))) + raise SystemExit(1) def main(): @@ -156,7 +125,8 @@ def main(): confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) - feeds = confcust.get_mapfeedjobs(sys.argv[0], deffeed='https://goc.egi.eu/gocdbpi/') + feed = confcust.get_topofeed() + logger.customer = confcust.get_custname() if len(args.date) == 0: print(parser.print_help()) @@ -169,105 +139,61 @@ def main(): timestamp = start.strftime('%Y_%m_%d') start = start.replace(hour=0, minute=0, second=0) end = end.replace(hour=23, minute=59, second=59) - except ValueError as e: - logger.error(e) + except ValueError as exc: + logger.error(exc) + raise SystemExit(1) + + uidservtype = confcust.get_uidserviceendpoints() + + auth_custopts = confcust.get_authopts() + auth_opts = cglob.merge_opts(auth_custopts, 'authentication') + auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') + if not auth_complete: + missing_err = ''.join(missing) + logger.error('Customer:{} authentication options incomplete, missing {}'.format(logger.customer, missing_err)) raise SystemExit(1) - j = 0 - for feed, jobcust in feeds.items(): - customers = set(map(lambda jc: confcust.get_custname(jc[1]), jobcust)) - customers = customers.pop() if len(customers) == 1 else '({0})'.format(','.join(customers)) - sjobs = set(map(lambda jc: jc[0], jobcust)) - jobs = list(sjobs)[0] if len(sjobs) == 1 else '({0})'.format(','.join(sjobs)) - logger.job = jobs - logger.customer = customers - uidjob = confcust.pass_uidserviceendpoints(sjobs) - - auth_custopts = confcust.get_authopts(feed, jobcust) - auth_opts = cglob.merge_opts(auth_custopts, 'authentication') - auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') - if not auth_complete: - logger.error('Customer:%s Jobs:%s %s options incomplete, missing %s' - % (logger.customer, logger.job, 'authentication', - ''.join(missing))) - continue + loop = uvloop.new_event_loop() + asyncio.set_event_loop(loop) + try: # we don't have multiple tenant definitions in one # customer file so we can safely assume one tenant/customer - cust = jobcust[0][1] - write_empty = confcust.send_empty(sys.argv[0], cust) - - # do fetch only once - if all_same(uidjob): - gocdb = GOCDBReader(feed, auth_opts, uidjob[j]) - if not write_empty: - dts = gocdb.getDowntimes(start, end) - else: - dts = [] - gocdb.state = True - - for job, cust in jobcust: - # fetch for every job because of different TopoUIDServiceEndpoints - # setting for each job - if not all_same(uidjob): - gocdb = GOCDBReader(feed, auth_opts, uidjob[j]) - if not write_empty: - dts = gocdb.getDowntimes(start, end) - else: - dts = [] - gocdb.state = True - - jobdir = confcust.get_fulldir(cust, job) - jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust, job) - - logger.customer = confcust.get_custname(cust) - logger.job = job - - ams_custopts = confcust.get_amsopts(cust) - ams_opts = cglob.merge_opts(ams_custopts, 'ams') - ams_complete, missopt = cglob.is_complete(ams_opts, 'ams') - if not ams_complete: - logger.error('Customer:%s Job:%s %s options incomplete, missing %s' % (logger.customer, job, 'ams', ' '.join(missopt))) - continue - - output.write_state(sys.argv[0], jobstatedir, gocdb.state, globopts['InputStateDays'.lower()], timestamp) - - if not gocdb.state: - continue - - if eval(globopts['GeneralPublishAms'.lower()]): - ams = output.AmsPublish(ams_opts['amshost'], - ams_opts['amsproject'], - ams_opts['amstoken'], - ams_opts['amstopic'], - confcust.get_jobdir(job), - ams_opts['amsbulk'], - ams_opts['amspacksinglemsg'], - logger, - int(globopts['ConnectionRetry'.lower()]), - int(globopts['ConnectionTimeout'.lower()])) - - ams.send(globopts['AvroSchemasDowntimes'.lower()], 'downtimes', - timestamp.replace('_', '-'), dts) - - if eval(globopts['GeneralWriteAvro'.lower()]): - filename = filename_date(logger, globopts['OutputDowntimes'.lower()], jobdir, stamp=timestamp) - avro = output.AvroWriter(globopts['AvroSchemasDowntimes'.lower()], filename) - ret, excep = avro.write(dts) - if not ret: - logger.error('Customer:%s Job:%s %s' % (logger.customer, logger.job, repr(excep))) - raise SystemExit(1) - - j += 1 - - if gocdb.state: - custs = set([cust for job, cust in jobcust]) - for cust in custs: - jobs = [job for job, lcust in jobcust if cust == lcust] - logger.info('Customer:%s Jobs:%s Fetched Date:%s Endpoints:%d' % (confcust.get_custname(cust), - jobs[0] if len(jobs) == 1 else '({0})'.format(','.join(jobs)), - args.date[0], len(dts))) + write_empty = confcust.send_empty(sys.argv[0]) + if not write_empty: + res = loop.run_until_complete( + fetch_data(feed, auth_opts, start, end) + ) + dts = parse_source(res, start, end, uidservtype) + else: + dts = [] + + loop.run_until_complete( + write_state(confcust, timestamp, True) + ) + + webapi_opts = get_webapi_opts(cglob, confcust) + + if eval(globopts['GeneralPublishWebAPI'.lower()]): + loop.run_until_complete( + send_webapi(webapi_opts, args.date[0], dts) + ) + + if dts or write_empty: + cust = list(confcust.get_customers())[0] + logger.info('Customer:%s Fetched Date:%s Endpoints:%d' % + (confcust.get_custname(cust), args.date[0], len(dts))) + + if eval(globopts['GeneralWriteAvro'.lower()]): + write_avro(confcust, dts, timestamp) + + except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc: + logger.error(repr(exc)) + loop.run_until_complete( + write_state(confcust, timestamp, False) + ) + loop.close() if __name__ == '__main__': main() diff --git a/bin/metricprofile-webapi-connector.py b/bin/metricprofile-webapi-connector.py index 0f0cc250..c167f5bd 100755 --- a/bin/metricprofile-webapi-connector.py +++ b/bin/metricprofile-webapi-connector.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # Copyright (c) 2013 GRNET S.A., SRCE, IN2P3 CNRS Computing Centre # @@ -28,14 +28,19 @@ import os import re import sys -import urlparse -from argo_egi_connectors import input -from argo_egi_connectors import output +import uvloop +import asyncio + +from argo_egi_connectors.io.http import SessionWithRetry +from argo_egi_connectors.exceptions import ConnectorHttpError, ConnectorParseError +from argo_egi_connectors.io.avrowrite import AvroWriter +from argo_egi_connectors.io.statewrite import state_write from argo_egi_connectors.log import Logger from argo_egi_connectors.config import CustomerConf, Global -from argo_egi_connectors.helpers import filename_date, module_class_name, datestamp, date_check +from argo_egi_connectors.utils import filename_date, module_class_name, datestamp, date_check +from argo_egi_connectors.parse.webapi_metricprofile import ParseMetricProfiles logger = None @@ -44,80 +49,40 @@ API_PATH = '/api/v2/metric_profiles' -class WebAPI(object): - def __init__(self, customer, job, profiles, namespace, host, token): - self.state = True - self.customer = customer - self.job = job - self.host = host - self.token = token - self.profiles = profiles - self.namespace = namespace - - def get_profiles(self): - try: - fetched_profiles = self._fetch() - target_profiles = filter(lambda profile: profile['name'] in self.profiles, fetched_profiles) - profile_list = list() - - if len(target_profiles) == 0: - self.state = False - logger.error('Customer:' + self.customer + ' Job:' + self.job + ': No profiles {0} were found!'.format(', '.join(self.profiles))) - - raise SystemExit(1) - - for profile in target_profiles: - for service in profile['services']: - for metric in service['metrics']: - if self.namespace: - profile_name = '{0}.{1}'.format(self.namespace, profile['name']) - else: - profile_name = profile['name'] - profile_list.append({ - 'profile': profile_name, - 'metric': metric, - 'service': service['service'] - }) - - except (KeyError, IndexError, AttributeError, TypeError) as e: - self.state = False - logger.error(module_class_name(self) + ' Customer:%s : Error parsing feed %s - %s' % (logger.customer, - self.host + API_PATH, - repr(e).replace('\'', '').replace('\"', ''))) - return [] - else: - return self._format(profile_list) - - def _fetch(self): - try: - res = input.connection(logger, module_class_name(self), globopts, - 'https', self.host, API_PATH, - custauth={'WebAPIToken'.lower(): self.token}) - if not res: - raise input.ConnectorError() - - json_data = input.parse_json(logger, module_class_name(self), - globopts, res, self.host + API_PATH) - - if not json_data or not json_data.get('data', False): - raise input.ConnectorError() - - return json_data['data'] - - except input.ConnectorError: - self.state = False - - def _format(self, profile_list): - profiles = [] - - for p in profile_list: - pt = dict() - pt['metric'] = p['metric'] - pt['profile'] = p['profile'] - pt['service'] = p['service'] - profiles.append(pt) - - return profiles +async def fetch_data(host, token): + session = SessionWithRetry(logger, os.path.basename(sys.argv[0]), globopts, + token=token) + res = await session.http_get('{}://{}{}'.format('https', host, API_PATH)) + return res + + +def parse_source(res, profiles, namespace): + metric_profiles = ParseMetricProfiles(logger, res, profiles, namespace).get_data() + return metric_profiles + + +async def write_state(cust, job, confcust, fixed_date, state): + jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust, job) + if fixed_date: + await state_write(sys.argv[0], jobstatedir, state, + globopts['InputStateDays'.lower()], + fixed_date.replace('-', '_')) + else: + await state_write(sys.argv[0], jobstatedir, state, + globopts['InputStateDays'.lower()]) + + +def write_avro(cust, job, confcust, fixed_date, fetched_profiles): + jobdir = confcust.get_fulldir(cust, job) + if fixed_date: + filename = filename_date(logger, globopts['OutputMetricProfile'.lower()], jobdir, fixed_date.replace('-', '_')) + else: + filename = filename_date(logger, globopts['OutputMetricProfile'.lower()], jobdir) + avro = AvroWriter(globopts['AvroSchemasMetricProfile'.lower()], filename) + ret, excep = avro.write(fetched_profiles) + if not ret: + logger.error('Customer:%s Job:%s %s' % (logger.customer, logger.job, repr(excep))) + raise SystemExit(1) def main(): @@ -144,6 +109,9 @@ def main(): confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) + loop = uvloop.new_event_loop() + asyncio.set_event_loop(loop) + for cust in confcust.get_customers(): custname = confcust.get_custname(cust) @@ -160,66 +128,27 @@ def main(): logger.error('Customer:%s Job:%s %s options incomplete, missing %s' % (custname, logger.job, 'webapi', ' '.join(missopt))) continue - webapi = WebAPI(custname, job, profiles, confcust.get_namespace(job), - webapi_opts['webapihost'], - webapi_opts['webapitoken']) - fetched_profiles = webapi.get_profiles() + try: + res = loop.run_until_complete( + fetch_data(webapi_opts['webapihost'], webapi_opts['webapitoken']) + ) - jobdir = confcust.get_fulldir(cust, job) - jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust, job) + fetched_profiles = parse_source(res, profiles, confcust.get_namespace(job)) - ams_custopts = confcust.get_amsopts(cust) - ams_opts = cglob.merge_opts(ams_custopts, 'ams') - ams_complete, missopt = cglob.is_complete(ams_opts, 'ams') - if not ams_complete: - logger.error('Customer:%s %s options incomplete, missing %s' % (custname, 'ams', ' '.join(missopt))) - continue + loop.run_until_complete( + write_state(cust, job, confcust, fixed_date, True) + ) - if fixed_date: - output.write_state(sys.argv[0], jobstatedir, - webapi.state, - globopts['InputStateDays'.lower()], - fixed_date.replace('-', '_')) - else: - output.write_state(sys.argv[0], jobstatedir, - webapi.state, - globopts['InputStateDays'.lower()]) - - if not webapi.state: - continue + if eval(globopts['GeneralWriteAvro'.lower()]): + write_avro(cust, job, confcust, fixed_date, fetched_profiles) + + logger.info('Customer:' + custname + ' Job:' + job + ' Profiles:%s Tuples:%d' % (', '.join(profiles), len(fetched_profiles))) - if eval(globopts['GeneralPublishAms'.lower()]): - if fixed_date: - partdate = fixed_date - else: - partdate = datestamp(1).replace('_', '-') - - ams = output.AmsPublish(ams_opts['amshost'], - ams_opts['amsproject'], - ams_opts['amstoken'], - ams_opts['amstopic'], - confcust.get_jobdir(job), - ams_opts['amsbulk'], - ams_opts['amspacksinglemsg'], - logger, - int(globopts['ConnectionRetry'.lower()]), - int(globopts['ConnectionTimeout'.lower()])) - - ams.send(globopts['AvroSchemasMetricProfile'.lower()], 'metric_profile', - partdate, fetched_profiles) - - if eval(globopts['GeneralWriteAvro'.lower()]): - if fixed_date: - filename = filename_date(logger, globopts['OutputMetricProfile'.lower()], jobdir, fixed_date.replace('-', '_')) - else: - filename = filename_date(logger, globopts['OutputMetricProfile'.lower()], jobdir) - avro = output.AvroWriter(globopts['AvroSchemasMetricProfile'.lower()], filename) - ret, excep = avro.write(fetched_profiles) - if not ret: - logger.error('Customer:%s Job:%s %s' % (logger.customer, logger.job, repr(excep))) - raise SystemExit(1) - - logger.info('Customer:' + custname + ' Job:' + job + ' Profiles:%s Tuples:%d' % (', '.join(profiles), len(fetched_profiles))) + except (ConnectorHttpError, KeyboardInterrupt, ConnectorParseError) as exc: + logger.error(repr(exc)) + loop.run_until_complete( + write_state(cust, job, confcust, fixed_date, False) + ) if __name__ == '__main__': diff --git a/bin/replay-avro-data.py b/bin/replay-avro-data.py index 77fbf7b8..f854dcd8 100755 --- a/bin/replay-avro-data.py +++ b/bin/replay-avro-data.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import argparse from argo_egi_connectors.output import AmsPublish, load_schema from avro.datafile import DataFileReader @@ -82,7 +82,7 @@ def main(): avro = AvroReader(args.avrofile) ret, data = avro.read() if not ret: - print "Error: " + data + print("Error: " + data) raise SystemExit(1) amshost = args.host # ams host diff --git a/bin/topology-csv-connector.py b/bin/topology-csv-connector.py new file mode 100755 index 00000000..7a4a87d1 --- /dev/null +++ b/bin/topology-csv-connector.py @@ -0,0 +1,223 @@ +#!/usr/bin/python3 + +# Copyright (c) 2013 GRNET S.A., SRCE, IN2P3 CNRS Computing Centre +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the +# License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS +# IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language +# governing permissions and limitations under the License. +# +# The views and conclusions contained in the software and +# documentation are those of the authors and should not be +# interpreted as representing official policies, either expressed +# or implied, of either GRNET S.A., SRCE or IN2P3 CNRS Computing +# Centre +# +# The work represented by this source file is partially funded by +# the EGI-InSPIRE project through the European Commission's 7th +# Framework Programme (contract # INFSO-RI-261323) + +import argparse +import copy +import os +import sys +import re +import xml.dom.minidom + +import uvloop +import asyncio +from concurrent.futures import ProcessPoolExecutor +from functools import partial + +from argo_egi_connectors.io.http import SessionWithRetry +from argo_egi_connectors.exceptions import ConnectorHttpError, ConnectorParseError +from argo_egi_connectors.io.webapi import WebAPI +from argo_egi_connectors.io.avrowrite import AvroWriter +from argo_egi_connectors.io.statewrite import state_write +from argo_egi_connectors.mesh.contacts import attach_contacts_topodata +from argo_egi_connectors.log import Logger +from argo_egi_connectors.parse.flat_topology import ParseFlatEndpoints, ParseContacts + +from argo_egi_connectors.config import Global, CustomerConf +from argo_egi_connectors.utils import filename_date, module_class_name, datestamp, date_check +from urllib.parse import urlparse + +logger = None + +globopts = {} +custname = '' + +isok = True + + +def parse_source_topo(res, custname, uidservtype): + topo = ParseFlatEndpoints(logger, res, custname, uidservtype, is_csv=True) + group_groups = topo.get_groupgroups() + group_endpoints = topo.get_groupendpoints() + return group_groups, group_endpoints + + +def get_webapi_opts(cglob, confcust): + webapi_custopts = confcust.get_webapiopts() + webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') + webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') + if not webapi_complete: + logger.error('Customer:%s %s options incomplete, missing %s' % (logger.customer, 'webapi', ' '.join(missopt))) + raise SystemExit(1) + return webapi_opts + + +async def write_state(confcust, fixed_date, state): + # safely assume here one customer defined in customer file + cust = list(confcust.get_customers())[0] + statedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust) + if fixed_date: + await state_write(sys.argv[0], statedir, state, + globopts['InputStateDays'.lower()], + fixed_date.replace('-', '_')) + else: + await state_write(sys.argv[0], statedir, state, + globopts['InputStateDays'.lower()]) + + +async def fetch_data(feed, auth_opts): + feed_parts = urlparse(feed) + session = SessionWithRetry(logger, os.path.basename(sys.argv[0]), + globopts, custauth=auth_opts) + res = await session.http_get('{}://{}{}?{}'.format(feed_parts.scheme, + feed_parts.netloc, + feed_parts.path, + feed_parts.query)) + return res + + + +async def send_webapi(webapi_opts, data, topotype, fixed_date=None): + webapi = WebAPI(sys.argv[0], webapi_opts['webapihost'], + webapi_opts['webapitoken'], logger, + int(globopts['ConnectionRetry'.lower()]), + int(globopts['ConnectionTimeout'.lower()]), + int(globopts['ConnectionSleepRetry'.lower()]), + date=fixed_date) + await webapi.send(data, topotype) + + +def write_avro(confcust, group_groups, group_endpoints, fixed_date): + custdir = confcust.get_custdir() + if fixed_date: + filename = filename_date(logger, globopts['OutputTopologyGroupOfGroups'.lower()], custdir, fixed_date.replace('-', '_')) + else: + filename = filename_date(logger, globopts['OutputTopologyGroupOfGroups'.lower()], custdir) + avro = AvroWriter(globopts['AvroSchemasTopologyGroupOfGroups'.lower()], filename) + ret, excep = avro.write(group_groups) + if not ret: + logger.error('Customer:%s : %s' % (logger.customer, repr(excep))) + raise SystemExit(1) + + if fixed_date: + filename = filename_date(logger, globopts['OutputTopologyGroupOfEndpoints'.lower()], custdir, fixed_date.replace('-', '_')) + else: + filename = filename_date(logger, globopts['OutputTopologyGroupOfEndpoints'.lower()], custdir) + avro = AvroWriter(globopts['AvroSchemasTopologyGroupOfEndpoints'.lower()], filename) + ret, excep = avro.write(group_endpoints) + if not ret: + logger.error('Customer:%s: %s' % (logger.customer, repr(excep))) + raise SystemExit(1) + + +def main(): + global logger, globopts, confcust + parser = argparse.ArgumentParser(description="""Fetch entities (ServiceGroups, Sites, Endpoints) + from CSV topology feed for every customer and job listed in customer.conf and write them + in an appropriate place""") + parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False) + parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) + parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) + args = parser.parse_args() + group_endpoints, group_groups = [], [] + logger = Logger(os.path.basename(sys.argv[0])) + + fixed_date = None + if args.date and date_check(args.date): + fixed_date = args.date + + confpath = args.gloconf[0] if args.gloconf else None + cglob = Global(sys.argv[0], confpath) + globopts = cglob.parse() + + confpath = args.custconf[0] if args.custconf else None + confcust = CustomerConf(sys.argv[0], confpath) + confcust.parse() + confcust.make_dirstruct() + confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) + topofeed = confcust.get_topofeed() + uidservtype = confcust.get_uidserviceendpoints() + topofetchtype = confcust.get_topofetchtype() + custname = confcust.get_custname() + logger.customer = custname + + auth_custopts = confcust.get_authopts() + auth_opts = cglob.merge_opts(auth_custopts, 'authentication') + auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') + if not auth_complete: + logger.error('%s options incomplete, missing %s' % ('authentication', ' '.join(missing))) + raise SystemExit(1) + + loop = uvloop.new_event_loop() + asyncio.set_event_loop(loop) + + try: + group_endpoints, group_groups = list(), list() + + # fetch topology data concurrently in coroutines + fetched_topology = loop.run_until_complete(fetch_data(topofeed, auth_opts)) + + group_groups, group_endpoints = parse_source_topo(fetched_topology, + custname, + uidservtype) + contacts = ParseContacts(logger, fetched_topology, uidservtype, is_csv=True).get_contacts() + attach_contacts_topodata(logger, contacts, group_endpoints) + + + loop.run_until_complete( + write_state(confcust, fixed_date, True) + ) + + webapi_opts = get_webapi_opts(cglob, confcust) + + numgg = len(group_groups) + numge = len(group_endpoints) + + # send concurrently to WEB-API in coroutines + if eval(globopts['GeneralPublishWebAPI'.lower()]): + loop.run_until_complete( + asyncio.gather( + send_webapi(webapi_opts, group_groups, 'groups', fixed_date), + send_webapi(webapi_opts, group_endpoints,'endpoints', fixed_date) + ) + ) + + if eval(globopts['GeneralWriteAvro'.lower()]): + write_avro(confcust, group_groups, group_endpoints, fixed_date) + + logger.info('Customer:' + custname + ' Type:%s ' % (','.join(topofetchtype)) + 'Fetched Endpoints:%d' % (numge) + ' Groups:%d' % (numgg)) + + except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc: + logger.error(repr(exc)) + loop.run_until_complete( + write_state(confcust, fixed_date, False ) + ) + + finally: + loop.close() + + +if __name__ == '__main__': + main() diff --git a/bin/topology-eosc-connector.py b/bin/topology-eosc-connector.py deleted file mode 100755 index d71306f9..00000000 --- a/bin/topology-eosc-connector.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/python - -import argparse -import os -import sys -import json - -from urlparse import urlparse - -from argo_egi_connectors import input -from argo_egi_connectors import output -from argo_egi_connectors.log import Logger -from argo_egi_connectors.config import Global, CustomerConf -from argo_egi_connectors.helpers import filename_date, datestamp, date_check - - -def is_feed(feed): - data = urlparse(feed) - - if not data.netloc: - return False - else: - return True - - -class EOSCReader(object): - def __init__(self, data, uidservtype=False, fetchtype='ServiceGroups'): - self.data = data - self.uidservtype = uidservtype - self.fetchtype = fetchtype - - def _construct_fqdn(self, http_endpoint): - return urlparse(http_endpoint).netloc - - def get_groupgroups(self): - groups = list() - - for entity in self.data: - tmp_dict = dict() - - tmp_dict['type'] = 'PROJECT' - tmp_dict['group'] = 'EOSC' - tmp_dict['subgroup'] = entity['SITENAME-SERVICEGROUP'] - tmp_dict['tags'] = {'monitored': '1', 'scope': 'EOSC'} - - groups.append(tmp_dict) - - return groups - - def get_groupendpoints(self): - groups = list() - - for entity in self.data: - tmp_dict = dict() - - tmp_dict['type'] = self.fetchtype.upper() - tmp_dict['group'] = entity['SITENAME-SERVICEGROUP'] - tmp_dict['service'] = entity['SERVICE_TYPE'] - info_url = entity['URL'] - if self.uidservtype: - tmp_dict['hostname'] = '{1}_{0}'.format(entity['Service Unique ID'], self._construct_fqdn(info_url)) - else: - tmp_dict['hostname'] = self._construct_fqdn(entity['URL']) - tmp_dict['tags'] = {'scope': 'EOSC', 'monitored': '1', 'info.URL': info_url} - - groups.append(tmp_dict) - - return groups - - -def main(): - parser = argparse.ArgumentParser(description="""Fetch and construct entities from EOSC-PORTAL feed""") - parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False) - parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) - parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) - args = parser.parse_args() - group_endpoints, group_groups = [], [] - logger = Logger(os.path.basename(sys.argv[0])) - - fixed_date = None - if args.date and date_check(args.date): - fixed_date = args.date - - confpath = args.gloconf[0] if args.gloconf else None - cglob = Global(sys.argv[0], confpath) - globopts = cglob.parse() - - confpath = args.custconf[0] if args.custconf else None - confcust = CustomerConf(sys.argv[0], confpath) - confcust.parse() - confcust.make_dirstruct() - confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) - - for cust in confcust.get_customers(): - custname = confcust.get_custname(cust) - - for job in confcust.get_jobs(cust): - jobdir = confcust.get_fulldir(cust, job) - logger.customer = confcust.get_custname(cust) - jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust, job) - fetchtype = confcust.get_fetchtype(job) - - state = None - logger.job = job - logger.customer = custname - - uidservtype = confcust.pass_uidserviceendpoints(job) - ams_custopts = confcust.get_amsopts(cust) - ams_opts = cglob.merge_opts(ams_custopts, 'ams') - ams_complete, missopt = cglob.is_complete(ams_opts, 'ams') - - feeds = confcust.get_mapfeedjobs(sys.argv[0]) - if is_feed(feeds.keys()[0]): - remote_topo = urlparse(feeds.keys()[0]) - res = input.connection(logger, 'EOSC', globopts, remote_topo.scheme, remote_topo.netloc, remote_topo.path) - if not res: - raise input.ConnectorError() - - doc = input.parse_json(logger, 'EOSC', globopts, res, - remote_topo.scheme + '://' + - remote_topo.netloc + remote_topo.path) - eosc = EOSCReader(doc, uidservtype, fetchtype) - group_groups = eosc.get_groupgroups() - group_endpoints = eosc.get_groupendpoints() - state = True - else: - try: - with open(feeds.keys()[0]) as fp: - js = json.load(fp) - eosc = EOSCReader(js, uidservtype, fetchtype) - group_groups = eosc.get_groupgroups() - group_endpoints = eosc.get_groupendpoints() - state = True - except IOError as exc: - logger.error('Customer:%s Job:%s : Problem opening %s - %s' % (logger.customer, logger.job, feeds.keys()[0], repr(exc))) - state = False - - if fixed_date: - output.write_state(sys.argv[0], jobstatedir, state, - globopts['InputStateDays'.lower()], - fixed_date.replace('-', '_')) - else: - output.write_state(sys.argv[0], jobstatedir, state, - globopts['InputStateDays'.lower()]) - - if not state: - continue - - numge = len(group_endpoints) - numgg = len(group_groups) - - if eval(globopts['GeneralPublishAms'.lower()]): - if fixed_date: - partdate = fixed_date - else: - partdate = datestamp(1).replace('_', '-') - - ams = output.AmsPublish(ams_opts['amshost'], - ams_opts['amsproject'], - ams_opts['amstoken'], - ams_opts['amstopic'], - confcust.get_jobdir(job), - ams_opts['amsbulk'], - ams_opts['amspacksinglemsg'], - logger, - int(globopts['ConnectionRetry'.lower()]), - int(globopts['ConnectionTimeout'.lower()])) - - ams.send(globopts['AvroSchemasTopologyGroupOfGroups'.lower()], - 'group_groups', partdate, group_groups) - - ams.send(globopts['AvroSchemasTopologyGroupOfEndpoints'.lower()], - 'group_endpoints', partdate, group_endpoints) - - if eval(globopts['GeneralWriteAvro'.lower()]): - if fixed_date: - filename = filename_date(logger, globopts['OutputTopologyGroupOfGroups'.lower()], jobdir, fixed_date.replace('-', '_')) - else: - filename = filename_date(logger, globopts['OutputTopologyGroupOfGroups'.lower()], jobdir) - avro = output.AvroWriter(globopts['AvroSchemasTopologyGroupOfGroups'.lower()], filename) - ret, excep = avro.write(group_groups) - if not ret: - logger.error('Customer:%s Job:%s : %s' % (logger.customer, logger.job, repr(excep))) - raise SystemExit(1) - - if fixed_date: - filename = filename_date(logger, globopts['OutputTopologyGroupOfEndpoints'.lower()], jobdir, fixed_date.replace('-', '_')) - else: - filename = filename_date(logger, globopts['OutputTopologyGroupOfEndpoints'.lower()], jobdir) - avro = output.AvroWriter(globopts['AvroSchemasTopologyGroupOfEndpoints'.lower()], filename) - ret, excep = avro.write(group_endpoints) - if not ret: - logger.error('Customer:%s Job:%s : %s' % (logger.customer, logger.job, repr(excep))) - raise SystemExit(1) - - logger.info('Customer:' + custname + ' Job:' + job + ' Fetched Endpoints:%d' % (numge) + ' Groups(%s):%d' % (fetchtype, numgg)) - - -if __name__ == '__main__': - main() diff --git a/bin/topology-gocdb-connector.py b/bin/topology-gocdb-connector.py index 1be54cd4..6db9bc26 100755 --- a/bin/topology-gocdb-connector.py +++ b/bin/topology-gocdb-connector.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # Copyright (c) 2013 GRNET S.A., SRCE, IN2P3 CNRS Computing Centre # @@ -25,26 +25,49 @@ # Framework Programme (contract # INFSO-RI-261323) import argparse -import copy import os import sys -import re +import xml.dom.minidom -from argo_egi_connectors import input -from argo_egi_connectors import output +from concurrent.futures import ProcessPoolExecutor +from functools import partial +from urllib.parse import urlparse + +import asyncio +import uvloop + +from argo_egi_connectors.exceptions import ConnectorParseError, ConnectorHttpError +from argo_egi_connectors.io.http import SessionWithRetry +from argo_egi_connectors.io.ldap import LDAPSessionWithRetry +from argo_egi_connectors.io.webapi import WebAPI +from argo_egi_connectors.io.avrowrite import AvroWriter +from argo_egi_connectors.io.statewrite import state_write from argo_egi_connectors.log import Logger +from argo_egi_connectors.mesh.srm_port import attach_srmport_topodata +from argo_egi_connectors.mesh.storage_element_path import attach_sepath_topodata +from argo_egi_connectors.mesh.contacts import attach_contacts_topodata +from argo_egi_connectors.parse.gocdb_topology import ParseServiceGroups, ParseServiceEndpoints, ParseSites +from argo_egi_connectors.parse.gocdb_contacts import ParseSiteContacts, ParseServiceEndpointContacts, ParseServiceGroupRoles, ParseSitesWithContacts, ParseServiceGroupWithContacts from argo_egi_connectors.config import Global, CustomerConf -from argo_egi_connectors.helpers import filename_date, module_class_name, datestamp, date_check -from urlparse import urlparse + +from argo_egi_connectors.utils import filename_date, date_check logger = None -SERVENDPI = '/gocdbpi/private/?method=get_service_endpoint' -SITESPI = '/gocdbpi/private/?method=get_site' -SERVGROUPPI = '/gocdbpi/private/?method=get_service_group' +# GOCDB explicitly says &scope='' for all scopes +# TODO: same methods can be served on different paths +SERVICE_ENDPOINTS_PI = '/gocdbpi/private/?method=get_service_endpoint&scope=' +SITES_PI = '/gocdbpi/private/?method=get_site&scope=' +SERVICE_GROUPS_PI = '/gocdbpi/private/?method=get_service_group&scope=' + +# SITES_PI = '/vapor/downloadLavoisier/option/xml/view/vapor_sites/param/vo=biomed' +# SERVICE_ENDPOINTS_PI = '/vapor/downloadLavoisier/option/xml/view/vapor_endpoints' -fetchtype = '' +ROC_CONTACTS = '/gocdbpi/private/?method=get_roc_contacts' +SITE_CONTACTS = '/gocdbpi/private/?method=get_site_contacts' +PROJECT_CONTACTS = '/gocdbpi/private/?method=get_project_contacts' +SERVICEGROUP_CONTACTS = '/gocdbpi/private/?method=get_service_group_role' globopts = {} custname = '' @@ -52,367 +75,200 @@ isok = True -def rem_nonalpha(string): - return re.sub(r'\W*', '', string) - - -def getText(nodelist): - rc = [] - for node in nodelist: - if node.nodeType == node.TEXT_NODE: - rc.append(node.data) - return ''.join(rc) - - -class GOCDBReader: - def __init__(self, feed, scopes, paging=False, auth=None): - self._o = urlparse(feed) - self.scopes = scopes if scopes else set(['NoScope']) - for scope in self.scopes: - code = "self.serviceList%s = dict(); " % rem_nonalpha(scope) - code += "self.groupList%s = dict();" % rem_nonalpha(scope) - code += "self.siteList%s = dict()" % rem_nonalpha(scope) - exec code - self.fetched = False - self.state = True - self.paging = paging - self.custauth = auth - - def getGroupOfServices(self, uidservtype=False): - if not self.fetched: - if not self.state or not self.loadDataIfNeeded(): - return [] - - groups, gl = list(), list() - - for scope in self.scopes: - code = "gl = gl + [value for key, value in self.groupList%s.iteritems()]" % rem_nonalpha(scope) - exec code - - for d in gl: - for service in d['services']: - g = dict() - g['type'] = fetchtype.upper() - g['group'] = d['name'] - g['service'] = service['type'] - if uidservtype: - g['hostname'] = '{1}_{0}'.format(service['service_id'], service['hostname']) - else: - g['hostname'] = service['hostname'] - g['group_monitored'] = d['monitored'] - g['tags'] = {'scope' : d['scope'], \ - 'monitored' : '1' if service['monitored'].lower() == 'Y'.lower() or \ - service['monitored'].lower() == 'True'.lower() else '0', \ - 'production' : '1' if service['production'].lower() == 'Y'.lower() or \ - service['production'].lower() == 'True'.lower() else '0'} - groups.append(g) - - return groups - - def getGroupOfGroups(self): - if not self.fetched: - if not self.state or not self.loadDataIfNeeded(): - return [] - - groupofgroups, gl = list(), list() - - if fetchtype == "ServiceGroups": - for scope in self.scopes: - code = "gl = gl + [value for key, value in self.groupList%s.iteritems()]" % rem_nonalpha(scope) - exec code - for d in gl: - g = dict() - g['type'] = 'PROJECT' - g['group'] = custname - g['subgroup'] = d['name'] - g['tags'] = {'monitored' : '1' if d['monitored'].lower() == 'Y'.lower() or \ - d['monitored'].lower() == 'True'.lower() else '0', - 'scope' : d['scope']} - groupofgroups.append(g) - else: - gg = [] - for scope in self.scopes: - code = "gg = gg + sorted([value for key, value in self.siteList%s.iteritems()], key=lambda s: s['ngi'])" % rem_nonalpha(scope) - exec code - - for gr in gg: - g = dict() - g['type'] = 'NGI' - g['group'] = gr['ngi'] - g['subgroup'] = gr['site'] - g['tags'] = {'certification' : gr['certification'], \ - 'scope' : gr['scope'], \ - 'infrastructure' : gr['infrastructure']} - - groupofgroups.append(g) - - return groupofgroups - - def getGroupOfEndpoints(self, uidservtype=False): - if not self.fetched: - if not self.state or not self.loadDataIfNeeded(): - return [] - - groupofendpoints, ge = list(), list() - for scope in self.scopes: - code = "ge = ge + sorted([value for key, value in self.serviceList%s.iteritems()], key=lambda s: s['site'])" % rem_nonalpha(scope) - exec code - - for gr in ge: - g = dict() - g['type'] = fetchtype.upper() - g['group'] = gr['site'] - g['service'] = gr['type'] - if uidservtype: - g['hostname'] = '{1}_{0}'.format(gr['service_id'], gr['hostname']) - else: - g['hostname'] = gr['hostname'] - g['tags'] = {'scope': gr['scope'], \ - 'monitored': '1' if gr['monitored'] == 'Y' or \ - gr['monitored'] == 'True' else '0', \ - 'production': '1' if gr['production'] == 'Y' or \ - gr['production'] == 'True' else '0'} - groupofendpoints.append(g) - - return groupofendpoints - - def loadDataIfNeeded(self): - for scope in self.scopes: - scopequery = "'&scope='+scope" if scope != 'NoScope' else "'&scope='" - try: - eval("self.getServiceEndpoints(self.serviceList%s, %s)" % (rem_nonalpha(scope), scopequery)) - eval("self.getServiceGroups(self.groupList%s, %s)" % (rem_nonalpha(scope), scopequery)) - eval("self.getSitesInternal(self.siteList%s, %s)" % (rem_nonalpha(scope), scopequery)) - self.fetched = True - except Exception: - self.state = False - return False - - return True - - def _get_xmldata(self, scope, pi): - res = input.connection(logger, module_class_name(self), globopts, - self._o.scheme, self._o.netloc, pi + scope, custauth=self.custauth) - if not res: - raise input.ConnectorError() - - doc = input.parse_xml(logger, module_class_name(self), globopts, res, - self._o.scheme + '://' + self._o.netloc + pi) - return doc - - def _get_service_endpoints(self, serviceList, scope, doc): - try: - services = doc.getElementsByTagName('SERVICE_ENDPOINT') - for service in services: - serviceId = '' - if service.getAttributeNode('PRIMARY_KEY'): - serviceId = str(service.attributes['PRIMARY_KEY'].value) - if serviceId not in serviceList: - serviceList[serviceId] = {} - serviceList[serviceId]['hostname'] = getText(service.getElementsByTagName('HOSTNAME')[0].childNodes) - serviceList[serviceId]['type'] = getText(service.getElementsByTagName('SERVICE_TYPE')[0].childNodes) - serviceList[serviceId]['monitored'] = getText(service.getElementsByTagName('NODE_MONITORED')[0].childNodes) - serviceList[serviceId]['production'] = getText(service.getElementsByTagName('IN_PRODUCTION')[0].childNodes) - serviceList[serviceId]['site'] = getText(service.getElementsByTagName('SITENAME')[0].childNodes) - serviceList[serviceId]['roc'] = getText(service.getElementsByTagName('ROC_NAME')[0].childNodes) - serviceList[serviceId]['service_id'] = serviceId - serviceList[serviceId]['scope'] = scope.split('=')[1] - serviceList[serviceId]['sortId'] = serviceList[serviceId]['hostname'] + '-' + serviceList[serviceId]['type'] + '-' + serviceList[serviceId]['site'] - - except (KeyError, IndexError, TypeError, AttributeError, AssertionError) as e: - logger.error(module_class_name(self) + 'Customer:%s Job:%s : Error parsing feed %s - %s' % (logger.customer, logger.job, self._o.scheme + '://' + self._o.netloc + SERVENDPI, - repr(e).replace('\'', '').replace('\"', ''))) - raise e - - def getServiceEndpoints(self, serviceList, scope): - try: - if self.paging: - count, cursor = 1, 0 - while count != 0: - doc = self._get_xmldata(scope, SERVENDPI + '&next_cursor=' + str(cursor)) - count = int(doc.getElementsByTagName('count')[0].childNodes[0].data) - links = doc.getElementsByTagName('link') - for le in links: - if le.getAttribute('rel') == 'next': - href = le.getAttribute('href') - for e in href.split('&'): - if 'next_cursor' in e: - cursor = e.split('=')[1] - self._get_service_endpoints(serviceList, scope, doc) - - else: - doc = self._get_xmldata(scope, SERVENDPI) - self._get_service_endpoints(serviceList, scope, doc) - - except input.ConnectorError as e: - raise e - - except Exception as e: - raise e - - def _get_sites_internal(self, siteList, scope, doc): - try: - sites = doc.getElementsByTagName('SITE') - for site in sites: - siteName = site.getAttribute('NAME') - if siteName not in siteList: - siteList[siteName] = {'site': siteName} - siteList[siteName]['infrastructure'] = getText(site.getElementsByTagName('PRODUCTION_INFRASTRUCTURE')[0].childNodes) - siteList[siteName]['certification'] = getText(site.getElementsByTagName('CERTIFICATION_STATUS')[0].childNodes) - siteList[siteName]['ngi'] = getText(site.getElementsByTagName('ROC')[0].childNodes) - siteList[siteName]['scope'] = scope.split('=')[1] - - except (KeyError, IndexError, TypeError, AttributeError, AssertionError) as e: - logger.error(module_class_name(self) + 'Customer:%s Job:%s : Error parsing feed %s - %s' % (logger.customer, logger.job, self._o.scheme + '://' + self._o.netloc + SITESPI, - repr(e).replace('\'', '').replace('\"', ''))) - raise e - - def getSitesInternal(self, siteList, scope): - try: - if self.paging: - count, cursor = 1, 0 - while count != 0: - doc = self._get_xmldata(scope, SITESPI + '&next_cursor=' + str(cursor)) - count = int(doc.getElementsByTagName('count')[0].childNodes[0].data) - links = doc.getElementsByTagName('link') - for le in links: - if le.getAttribute('rel') == 'next': - href = le.getAttribute('href') - for e in href.split('&'): - if 'next_cursor' in e: - cursor = e.split('=')[1] - self._get_sites_internal(siteList, scope, doc) - - else: - doc = self._get_xmldata(scope, SITESPI) - self._get_sites_internal(siteList, scope, doc) - - except input.ConnectorError as e: - raise e - - except Exception as e: - raise e - - def _get_service_groups(self, groupList, scope, doc): - try: - doc = self._get_xmldata(scope, SERVGROUPPI) - groups = doc.getElementsByTagName('SERVICE_GROUP') - for group in groups: - groupId = group.getAttribute('PRIMARY_KEY') - if groupId not in groupList: - groupList[groupId] = {} - groupList[groupId]['name'] = getText(group.getElementsByTagName('NAME')[0].childNodes) - groupList[groupId]['monitored'] = getText(group.getElementsByTagName('MONITORED')[0].childNodes) - groupList[groupId]['scope'] = scope.split('=')[1] - groupList[groupId]['services'] = [] - services = group.getElementsByTagName('SERVICE_ENDPOINT') - for service in services: - serviceDict = {} - serviceDict['hostname'] = getText(service.getElementsByTagName('HOSTNAME')[0].childNodes) - try: - serviceDict['service_id'] = getText(service.getElementsByTagName('PRIMARY_KEY')[0].childNodes) - except IndexError: - serviceDict['service_id'] = service.getAttribute('PRIMARY_KEY') - serviceDict['type'] = getText(service.getElementsByTagName('SERVICE_TYPE')[0].childNodes) - serviceDict['monitored'] = getText(service.getElementsByTagName('NODE_MONITORED')[0].childNodes) - serviceDict['production'] = getText(service.getElementsByTagName('IN_PRODUCTION')[0].childNodes) - groupList[groupId]['services'].append(serviceDict) - - except (KeyError, IndexError, TypeError, AttributeError, AssertionError) as e: - logger.error(module_class_name(self) + 'Customer:%s Job:%s : Error parsing feed %s - %s' % (logger.customer, logger.job, self._o.scheme + '://' + self._o.netloc + SERVGROUPPI, - repr(e).replace('\'','').replace('\"', ''))) - raise e - - def getServiceGroups(self, groupList, scope): - try: - if self.paging: - count, cursor = 1, 0 - while count != 0: - doc = self._get_xmldata(scope, SERVGROUPPI + '&next_cursor=' + str(cursor)) - count = int(doc.getElementsByTagName('count')[0].childNodes[0].data) - links = doc.getElementsByTagName('link') - for le in links: - if le.getAttribute('rel') == 'next': - href = le.getAttribute('href') - for e in href.split('&'): - if 'next_cursor' in e: - cursor = e.split('=')[1] - self._get_service_groups(groupList, scope, doc) - - else: - doc = self._get_xmldata(scope, SERVGROUPPI) - self._get_service_groups(groupList, scope, doc) - - except input.ConnectorError as e: - raise e - - except Exception as e: - raise e - - -class TopoFilter(object): - def __init__(self, gg, ge, ggfilter, gefilter): - self.gg = gg - self.ge = ge - self.ggfilter = copy.copy(ggfilter) - self.gefilter = copy.copy(gefilter) - self.subgroupfilter = self.extract_filter('site', self.ggfilter) or \ - self.extract_filter('servicegroup', self.ggfilter) - self.groupfilter = self.extract_filter('ngi', self.ggfilter) - self.topofilter() - - def topofilter(self): - if self.subgroupfilter: - self.gg = filter(lambda e: e['subgroup'].lower() in self.subgroupfilter, self.gg) - - if self.groupfilter: - self.gg = filter(lambda e: e['group'].lower() in self.groupfilter, self.gg) - - if self.ggfilter: - self.gg = self.filter_tags(self.ggfilter, self.gg) - - allsubgroups = set([e['subgroup'] for e in self.gg]) - if allsubgroups: - self.ge = filter(lambda e: e['group'] in allsubgroups, self.ge) - - if self.gefilter: - self.ge = self.filter_tags(self.gefilter, self.ge) - - def extract_filter(self, tag, ggtags): - gg = None - if tag.lower() in [t.lower() for t in ggtags.iterkeys()]: - for k, v in ggtags.iteritems(): - if tag.lower() in k.lower(): - gg = ggtags[k] - key = k - ggtags.pop(key) - if isinstance(gg, list): - gg = [t.lower() for t in gg] - else: - gg = gg.lower() - - return gg - - def filter_tags(self, tags, listofelem): - for attr in tags.keys(): - def getit(elem): - value = elem['tags'][attr.lower()] - if value == '1': - value = 'Y' - elif value == '0': - value = 'N' - if isinstance(tags[attr], list): - for a in tags[attr]: - if value.lower() == a.lower(): - return True - else: - if value.lower() == tags[attr].lower(): - return True - try: - listofelem = filter(getit, listofelem) - except KeyError as e: - logger.error('Customer:%s Job:%s : Wrong tags specified: %s' % (logger.customer, logger.job, e)) - return listofelem +def parse_source_servicegroups(res, custname, uidservtype, pass_extensions): + group_groups = ParseServiceGroups(logger, res, custname, uidservtype, + pass_extensions).get_group_groups() + group_endpoints = ParseServiceGroups(logger, res, custname, uidservtype, + pass_extensions).get_group_endpoints() + + return group_groups, group_endpoints + + +def parse_source_endpoints(res, custname, uidservtype, pass_extensions): + group_endpoints = ParseServiceEndpoints(logger, res, custname, uidservtype, + pass_extensions).get_group_endpoints() + + return group_endpoints + + +def parse_source_sites(res, custname, uidservtype, pass_extensions): + group_groups = ParseSites(logger, res, custname, uidservtype, + pass_extensions).get_group_groups() + + return group_groups + + +def parse_source_sitescontacts(res, custname): + contacts = ParseSiteContacts(logger, res) + return contacts.get_contacts() + + +def parse_source_siteswithcontacts(res, custname): + contacts = ParseSitesWithContacts(logger, res) + return contacts.get_contacts() + +def parse_source_servicegroupscontacts(res, custname): + contacts = ParseServiceGroupWithContacts(logger, res) + return contacts.get_contacts() + +def parse_source_servicegroupsroles(res, custname): + contacts = ParseServiceGroupRoles(logger, res) + return contacts.get_contacts() + +def parse_source_serviceendpoints_contacts(res, custname): + contacts = ParseServiceEndpointContacts(logger, res) + return contacts.get_contacts() + + +def get_webapi_opts(cglob, confcust): + webapi_custopts = confcust.get_webapiopts() + webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') + webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') + if not webapi_complete: + logger.error('Customer:%s %s options incomplete, missing %s' % (logger.customer, 'webapi', ' '.join(missopt))) + raise SystemExit(1) + return webapi_opts + + +async def write_state(confcust, fixed_date, state): + # safely assume here one customer defined in customer file + cust = list(confcust.get_customers())[0] + statedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust) + if fixed_date: + await state_write(sys.argv[0], statedir, state, + globopts['InputStateDays'.lower()], + fixed_date.replace('-', '_')) + else: + await state_write(sys.argv[0], statedir, state, + globopts['InputStateDays'.lower()]) + + +def find_next_paging_cursor_count(res): + cursor, count = None, None + + doc = xml.dom.minidom.parseString(res) + count = int(doc.getElementsByTagName('count')[0].childNodes[0].data) + links = doc.getElementsByTagName('link') + for link in links: + if link.getAttribute('rel') == 'next': + href = link.getAttribute('href') + for query in href.split('&'): + if 'next_cursor' in query: + cursor = query.split('=')[1] + + return count, cursor + + +def filter_multiple_tags(data): + """ + Paginated content is represented with multiple XML enclosing tags + in a single buffer: + + + + topology entities + + + + topology entities + + ... + + Remove them and leave only one enclosing. + """ + data_lines = data.split('\n') + data_lines = list(filter(lambda line: + '' not in line + and '' not in line + and '') + data_lines.insert(1, '') + data_lines.append('') + return '\n'.join(data_lines) + + +async def fetch_data(api, auth_opts, paginated): + feed_parts = urlparse(api) + fetched_data = list() + if paginated: + count, cursor = 1, 0 + while count != 0: + session = SessionWithRetry(logger, os.path.basename(sys.argv[0]), + globopts, custauth=auth_opts) + res = await session.http_get('{}&next_cursor={}'.format(api, + cursor)) + count, cursor = find_next_paging_cursor_count(res) + fetched_data.append(res) + return filter_multiple_tags(''.join(fetched_data)) + + else: + session = SessionWithRetry(logger, os.path.basename(sys.argv[0]), + globopts, custauth=auth_opts) + res = await session.http_get(api) + return res + + +async def send_webapi(webapi_opts, data, topotype, fixed_date=None): + webapi = WebAPI(sys.argv[0], webapi_opts['webapihost'], + webapi_opts['webapitoken'], logger, + int(globopts['ConnectionRetry'.lower()]), + int(globopts['ConnectionTimeout'.lower()]), + int(globopts['ConnectionSleepRetry'.lower()]), + date=fixed_date) + await webapi.send(data, topotype) + + +def write_avro(confcust, group_groups, group_endpoints, fixed_date): + custdir = confcust.get_custdir() + if fixed_date: + filename = filename_date(logger, globopts['OutputTopologyGroupOfGroups'.lower()], custdir, fixed_date.replace('-', '_')) + else: + filename = filename_date(logger, globopts['OutputTopologyGroupOfGroups'.lower()], custdir) + avro = AvroWriter(globopts['AvroSchemasTopologyGroupOfGroups'.lower()], filename) + ret, excep = avro.write(group_groups) + if not ret: + logger.error('Customer:%s : %s' % (logger.customer, repr(excep))) + raise SystemExit(1) + + if fixed_date: + filename = filename_date(logger, globopts['OutputTopologyGroupOfEndpoints'.lower()], custdir, fixed_date.replace('-', '_')) + else: + filename = filename_date(logger, globopts['OutputTopologyGroupOfEndpoints'.lower()], custdir) + avro = AvroWriter(globopts['AvroSchemasTopologyGroupOfEndpoints'.lower()], filename) + ret, excep = avro.write(group_endpoints) + if not ret: + logger.error('Customer:%s: %s' % (logger.customer, repr(excep))) + raise SystemExit(1) + + +def get_bdii_opts(confcust): + bdii_custopts = confcust._get_cust_options('BDIIOpts') + if bdii_custopts: + bdii_complete, missing = confcust.is_complete_bdii(bdii_custopts) + if not bdii_complete: + logger.error('%s options incomplete, missing %s' % ('bdii', ' '.join(missing))) + raise SystemExit(1) + return bdii_custopts + else: + return None + + +# Fetches data from LDAP, connection parameters are set in customer.conf +async def fetch_ldap_data(host, port, base, filter, attributes): + ldap_session = LDAPSessionWithRetry(logger, int(globopts['ConnectionRetry'.lower()]), + int(globopts['ConnectionSleepRetry'.lower()]), int(globopts['ConnectionTimeout'.lower()])) + + res = await ldap_session.search(host, port, base, filter, attributes) + return res + + +def contains_exception(list): + for a in list: + if isinstance(a, Exception): + return (True, a) + + return (False, None) def main(): @@ -434,135 +290,236 @@ def main(): confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() + pass_extensions = eval(globopts['GeneralPassExtensions'.lower()]) confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) - feeds = confcust.get_mapfeedjobs(sys.argv[0], 'GOCDB', deffeed='https://goc.egi.eu/gocdbpi/') - - for feed, jobcust in feeds.items(): - scopes = confcust.get_feedscopes(feed, jobcust) - paging = confcust.is_paginated(feed, jobcust) - auth_custopts = confcust.get_authopts(feed, jobcust) - auth_opts = cglob.merge_opts(auth_custopts, 'authentication') - auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') - if auth_complete: - gocdb = GOCDBReader(feed, scopes, paging, auth=auth_opts) + topofeed = confcust.get_topofeed() + topofeedpaging = confcust.get_topofeedpaging() + uidservtype = confcust.get_uidserviceendpoints() + topofetchtype = confcust.get_topofetchtype() + custname = confcust.get_custname() + logger.customer = custname + + auth_custopts = confcust.get_authopts() + auth_opts = cglob.merge_opts(auth_custopts, 'authentication') + auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') + if not auth_complete: + logger.error('%s options incomplete, missing %s' % ('authentication', ' '.join(missing))) + raise SystemExit(1) + + bdii_opts = get_bdii_opts(confcust) + + loop = uvloop.new_event_loop() + asyncio.set_event_loop(loop) + + group_endpoints, group_groups = list(), list() + parsed_site_contacts, parsed_servicegroups_contacts, parsed_serviceendpoint_contacts = None, None, None + + try: + contact_coros = [ + fetch_data(topofeed + SITE_CONTACTS, auth_opts, False), + fetch_data(topofeed + SERVICEGROUP_CONTACTS, auth_opts, False) + ] + contacts = loop.run_until_complete(asyncio.gather(*contact_coros, return_exceptions=True)) + + exc_raised, exc = contains_exception(contacts) + if exc_raised: + raise ConnectorHttpError(repr(exc)) + + parsed_site_contacts = parse_source_sitescontacts(contacts[0], custname) + parsed_servicegroups_contacts = parse_source_servicegroupsroles(contacts[1], custname) + + + except (ConnectorHttpError, ConnectorParseError) as exc: + logger.warn('SITE_CONTACTS and SERVICERGOUP_CONTACT methods not implemented') + + try: + toposcope = confcust.get_toposcope() + topofeedendpoints = confcust.get_topofeedendpoints() + topofeedservicegroups = confcust.get_topofeedservicegroups() + topofeedsites = confcust.get_topofeedsites() + global SERVICE_ENDPOINTS_PI, SERVICE_GROUPS_PI, SITES_PI + if toposcope: + SERVICE_ENDPOINTS_PI = SERVICE_ENDPOINTS_PI + toposcope + SERVICE_GROUPS_PI = SERVICE_GROUPS_PI + toposcope + SITES_PI = SITES_PI + toposcope + if topofeedendpoints: + SERVICE_ENDPOINTS_PI = topofeedendpoints + else: + SERVICE_ENDPOINTS_PI = topofeed + SERVICE_ENDPOINTS_PI + if topofeedservicegroups: + SERVICE_GROUPS_PI = topofeedservicegroups + else: + SERVICE_GROUPS_PI = topofeed + SERVICE_GROUPS_PI + if topofeedsites: + SITES_PI = topofeedsites else: - logger.error('%s options incomplete, missing %s' % ('authentication', ' '.join(missing))) - continue - - for job, cust in jobcust: - jobdir = confcust.get_fulldir(cust, job) - jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust, job) - - global fetchtype, custname - fetchtype = confcust.get_fetchtype(job) - uidservtype = confcust.pass_uidserviceendpoints(job) - custname = confcust.get_custname(cust) - - logger.customer = custname - logger.job = job - - ams_custopts = confcust.get_amsopts(cust) - ams_opts = cglob.merge_opts(ams_custopts, 'ams') - ams_complete, missopt = cglob.is_complete(ams_opts, 'ams') - if not ams_complete: - logger.error('Customer:%s Job:%s %s options incomplete, missing %s' % (custname, logger.job, 'ams', ' '.join(missopt))) - continue - - if fetchtype == 'ServiceGroups': - group_endpoints = gocdb.getGroupOfServices(uidservtype) - else: - group_endpoints = gocdb.getGroupOfEndpoints(uidservtype) - group_groups = gocdb.getGroupOfGroups() - - if fixed_date: - output.write_state(sys.argv[0], jobstatedir, gocdb.state, - globopts['InputStateDays'.lower()], - fixed_date.replace('-', '_')) - else: - output.write_state(sys.argv[0], jobstatedir, gocdb.state, - globopts['InputStateDays'.lower()]) - - if not gocdb.state: - continue - - numge = len(group_endpoints) - numgg = len(group_groups) - - ggtags = confcust.get_gocdb_ggtags(job) - getags = confcust.get_gocdb_getags(job) - tf = TopoFilter(group_groups, group_endpoints, ggtags, getags) - group_groups = tf.gg - group_endpoints = tf.ge - - if eval(globopts['GeneralPublishAms'.lower()]): - if fixed_date: - partdate = fixed_date - else: - partdate = datestamp(1).replace('_', '-') - - ams = output.AmsPublish(ams_opts['amshost'], - ams_opts['amsproject'], - ams_opts['amstoken'], - ams_opts['amstopic'], - confcust.get_jobdir(job), - ams_opts['amsbulk'], - ams_opts['amspacksinglemsg'], - logger, - int(globopts['ConnectionRetry'.lower()]), - int(globopts['ConnectionTimeout'.lower()])) - - ams.send(globopts['AvroSchemasTopologyGroupOfGroups'.lower()], - 'group_groups', partdate, group_groups) - - ams.send(globopts['AvroSchemasTopologyGroupOfEndpoints'.lower()], - 'group_endpoints', partdate, group_endpoints) - - if eval(globopts['GeneralWriteAvro'.lower()]): - if fixed_date: - filename = filename_date(logger, globopts['OutputTopologyGroupOfGroups'.lower()], jobdir, fixed_date.replace('-', '_')) - else: - filename = filename_date(logger, globopts['OutputTopologyGroupOfGroups'.lower()], jobdir) - avro = output.AvroWriter(globopts['AvroSchemasTopologyGroupOfGroups'.lower()], filename) - ret, excep = avro.write(group_groups) - if not ret: - logger.error('Customer:%s Job:%s : %s' % (logger.customer, logger.job, repr(excep))) - raise SystemExit(1) - - if fixed_date: - filename = filename_date(logger, globopts['OutputTopologyGroupOfEndpoints'.lower()], jobdir, fixed_date.replace('-', '_')) - else: - filename = filename_date(logger, globopts['OutputTopologyGroupOfEndpoints'.lower()], jobdir) - avro = output.AvroWriter(globopts['AvroSchemasTopologyGroupOfEndpoints'.lower()], filename) - ret, excep = avro.write(group_endpoints) - if not ret: - logger.error('Customer:%s Job:%s : %s' % (logger.customer, logger.job, repr(excep))) - raise SystemExit(1) - - logger.info('Customer:' + custname + ' Job:' + job + ' Fetched Endpoints:%d' % (numge) + ' Groups(%s):%d' % (fetchtype, numgg)) - if getags or ggtags: - selstr = 'Customer:%s Job:%s Selected ' % (custname, job) - selge, selgg = '', '' - if getags: - for key, value in getags.items(): - if isinstance(value, list): - value = '[' + ','.join(value) + ']' - selge += '%s:%s,' % (key, value) - selstr += 'Endpoints(%s):' % selge[:len(selge) - 1] - selstr += '%d ' % (len(group_endpoints)) - if ggtags: - for key, value in ggtags.items(): - if isinstance(value, list): - value = '[' + ','.join(value) + ']' - selgg += '%s:%s,' % (key, value) - selstr += 'Groups(%s):' % selgg[:len(selgg) - 1] - selstr += '%d' % (len(group_groups)) - - logger.info(selstr) + SITES_PI = topofeed + SITES_PI + + fetched_sites, fetched_servicegroups, fetched_endpoints = None, None, None + fetched_bdii = None + + coros = [fetch_data(SERVICE_ENDPOINTS_PI, auth_opts, topofeedpaging)] + if 'servicegroups' in topofetchtype: + coros.append(fetch_data(SERVICE_GROUPS_PI, auth_opts, topofeedpaging)) + if 'sites' in topofetchtype: + coros.append(fetch_data(SITES_PI, auth_opts, topofeedpaging)) + + if bdii_opts and eval(bdii_opts['bdii']): + host = bdii_opts['bdiihost'] + port = bdii_opts['bdiiport'] + base = bdii_opts['bdiiquerybase'] + + coros.append(fetch_ldap_data(host, port, base, + bdii_opts['bdiiqueryfiltersrm'], + bdii_opts['bdiiqueryattributessrm'].split(' '))) + + coros.append(fetch_ldap_data(host, port, base, + bdii_opts['bdiiqueryfiltersepath'], + bdii_opts['bdiiqueryattributessepath'].split(' '))) + + # fetch topology data concurrently in coroutines + fetched_topology = loop.run_until_complete(asyncio.gather(*coros, return_exceptions=True)) + + fetched_endpoints = fetched_topology[0] + if bdii_opts and eval(bdii_opts['bdii']): + fetched_bdii = list() + fetched_bdii.append(fetched_topology[-2]) + fetched_bdii.append(fetched_topology[-1]) + if 'sites' in topofetchtype and 'servicegroups' in topofetchtype: + fetched_servicegroups, fetched_sites = (fetched_topology[1], fetched_topology[2]) + elif 'sites' in topofetchtype: + fetched_sites = fetched_topology[1] + elif 'servicegroups' in topofetchtype: + fetched_servicegroups = fetched_topology[1] + + exc_raised, exc = contains_exception(fetched_topology) + if exc_raised: + raise ConnectorHttpError(repr(exc)) + + # proces data in parallel using multiprocessing + executor = ProcessPoolExecutor(max_workers=3) + parse_workers = list() + exe_parse_source_endpoints = partial(parse_source_endpoints, + fetched_endpoints, custname, + uidservtype, pass_extensions) + exe_parse_source_servicegroups = partial(parse_source_servicegroups, + fetched_servicegroups, + custname, uidservtype, + pass_extensions) + exe_parse_source_sites = partial(parse_source_sites, fetched_sites, + custname, uidservtype, + pass_extensions) + + # parse topology depend on configured components fetch. we can fetch + # only sites, only servicegroups or both. + if fetched_servicegroups and fetched_sites: + parse_workers.append( + loop.run_in_executor(executor, exe_parse_source_endpoints) + ) + parse_workers.append( + loop.run_in_executor(executor, exe_parse_source_servicegroups) + ) + parse_workers.append( + loop.run_in_executor(executor, exe_parse_source_sites) + ) + elif fetched_servicegroups and not fetched_sites: + parse_workers.append( + loop.run_in_executor(executor, exe_parse_source_servicegroups) + ) + elif fetched_sites and not fetched_servicegroups: + parse_workers.append( + loop.run_in_executor(executor, exe_parse_source_endpoints) + ) + parse_workers.append( + loop.run_in_executor(executor, exe_parse_source_sites) + ) + + parsed_topology = loop.run_until_complete(asyncio.gather(*parse_workers)) + + if fetched_servicegroups and fetched_sites: + group_endpoints = parsed_topology[0] + group_groups, group_endpoints_sg = parsed_topology[1] + group_endpoints += group_endpoints_sg + group_groups += parsed_topology[2] + elif fetched_servicegroups and not fetched_sites: + group_groups, group_endpoints = parsed_topology[0] + elif fetched_sites and not fetched_servicegroups: + group_endpoints = parsed_topology[0] + group_groups = parsed_topology[1] + + # check if we fetched SRM port info and attach it appropriate endpoint + # data + if bdii_opts and eval(bdii_opts['bdii']): + attach_srmport_topodata(logger, bdii_opts['bdiiqueryattributessrm'].split(' ')[0], fetched_bdii[0], group_endpoints) + attach_sepath_topodata(logger, bdii_opts['bdiiqueryattributessepath'].split(' ')[0], fetched_bdii[1], group_endpoints) + + # parse contacts from fetched service endpoints topology, if there are + # any + parsed_serviceendpoint_contacts = parse_source_serviceendpoints_contacts(fetched_endpoints, custname) + + if not parsed_site_contacts and fetched_sites: + # GOCDB has not SITE_CONTACTS, try to grab contacts from fetched + # sites topology entities + parsed_site_contacts = parse_source_siteswithcontacts(fetched_sites, custname) + + attach_contacts_workers = [ + loop.run_in_executor(executor, partial(attach_contacts_topodata, + logger, + parsed_site_contacts, + group_groups)), + loop.run_in_executor(executor, partial(attach_contacts_topodata, + logger, + parsed_serviceendpoint_contacts, + group_endpoints)) + ] + + executor = ProcessPoolExecutor(max_workers=2) + group_groups, group_endpoints = loop.run_until_complete(asyncio.gather(*attach_contacts_workers)) + + if parsed_servicegroups_contacts: + attach_contacts_topodata(logger, parsed_servicegroups_contacts, group_groups) + elif fetched_servicegroups: + # GOCDB has not SERVICEGROUP_CONTACTS, try to grab contacts from fetched + # servicegroups topology entities + parsed_servicegroups_contacts = parse_source_servicegroupscontacts(fetched_servicegroups, custname) + attach_contacts_topodata(logger, parsed_servicegroups_contacts, group_groups) + + loop.run_until_complete( + write_state(confcust, fixed_date, True) + ) + + webapi_opts = get_webapi_opts(cglob, confcust) + + numge = len(group_endpoints) + numgg = len(group_groups) + + # send concurrently to WEB-API in coroutines + if eval(globopts['GeneralPublishWebAPI'.lower()]): + loop.run_until_complete( + asyncio.gather( + send_webapi(webapi_opts, group_groups, 'groups', fixed_date), + send_webapi(webapi_opts, group_endpoints,'endpoints', fixed_date) + ) + ) + + if eval(globopts['GeneralWriteAvro'.lower()]): + write_avro(confcust, group_groups, group_endpoints, fixed_date) + + logger.info('Customer:' + custname + ' Type:%s ' % (','.join(topofetchtype)) + 'Fetched Endpoints:%d' % (numge) + ' Groups:%d' % (numgg)) + + except (ConnectorParseError, ConnectorHttpError, KeyboardInterrupt) as exc: + logger.error(repr(exc)) + loop.run_until_complete( + write_state(confcust, fixed_date, False) + ) + + finally: + loop.close() if __name__ == '__main__': diff --git a/bin/topology-json-connector.py b/bin/topology-json-connector.py new file mode 100755 index 00000000..fb1c3392 --- /dev/null +++ b/bin/topology-json-connector.py @@ -0,0 +1,197 @@ +#!/usr/bin/python3 + +import argparse +import os +import sys +import json + +import uvloop +import asyncio + +from argo_egi_connectors.io.http import SessionWithRetry +from argo_egi_connectors.exceptions import ConnectorHttpError, ConnectorParseError +from argo_egi_connectors.io.webapi import WebAPI +from argo_egi_connectors.io.avrowrite import AvroWriter +from argo_egi_connectors.io.statewrite import state_write +from argo_egi_connectors.log import Logger +from argo_egi_connectors.config import Global, CustomerConf +from argo_egi_connectors.utils import filename_date, datestamp, date_check +from argo_egi_connectors.parse.flat_topology import ParseFlatEndpoints, ParseContacts +from argo_egi_connectors.mesh.contacts import attach_contacts_topodata + +from urllib.parse import urlparse + +logger = None +globopts = {} +custname = '' + + +def is_feed(feed): + data = urlparse(feed) + + if not data.netloc: + return False + else: + return True + + +async def send_webapi(webapi_opts, data, topotype, fixed_date=None): + webapi = WebAPI(sys.argv[0], webapi_opts['webapihost'], + webapi_opts['webapitoken'], logger, + int(globopts['ConnectionRetry'.lower()]), + int(globopts['ConnectionTimeout'.lower()]), + int(globopts['ConnectionSleepRetry'.lower()]), + date=fixed_date) + await webapi.send(data, topotype) + + +def get_webapi_opts(cglob, confcust): + webapi_custopts = confcust.get_webapiopts() + webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') + webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') + if not webapi_complete: + logger.error('Customer:%s %s options incomplete, missing %s' % (logger.customer, 'webapi', ' '.join(missopt))) + raise SystemExit(1) + return webapi_opts + + +async def fetch_data(feed): + remote_topo = urlparse(feed) + session = SessionWithRetry(logger, custname, globopts) + res = await session.http_get('{}://{}{}'.format(remote_topo.scheme, + remote_topo.netloc, + remote_topo.path)) + return res + + +def parse_source_topo(res, uidservtype, fetchtype): + # group_groups, group_endpoints = ParseEoscTopo(logger, res, uidservtype, fetchtype).get_data() + topo = ParseFlatEndpoints(logger, res, custname, uidservtype, fetchtype, scope=custname) + group_groups = topo.get_groupgroups() + group_endpoints = topo.get_groupendpoints() + + return group_groups, group_endpoints + + +async def write_state(confcust, fixed_date, state): + cust = list(confcust.get_customers())[0] + jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust) + fetchtype = confcust.get_topofetchtype() + if fixed_date: + await state_write(sys.argv[0], jobstatedir, state, + globopts['InputStateDays'.lower()], + fixed_date.replace('-', '_')) + else: + await state_write(sys.argv[0], jobstatedir, state, + globopts['InputStateDays'.lower()]) + + +def write_avro(confcust, group_groups, group_endpoints, fixed_date): + custdir = confcust.get_custdir() + if fixed_date: + filename = filename_date(logger, globopts['OutputTopologyGroupOfGroups'.lower()], custdir, fixed_date.replace('-', '_')) + else: + filename = filename_date(logger, globopts['OutputTopologyGroupOfGroups'.lower()], custdir) + avro = AvroWriter(globopts['AvroSchemasTopologyGroupOfGroups'.lower()], filename) + ret, excep = avro.write(group_groups) + if not ret: + logger.error('Customer:%s : %s' % (logger.customer, repr(excep))) + raise SystemExit(1) + + if fixed_date: + filename = filename_date(logger, globopts['OutputTopologyGroupOfEndpoints'.lower()], custdir, fixed_date.replace('-', '_')) + else: + filename = filename_date(logger, globopts['OutputTopologyGroupOfEndpoints'.lower()], custdir) + avro = AvroWriter(globopts['AvroSchemasTopologyGroupOfEndpoints'.lower()], filename) + ret, excep = avro.write(group_endpoints) + if not ret: + logger.error('Customer:%s : %s' % (logger.customer, repr(excep))) + raise SystemExit(1) + + +def main(): + global logger, globopts, confcust + + parser = argparse.ArgumentParser(description="""Fetch and construct entities from EOSC-PORTAL feed""") + parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False) + parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) + parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) + args = parser.parse_args() + group_endpoints, group_groups = list(), list() + logger = Logger(os.path.basename(sys.argv[0])) + + fixed_date = None + if args.date and date_check(args.date): + fixed_date = args.date + + confpath = args.gloconf[0] if args.gloconf else None + cglob = Global(sys.argv[0], confpath) + globopts = cglob.parse() + + confpath = args.custconf[0] if args.custconf else None + confcust = CustomerConf(sys.argv[0], confpath) + confcust.parse() + confcust.make_dirstruct() + confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) + global custname + custname = confcust.get_custname() + + # safely assume here one customer defined in customer file + cust = list(confcust.get_customers())[0] + jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust) + fetchtype = confcust.get_topofetchtype()[0] + + state = None + logger.customer = custname + uidservtype = confcust.get_uidserviceendpoints() + topofeed = confcust.get_topofeed() + + loop = uvloop.new_event_loop() + asyncio.set_event_loop(loop) + + try: + if is_feed(topofeed): + res = loop.run_until_complete(fetch_data(topofeed)) + group_groups, group_endpoints = parse_source_topo(res, uidservtype, fetchtype) + contacts = ParseContacts(logger, res, uidservtype, is_csv=False).get_contacts() + attach_contacts_topodata(logger, contacts, group_endpoints) + else: + try: + with open(topofeed) as fp: + js = json.load(fp) + group_groups, group_endpoints = parse_source_topo(js, uidservtype, fetchtype) + except IOError as exc: + logger.error('Customer:%s : Problem opening %s - %s' % (logger.customer, topofeed, repr(exc))) + + loop.run_until_complete( + write_state(confcust, fixed_date, True) + ) + + webapi_opts = get_webapi_opts(cglob, confcust) + + numge = len(group_endpoints) + numgg = len(group_groups) + + # send concurrently to WEB-API in coroutines + if eval(globopts['GeneralPublishWebAPI'.lower()]): + loop.run_until_complete( + asyncio.gather( + send_webapi(webapi_opts, group_groups, 'groups', fixed_date), + send_webapi(webapi_opts, group_endpoints,'endpoints', fixed_date) + ) + ) + + if eval(globopts['GeneralWriteAvro'.lower()]): + write_avro(confcust, group_groups, group_endpoints, fixed_date) + + logger.info('Customer:' + custname + ' Fetched Endpoints:%d' % (numge) + ' Groups(%s):%d' % (fetchtype, numgg)) + + except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc: + logger.error(repr(exc)) + loop.run_until_complete( + write_state(confcust, fixed_date, False ) + ) + + +if __name__ == '__main__': + main() diff --git a/bin/weights-vapor-connector.py b/bin/weights-vapor-connector.py index bb12541b..36489562 100755 --- a/bin/weights-vapor-connector.py +++ b/bin/weights-vapor-connector.py @@ -1,4 +1,4 @@ -#!/bin/env python +#!/usr/bin/python3 # Copyright (c) 2013 GRNET S.A., SRCE, IN2P3 CNRS Computing Centre # @@ -28,67 +28,83 @@ import os import sys -from argo_egi_connectors import input -from argo_egi_connectors import output +import uvloop +import asyncio + +from argo_egi_connectors.io.http import SessionWithRetry +from argo_egi_connectors.exceptions import ConnectorHttpError, ConnectorParseError +from argo_egi_connectors.io.webapi import WebAPI +from argo_egi_connectors.io.avrowrite import AvroWriter +from argo_egi_connectors.io.statewrite import state_write from argo_egi_connectors.log import Logger +from argo_egi_connectors.parse.vapor import ParseWeights from argo_egi_connectors.config import Global, CustomerConf -from argo_egi_connectors.helpers import filename_date, module_class_name, datestamp, date_check -from urlparse import urlparse +from argo_egi_connectors.utils import filename_date, module_class_name, date_check +from urllib.parse import urlparse globopts = {} logger = None VAPORPI = 'https://operations-portal.egi.eu/vapor/downloadLavoisier/option/json/view/VAPOR_Ngi_Sites_Info' -class Vapor: - def __init__(self, feed): - self._o = urlparse(feed) - self.state = True - def getWeights(self): - try: - res = input.connection(logger, module_class_name(self), globopts, - self._o.scheme, self._o.netloc, - self._o.path) - if not res: - raise input.ConnectorError() - - json_data = input.parse_json(logger, module_class_name(self), globopts, res, - self._o.scheme + '://' + self._o.netloc + self._o.path) - - if not json_data: - raise input.ConnectorError() - - except input.ConnectorError: - self.state = False - return [] - - else: - try: - weights = dict() - for ngi in json_data: - for site in ngi['site']: - key = site['id'] - if 'ComputationPower' in site: - val = site['ComputationPower'] - else: - logger.warn(module_class_name(self) + ': No ComputationPower value for NGI:%s Site:%s' % (ngi['ngi'] ,site['id'])) - val = '0' - weights[key] = val - return weights - except (KeyError, IndexError) as e: - self.state = False - logger.error(module_class_name(self) + ': Error parsing feed %s - %s' % (self._o.scheme + '://' + self._o.netloc + self._o.path, - repr(e).replace('\'',''))) - - -def data_out(data): - datawr = [] - for key in data: - w = data[key] - datawr.append({'type': 'computationpower', 'site': key, 'weight': w}) - return datawr +async def fetch_data(feed): + feed_parts = urlparse(feed) + session = SessionWithRetry(logger, os.path.basename(sys.argv[0]), globopts) + res = await session.http_get('{}://{}{}'.format(feed_parts.scheme, + feed_parts.netloc, + feed_parts.path)) + return res + + +def parse_source(res): + weights = ParseWeights(logger, res).get_data() + return weights + + +def get_webapi_opts(cust, job, cglob, confcust): + webapi_custopts = confcust.get_webapiopts(cust) + webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') + webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') + if not webapi_complete: + logger.error('Customer:%s Job:%s %s options incomplete, missing %s' % (logger.customer, job, 'webapi', ' '.join(missopt))) + return webapi_opts + + +async def send_webapi(job, confcust, webapi_opts, fixed_date, weights): + webapi = WebAPI(sys.argv[0], webapi_opts['webapihost'], + webapi_opts['webapitoken'], logger, + int(globopts['ConnectionRetry'.lower()]), + int(globopts['ConnectionTimeout'.lower()]), + int(globopts['ConnectionSleepRetry'.lower()]), + report=confcust.get_jobdir(job), endpoints_group='SITES', + date=fixed_date) + await webapi.send(weights) + + +async def write_state(cust, job, confcust, fixed_date, state): + jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust, job) + if fixed_date: + await state_write(sys.argv[0], jobstatedir, state, + globopts['InputStateDays'.lower()], + fixed_date.replace('-', '_')) + else: + await state_write(sys.argv[0], jobstatedir, state, + globopts['InputStateDays'.lower()]) + + +def write_avro(cust, job, confcust, fixed_date, weights): + jobdir = confcust.get_fulldir(cust, job) + if fixed_date: + filename = filename_date(logger, globopts['OutputWeights'.lower()], jobdir, fixed_date.replace('-', '_')) + else: + filename = filename_date(logger, globopts['OutputWeights'.lower()], jobdir) + avro = AvroWriter(globopts['AvroSchemasWeights'.lower()], filename) + ret, excep = avro.write(weights) + if not ret: + logger.error('Customer:%s Job:%s %s' % (logger.customer, logger.job, repr(excep))) + raise SystemExit(1) def main(): @@ -117,11 +133,10 @@ def main(): confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) feeds = confcust.get_mapfeedjobs(sys.argv[0], deffeed=VAPORPI) - j = 0 - for feed, jobcust in feeds.items(): - weights = Vapor(feed) - datawr = None + loop = uvloop.new_event_loop() + asyncio.set_event_loop(loop) + for feed, jobcust in feeds.items(): customers = set(map(lambda jc: confcust.get_custname(jc[1]), jobcust)) customers = customers.pop() if len(customers) == 1 else '({0})'.format(','.join(customers)) sjobs = set(map(lambda jc: jc[0], jobcust)) @@ -129,81 +144,49 @@ def main(): logger.job = jobs logger.customer = customers - for job, cust in jobcust: - logger.customer = confcust.get_custname(cust) - logger.job = job - - write_empty = confcust.send_empty(sys.argv[0], cust) - - if not write_empty: - w = weights.getWeights() - else: - w = [] - weights.state = True - - jobdir = confcust.get_fulldir(cust, job) - jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust, job) - - custname = confcust.get_custname(cust) - ams_custopts = confcust.get_amsopts(cust) - ams_opts = cglob.merge_opts(ams_custopts, 'ams') - ams_complete, missopt = cglob.is_complete(ams_opts, 'ams') - if not ams_complete: - logger.error('Customer:%s %s options incomplete, missing %s' % (custname, 'ams', ' '.join(missopt))) - continue - - if fixed_date: - output.write_state(sys.argv[0], jobstatedir, weights.state, - globopts['InputStateDays'.lower()], - fixed_date.replace('-', '_')) - else: - output.write_state(sys.argv[0], jobstatedir, weights.state, - globopts['InputStateDays'.lower()]) - - if not weights.state: - continue - - datawr = data_out(w) - if eval(globopts['GeneralPublishAms'.lower()]): - if fixed_date: - partdate = fixed_date - else: - partdate = datestamp(1).replace('_', '-') - - ams = output.AmsPublish(ams_opts['amshost'], - ams_opts['amsproject'], - ams_opts['amstoken'], - ams_opts['amstopic'], - confcust.get_jobdir(job), - ams_opts['amsbulk'], - ams_opts['amspacksinglemsg'], - logger, - int(globopts['ConnectionRetry'.lower()]), - int(globopts['ConnectionTimeout'.lower()])) - - ams.send(globopts['AvroSchemasWeights'.lower()], 'weights', - partdate, datawr) - - if eval(globopts['GeneralWriteAvro'.lower()]): - if fixed_date: - filename = filename_date(logger, globopts['OutputWeights'.lower()], jobdir, fixed_date.replace('-', '_')) - else: - filename = filename_date(logger, globopts['OutputWeights'.lower()], jobdir) - avro = output.AvroWriter(globopts['AvroSchemasWeights'.lower()], filename) - ret, excep = avro.write(datawr) - if not ret: - logger.error('Customer:%s Job:%s %s' % (logger.customer, logger.job, repr(excep))) - raise SystemExit(1) - - j += 1 - - if datawr or write_empty: - custs = set([cust for job, cust in jobcust]) - for cust in custs: - jobs = [job for job, lcust in jobcust if cust == lcust] - logger.info('Customer:%s Jobs:%s Sites:%d' % (confcust.get_custname(cust), - jobs[0] if len(jobs) == 1 else '({0})'.format(','.join(jobs)), - len(datawr))) + try: + res = loop.run_until_complete( + fetch_data(feed) + ) + weights = parse_source(res) + + for job, cust in jobcust: + logger.customer = confcust.get_custname(cust) + logger.job = job + + write_empty = confcust.send_empty(sys.argv[0], cust) + + if write_empty: + weights = [] + + webapi_opts = get_webapi_opts(cust, job, cglob, confcust) + + if eval(globopts['GeneralPublishWebAPI'.lower()]): + loop.run_until_complete( + send_webapi(job, confcust, webapi_opts, fixed_date, weights) + ) + + if eval(globopts['GeneralWriteAvro'.lower()]): + write_avro(cust, job, confcust, fixed_date, weights) + + loop.run_until_complete( + write_state(cust, job, confcust, fixed_date, True) + ) + + if weights or write_empty: + custs = set([cust for job, cust in jobcust]) + for cust in custs: + jobs = [job for job, lcust in jobcust if cust == lcust] + logger.info('Customer:%s Jobs:%s Sites:%d' % (confcust.get_custname(cust), + jobs[0] if len(jobs) == 1 else '({0})'.format(','.join(jobs)), + len(weights))) + + except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc: + logger.error(repr(exc)) + for job, cust in jobcust: + loop.run_until_complete( + write_state(cust, job, confcust, fixed_date, False) + ) if __name__ == '__main__': diff --git a/etc/customer.conf.template b/etc/customer.conf.template index baeecb13..210406aa 100644 --- a/etc/customer.conf.template +++ b/etc/customer.conf.template @@ -1,29 +1,27 @@ [CUSTOMER_EGI] Name = EGI OutputDir = /var/lib/argo-connectors/EGI/ -Jobs = JOB_EGICritical, JOB_EGIFedcloud, JOB_EOSCCritical +Jobs = Critical +AuthenticationUsePlainHttpAuth = False +AuthenticationHttpUser = xxxx +AuthenticationHttpPass = xxxx +WebAPIToken = xxxx +TopoFeed = https://goc.egi.eu/gocdbpi/ +TopoFetchType = Sites, ServiceGroups +TopoType = GOCDB +TopoFeedPaging = True +WeightsEmpty = False +DowntimesEmpty = False +BDII = True +BDIIHost = bdii.egi.cro-ngi.hr +BDIIPort = 2170 +BDIIQueryBase = o=grid +BDIIQueryFilterSRM = (&(objectClass=GlueService)(|(GlueServiceType=srm_v1)(GlueServiceType=srm))) +BDIIQueryAttributesSRM = GlueServiceEndpoint +BDIIQueryFilterSEPATH = (objectClass=GlueSATop) +BDIIQueryAttributesSEPATH = GlueVOInfoAccessControlBaseRule GlueVOInfoPath -[JOB_EGICritical] +[Critical] Dirname = EGI_Critical Profiles = ARGO_MON_CRITICAL -MetricProfileNamespace = ch.cern.SAM -TopoFetchType = Sites -TopoSelectGroupOfEndpoints = Production:Y, Monitored:Y, Scope:EGI -TopoSelectGroupOfGroups = NGI:EGI.eu, Certification:Certified, Infrastructure:Production, Scope:EGI -TopoFeedPaging = True - -[JOB_EGIFedcloud] -Dirname = EGI_Fedcloud -Profiles = FEDCLOUD -MetricProfileNamespace = ch.cern.SAM -TopoFetchType = Sites -TopoSelectGroupOfEndpoints = Monitored:Y, Scope:EGI -TopoSelectGroupOfGroups = Scope:EGI -TopoFeedPaging = True - -[JOB_EOSCCritical] -Dirname = Critical -Profiles = EOSC_CRITICAL -TopoFeed = https://marketplace.grid.cyfronet.pl/api/services -TopoFetchType = ServiceGroups -TopoUIDServiceEndpoints = True +MetricProfileNamespace = ch.cern.SAM \ No newline at end of file diff --git a/etc/global.conf.template b/etc/global.conf.template index 1c0a8a5f..e666b36c 100644 --- a/etc/global.conf.template +++ b/etc/global.conf.template @@ -2,16 +2,9 @@ SchemaDir = /etc/argo-egi-connectors/schemas/ [General] -PublishAms = True WriteAvro = True - -[AMS] -Host = messaging-devel.argo.grnet.gr -Token = EGIKEY -Project = EGI -Topic = TOPIC -Bulk = 100 -PackSingleMsg = True +PublishWebAPI = False +PassExtensions = True [Authentication] VerifyServerCert = False @@ -24,7 +17,7 @@ HttpUser = xxxx HttpPass = xxxx [WebAPI] -Token = xxxx +Token = xxxx Host = api.devel.argo.grnet.gr [Connection] diff --git a/etc/schemas/group_endpoints.avsc b/etc/schemas/group_endpoints.avsc index f069db5f..77ca0ae3 100644 --- a/etc/schemas/group_endpoints.avsc +++ b/etc/schemas/group_endpoints.avsc @@ -6,9 +6,28 @@ {"name": "group", "type": "string"}, {"name": "service", "type": "string"}, {"name": "hostname", "type": "string"}, - {"name": "tags", "type" : ["null", { "name" : "Tags", - "type" : "map", - "values" : "string" - }] + {"name": "notifications", "type" : [ "null", + { + "name": "contacts", + "type": "map", + "values": [ + { + "type": "array", + "name": "listOfContacts", + "items": "string" + }, + { + "type": "boolean", + "name": "enabled" + } + ] + } + ]}, + {"name": "tags", "type" : ["null", + { + "name" : "Tags", + "type" : "map", + "values" : "string" + }] }] } diff --git a/etc/schemas/group_groups.avsc b/etc/schemas/group_groups.avsc index abd2d013..1a334be1 100644 --- a/etc/schemas/group_groups.avsc +++ b/etc/schemas/group_groups.avsc @@ -5,9 +5,28 @@ {"name": "type", "type": "string"}, {"name": "group", "type": "string"}, {"name": "subgroup", "type": "string"}, - {"name": "tags", "type" : ["null", { "name" : "Tags", - "type" : "map", - "values" : "string" - }] + {"name": "notifications", "type" : [ "null", + { + "name": "contacts", + "type": "map", + "values": [ + { + "type": "array", + "name": "listOfContacts", + "items": "string" + }, + { + "type": "boolean", + "name": "enabled" + } + ] + } + ]}, + {"name": "tags", "type" : [ "null", + { + "name" : "Tags", + "type" : "map", + "values" : "string" + }] }] } diff --git a/modules/config.py b/modules/config.py index 23bf7d90..d2ad7de5 100644 --- a/modules/config.py +++ b/modules/config.py @@ -1,9 +1,9 @@ -import ConfigParser +import configparser import errno import os import re -from log import Logger +from .log import Logger class Global(object): @@ -11,8 +11,7 @@ class Global(object): Class represents parser for global.conf """ # options common for all connectors - conf_ams = {'AMS': ['Host', 'Token', 'Project', 'Topic', 'Bulk', 'PackSingleMsg']} - conf_general = {'General': ['PublishAms', 'WriteAvro']} + conf_general = {'General': ['WriteAvro', 'PublishWebAPI', 'PassExtensions']} conf_auth = {'Authentication': ['HostKey', 'HostCert', 'CAPath', 'CAFile', 'VerifyServerCert', 'UsePlainHttpAuth', 'HttpUser', 'HttpPass']} @@ -39,12 +38,10 @@ def __init__(self, caller, confpath=None, **kwargs): self._filename = '/etc/argo-egi-connectors/global.conf' if not confpath else confpath self._checkpath = kwargs['checkpath'] if 'checkpath' in kwargs.keys() else False - self.optional.update(self._lowercase_dict(self.conf_ams)) self.optional.update(self._lowercase_dict(self.conf_auth)) self.optional.update(self._lowercase_dict(self.conf_webapi)) - self.shared_secopts = self._merge_dict(self.conf_ams, - self.conf_general, + self.shared_secopts = self._merge_dict(self.conf_general, self.conf_auth, self.conf_conn, self.conf_state, self.conf_webapi) @@ -52,7 +49,7 @@ def __init__(self, caller, confpath=None, **kwargs): self._merge_dict(self.shared_secopts, self.conf_topo_schemas, self.conf_topo_output), - 'topology-eosc-connector.py': + 'topology-json-connector.py': self._merge_dict(self.shared_secopts, self.conf_topo_schemas, self.conf_topo_output), @@ -64,6 +61,10 @@ def __init__(self, caller, confpath=None, **kwargs): self._merge_dict(self.shared_secopts, self.conf_weights_schemas, self.conf_weights_output), + 'topology-csv-connector.py': + self._merge_dict(self.shared_secopts, + self.conf_topo_schemas, + self.conf_topo_output), 'metricprofile-webapi-connector.py': self._merge_dict(self.shared_secopts, self.conf_metricprofile_schemas, @@ -83,7 +84,7 @@ def _merge_dict(self, *args): def _lowercase_dict(self, d): newd = dict() - for k in d.iterkeys(): + for k in d.keys(): opts = [o.lower() for o in d[k]] newd[k.lower()] = opts return newd @@ -108,7 +109,7 @@ def is_complete(self, opts, section): def _concat_sectopt(self, d): opts = list() - for k in d.iterkeys(): + for k in d.keys(): for v in d[k]: opts.append(k + v) @@ -125,7 +126,7 @@ def _one_active(self, options): return False def parse(self): - config = ConfigParser.ConfigParser() + config = configparser.ConfigParser() if not os.path.exists(self._filename): self.logger.error('Could not find %s' % self._filename) @@ -138,9 +139,9 @@ def parse(self): try: for sect, opts in self.caller_secopts.items(): - if (sect.lower() not in lower_section and - sect.lower() not in self.optional.keys()): - raise ConfigParser.NoSectionError(sect.lower()) + if (sect.lower() not in lower_section and sect.lower() not in + self.optional.keys()): + raise configparser.NoSectionError(sect.lower()) for opt in opts: for section in config.sections(): @@ -150,14 +151,14 @@ def parse(self): if self._checkpath and os.path.isfile(optget) is False: raise OSError(errno.ENOENT, optget) - if ('output' in section.lower() and - 'DATE' not in optget): - logger.error('No DATE placeholder in %s' % option) - raise SystemExit(1) + if ('output' in section.lower() and 'DATE' not + in optget): + self.logger.error('No DATE placeholder in %s' % opt) + raise SystemExit(1) - options.update({(sect+opt).lower(): optget}) + options.update({(sect + opt).lower(): optget}) - except ConfigParser.NoOptionError as e: + except configparser.NoOptionError as e: s = e.section.lower() if (s in self.optional.keys() and e.option in self.optional[s]): @@ -171,10 +172,10 @@ def parse(self): self.logger.error('At least one of %s needs to be True' % (', '.join(self._concat_sectopt(self.conf_general)))) raise SystemExit(1) - except ConfigParser.NoOptionError as e: + except configparser.NoOptionError as e: self.logger.error(e.message) raise SystemExit(1) - except ConfigParser.NoSectionError as e: + except configparser.NoSectionError as e: self.logger.error("%s defined" % (e.args[0])) raise SystemExit(1) except OSError as e: @@ -190,23 +191,22 @@ class CustomerConf(object): """ _custattrs = None _cust = {} - _defjobattrs = {'topology-gocdb-connector.py': ['TopoFetchType', - 'TopoSelectGroupOfGroups', - 'TopoSelectGroupOfEndpoints', - 'TopoUIDServiceEndpoints', - 'TopoFeed', - 'TopoFeedPaging'], - 'topology-eosc-connector.py': ['TopoFeed', 'TopoFile', 'TopoFetchType', - 'TopoUIDServiceEndpoints'], + _defjobattrs = {'topology-gocdb-connector.py': [''], + 'topology-json-connector.py': [''], + 'topology-csv-connector.py': [''], 'metricprofile-webapi-connector.py': ['MetricProfileNamespace'], 'downtimes-gocdb-connector.py': ['DowntimesFeed', 'TopoUIDServiceEndpoints'], - 'weights-vapor-connector.py': ['WeightsFeed'] + 'weights-vapor-connector.py': ['WeightsFeed', + 'TopoFetchType'] } _jobs, _jobattrs = {}, None - _cust_optional = ['AmsHost', 'AmsProject', 'AmsToken', 'AmsTopic', - 'AmsPackSingleMsg', 'AuthenticationUsePlainHttpAuth', - 'AuthenticationHttpUser', 'AuthenticationHttpPass', - 'WebAPIToken', 'WeightsEmpty', 'DowntimesEmpty'] + _cust_optional = ['AuthenticationUsePlainHttpAuth', + 'TopoUIDServiceEnpoints', 'AuthenticationHttpUser', + 'AuthenticationHttpPass', 'BDII', 'BDIIHost', 'BDIIPort', + 'BDIIQueryBase', 'BDIIQueryFilterSRM', + 'BDIIQueryAttributesSRM', 'BDIIQueryFilterSEPATH', + 'BDIIQueryAttributesSEPATH', 'WebAPIToken', + 'WeightsEmpty', 'DowntimesEmpty'] tenantdir = '' deftopofeed = 'https://goc.egi.eu/gocdbpi/' @@ -222,7 +222,7 @@ def __init__(self, caller, confpath, **kwargs): self._custattrs = kwargs['custattrs'] def parse(self): - config = ConfigParser.ConfigParser() + config = configparser.ConfigParser() if not os.path.exists(self._filename): self.logger.error('Could not find %s' % self._filename) raise SystemExit(1) @@ -239,36 +239,58 @@ def parse(self): custjobs = [job.strip() for job in custjobs] custdir = config.get(section, 'OutputDir') custname = config.get(section, 'Name') + topofetchtype = config.get(section, 'TopoFetchType') + topofeed = config.get(section, 'TopoFeed') + topotype = config.get(section, 'TopoType') + topouidservendpoints = config.get(section, 'TopoUIDServiceEndpoints', fallback=False) + toposcope = config.get(section, 'TopoScope', fallback=None) + topofeedsites = config.get(section, 'TopoFeedSites', fallback=None) + topofeedendpoints = config.get(section, 'TopoFeedServiceEndpoints', fallback=None) + topofeedservicegroups = config.get(section, 'TopoFeedServiceGroups', fallback=None) + topofeedpaging = config.get(section, 'TopoFeedPaging', fallback='GOCDB') + + if not custdir.endswith('/'): + custdir = '{}/'.format(custdir) for o in lower_custopt: try: code = "optopts.update(%s = config.get(section, '%s'))" % (o, o) - exec code - except ConfigParser.NoOptionError as e: + exec(code) + except configparser.NoOptionError as e: if e.option in lower_custopt: pass else: raise e - except ConfigParser.NoOptionError as e: + except configparser.NoOptionError as e: self.logger.error(e.message) raise SystemExit(1) - self._cust.update({section: {'Jobs': custjobs, 'OutputDir': custdir, 'Name': custname}}) + self._cust.update({section: {'Jobs': custjobs, 'OutputDir': + custdir, 'Name': custname, + 'TopoFetchType': topofetchtype, + 'TopoFeedPaging': topofeedpaging, + 'TopoScope': toposcope, + 'TopoFeed': topofeed, + 'TopoFeedSites': topofeedsites, + 'TopoFeedServiceGroups': topofeedservicegroups, + 'TopoFeedEndpoints': topofeedendpoints, + 'TopoUIDServiceEnpoints': topouidservendpoints, + 'TopoType': topotype}}) if optopts: - ams, auth, webapi, empty_data = {}, {}, {}, {} - for k, v in optopts.iteritems(): - if k.startswith('ams'): - ams.update({k: v}) + auth, webapi, empty_data, bdii = {}, {}, {}, {} + for k, v in optopts.items(): if k.startswith('authentication'): auth.update({k: v}) if k.startswith('webapi'): webapi.update({k: v}) + if k.startswith('bdii'): + bdii.update({k: v}) if k.endswith('empty'): empty_data.update({k: v}) - self._cust[section].update(AmsOpts=ams) self._cust[section].update(AuthOpts=auth) self._cust[section].update(WebAPIOpts=webapi) + self._cust[section].update(BDIIOpts=bdii) self._cust[section].update(EmptyDataOpts=empty_data) if self._custattrs: @@ -276,14 +298,13 @@ def parse(self): if config.has_option(section, attr): self._cust[section].update({attr: config.get(section, attr)}) - for cust in self._cust: for job in self._cust[cust]['Jobs']: if config.has_section(job): try: profiles = config.get(job, 'Profiles') dirname = config.get(job, 'Dirname') - except ConfigParser.NoOptionError as e: + except configparser.NoOptionError as e: self.logger.error(e.message) raise SystemExit(1) @@ -323,36 +344,64 @@ def _dir_from_sect(self, sect, d): def get_jobdir(self, job): return self._dir_from_sect(job, self._jobs) - def get_amsopts(self, cust): - if 'AmsOpts' in self._cust[cust]: - return self._cust[cust]['AmsOpts'] + def get_authopts(self, feed=None, jobcust=None): + if jobcust: + for job, cust in jobcust: + if 'AuthOpts' in self._cust[cust]: + return self._cust[cust]['AuthOpts'] + else: + return dict() else: - return dict() + return self._get_cust_options('AuthOpts') - def get_authopts(self, feed, jobcust): - for job, cust in jobcust: - if 'AuthOpts' in self._cust[cust]: - return self._cust[cust]['AuthOpts'] + def get_bdiiopts(self, cust=None): + if cust: + if 'BDIIOpts' in self._cust[cust]: + return self._cust[cust]['BDIIOpts'] else: return dict() + else: + return self._get_cust_options('BDIIOpts') + + def is_complete_bdii(self, opts): + diff = [] + for opt in self._cust_optional: + if opt.lower().startswith('bdii'): + if opt.lower() not in opts: + diff.append(opt) + if len(diff) > 0: + return (False, diff) + return (True, None) def get_fulldir(self, cust, job): return self.get_custdir(cust) + '/' + self.get_jobdir(job) + '/' - def get_fullstatedir(self, root, cust, job): - return root + '/' + self.get_custname(cust) + '/' + self.get_jobdir(job) + def get_fullstatedir(self, root, cust, job=None): + if job: + return root + '/' + self.get_custname(cust) + '/' + self.get_jobdir(job) + else: + return root + '/' + self.get_custname(cust) + '/' - def get_custdir(self, cust): - return self._dir_from_sect(cust, self._cust) + def get_custdir(self, cust=None): + if cust: + return self._dir_from_sect(cust, self._cust) + else: + return self._get_cust_options('OutputDir') - def get_custname(self, cust): - return self._cust[cust]['Name'] + def get_custname(self, cust=None): + if cust: + return self._cust[cust]['Name'] + else: + return self._get_cust_options('Name') - def get_webapiopts(self, cust): - if 'WebAPIOpts' in self._cust[cust]: - return self._cust[cust]['WebAPIOpts'] + def get_webapiopts(self, cust=None): + if cust: + if 'WebAPIOpts' in self._cust[cust]: + return self._cust[cust]['WebAPIOpts'] + else: + return dict() else: - return dict() + return self._get_cust_options('WebAPIOpts') def make_dirstruct(self, root=None): dirs = [] @@ -424,6 +473,49 @@ def _get_feed(self, job, key): feed = '' return feed + def _get_cust_options(self, opt): + target_option = None + + # safely assume here only one customer definition in the config file + for options in self._cust.values(): + for option in options: + if option.lower() == opt.lower(): + target_option = options[option] + return target_option + + def get_topofeed(self): + return self._get_cust_options('TopoFeed') + + def get_topofeedsites(self): + return self._get_cust_options('TopoFeedSites') + + def get_topofeedendpoints(self): + return self._get_cust_options('TopoFeedEndpoints') + + def get_topofeedservicegroups(self): + return self._get_cust_options('TopoFeedServiceGroups') + + def get_topofeedpaging(self): + return eval(self._get_cust_options('TopoFeedPaging')) + + def get_toposcope(self): + return self._get_cust_options('TopoScope') + + def get_topofetchtype(self): + fetchtype = self._get_cust_options('TopoFetchType') + if ',' in fetchtype: + fetchtype = [type.strip().lower() for type in fetchtype.split(',')] + else: + fetchtype = [fetchtype.lower()] + return fetchtype + + def get_uidserviceendpoints(self): + uidservend = self._get_cust_options('TopoUIDServiceEnpoints') + if isinstance(uidservend, str): + return eval(uidservend) + else: + return False + def _is_paginated(self, job): paging = False @@ -441,20 +533,6 @@ def _update_feeds(self, feeds, feedurl, job, cust): feeds[feedurl] = [] feeds[feedurl].append((job, cust)) - def get_feedscopes(self, feed, jobcust): - distinct_scopes = set() - for job, cust in jobcust: - gg = self._get_tags(job, 'TopoSelectGroupOfGroups') - ge = self._get_tags(job, 'TopoSelectGroupOfEndpoints') - for g in gg.items() + ge.items(): - if 'Scope'.lower() == g[0].lower(): - if isinstance(g[1], list): - distinct_scopes.update(g[1]) - else: - distinct_scopes.update([g[1]]) - - return distinct_scopes - def is_paginated(self, feed, jobcust): paginated = False @@ -489,16 +567,7 @@ def get_mapfeedjobs(self, caller, name=None, deffeed=None): feeds = {} for c in self.get_customers(): for job in self.get_jobs(c): - if 'topology' in caller: - feedurl = self._get_feed(job, 'TopoFile') - if not feedurl: - feedurl = self._get_feed(job, 'TopoFeed') - if feedurl: - self._update_feeds(feeds, feedurl, job, c) - else: - feedurl = deffeed - self._update_feeds(feeds, feedurl, job, c) - elif 'downtimes' in caller: + if 'downtimes' in caller: feedurl = self._get_feed(job, 'DowntimesFeed') if feedurl: self._update_feeds(feeds, feedurl, job, c) @@ -515,12 +584,18 @@ def get_mapfeedjobs(self, caller, name=None, deffeed=None): return feeds - def send_empty(self, caller, cust): + def send_empty(self, caller, cust=None): try: - if 'downtimes' in caller: - return eval(self._cust[cust]['EmptyDataOpts']['downtimesempty']) - elif 'weights' in caller: - return eval(self._cust[cust]['EmptyDataOpts']['weightsempty']) + if cust: + if 'downtimes' in caller: + return eval(self._cust[cust]['EmptyDataOpts']['downtimesempty']) + elif 'weights' in caller: + return eval(self._cust[cust]['EmptyDataOpts']['weightsempty']) + else: + if 'downtimes' in caller: + return eval(self._get_cust_options('EmptyDataOpts')['downtimesempty']) + elif 'weights' in caller: + return eval(self._get_cust_options('EmptyDataOpts')['weightsempty']) except KeyError: return False diff --git a/modules/exceptions.py b/modules/exceptions.py new file mode 100644 index 00000000..4ec54513 --- /dev/null +++ b/modules/exceptions.py @@ -0,0 +1,8 @@ +class ConnectorParseError(Exception): + def __init__(self, msg=None): + self.msg = msg + + +class ConnectorHttpError(Exception): + def __init__(self, msg=None): + self.msg = msg diff --git a/modules/helpers.py b/modules/helpers.py deleted file mode 100644 index 31f6dec0..00000000 --- a/modules/helpers.py +++ /dev/null @@ -1,77 +0,0 @@ -import datetime -import re -import time - -strerr = '' -num_excp_expand = 0 -daysback = 1 - -class retry: - def __init__(self, func): - self.func = func - - def __call__(self, *args, **kwargs): - """ - Decorator that will repeat function calls in case of errors. - - First three arguments of decorated function should be: - - logger object - - prefix of each log msg that is usually name of object - constructing msg - - dictionary holding num of retries, timeout and sleepretry - parameters - """ - result = None - logger = args[0] - objname = args[1] - self.numr = int(args[2]['ConnectionRetry'.lower()]) - self.sleepretry = int(args[2]['ConnectionSleepRetry'.lower()]) - loops = self.numr + 1 - try: - i = 1 - while i <= range(loops): - try: - result = self.func(*args, **kwargs) - except Exception as e: - if i == loops: - raise e - else: - logger.warn('%s %s() Customer:%s Job:%s Retry:%d Sleeping:%d - %s' % - (objname, self.func.__name__, - logger.customer, logger.job, - i, self.sleepretry, repr(e))) - time.sleep(self.sleepretry) - pass - else: - break - i += 1 - except Exception as e: - logger.error('%s %s() Customer:%s Job:%s Giving up - %s' % (objname, self.func.__name__, logger.customer, - logger.job, repr(e))) - return False - return result - -def date_check(arg): - if re.search("[0-9]{4}-[0-9]{2}-[0-9]{2}", arg): - return True - else: - return False - -def datestamp(daysback=None): - if daysback: - dateback = datetime.datetime.now() - datetime.timedelta(days=daysback) - else: - dateback = datetime.datetime.now() - - return str(dateback.strftime('%Y_%m_%d')) - -def filename_date(logger, option, path, stamp=None): - stamp = stamp if stamp else datestamp(daysback) - filename = path + re.sub(r'DATE(.\w+)$', r'%s\1' % stamp, option) - - return filename - -def module_class_name(obj): - name = repr(obj.__class__.__name__) - - return name.replace("'",'') diff --git a/modules/input.py b/modules/input.py deleted file mode 100644 index 1d922621..00000000 --- a/modules/input.py +++ /dev/null @@ -1,126 +0,0 @@ -import base64 -import json -import requests -import socket -import xml.dom.minidom - -from argo_egi_connectors.helpers import retry - -from xml.parsers.expat import ExpatError -from urlparse import urlparse - - -class ConnectorError(Exception): - pass - - -@retry -def connection(logger, msgprefix, globopts, scheme, host, url, custauth=None): - try: - buf = None - - headers = {} - if custauth and msgprefix == 'WebAPI': - headers = {'x-api-key': custauth['WebApiToken'.lower()], - 'Accept': 'application/json'} - elif msgprefix != 'PoemReader' and custauth and eval(custauth['AuthenticationUsePlainHttpAuth'.lower()]): - userpass = base64.b64encode(custauth['AuthenticationHttpUser'.lower()] + ':' - + custauth['AuthenticationHttpPass'.lower()]) - headers = {'Authorization': 'Basic ' + userpass} - - if scheme.startswith('https'): - response = requests.get('https://' + host + url, headers=headers, - cert=(globopts['AuthenticationHostCert'.lower()], - globopts['AuthenticationHostKey'.lower()]), - verify=eval(globopts['AuthenticationVerifyServerCert'.lower()]), - timeout=int(globopts['ConnectionTimeout'.lower()])) - response.raise_for_status() - else: - response = requests.get('http://' + host + url, headers=headers, - timeout=int(globopts['ConnectionTimeout'.lower()])) - - - if response.status_code >= 300 and response.status_code < 400: - headers = response.headers - location = filter(lambda h: 'location' in h[0], headers) - if location: - redir = urlparse(location[0][1]) - else: - raise requests.exceptions.RequestException('No Location header set for redirect') - - return connection(logger, msgprefix, globopts, scheme, redir.netloc, redir.path + '?' + redir.query, custauth=custauth) - - elif response.status_code == 200: - buf = response.content - if not buf: - raise requests.exceptions.RequestException('Empty response') - - else: - raise requests.exceptions.RequestException('response: %s %s' % (response.status_code, response.reason)) - - return buf - - except requests.exceptions.SSLError as e: - if (getattr(e, 'args', False) and type(e.args) == tuple - and type(e.args[0]) == str - and 'timed out' in e.args[0]): - raise e - else: - logger.critical('%sCustomer:%s Job:%s SSL Error %s - %s' % (msgprefix + ' ' if msgprefix else '', - logger.customer, logger.job, - scheme + '://' + host + url, - repr(e))) - return False - - except(socket.error, socket.timeout) as e: - logger.warn('%sCustomer:%s Job:%s Connection error %s - %s' % (msgprefix + ' ' if msgprefix else '', - logger.customer, logger.job, - scheme + '://' + host + url, - repr(e))) - raise e - - except requests.exceptions.RequestException as e: - logger.warn('%sCustomer:%s Job:%s HTTP error %s - %s' % (msgprefix + ' ' if msgprefix else '', - logger.customer, logger.job, - scheme + '://' + host + url, - repr(e))) - raise e - - except Exception as e: - logger.critical('%sCustomer:%s Job:%s Error %s - %s' % (msgprefix + ' ' if msgprefix else '', - logger.customer, logger.job, - scheme + '://' + host + url, - repr(e))) - return False - - -def parse_xml(logger, objname, globopts, buf, method): - try: - doc = xml.dom.minidom.parseString(buf) - - except ExpatError as e: - logger.error(objname + ' Customer:%s Job:%s : Error parsing XML feed %s - %s' % (logger.customer, logger.job, method, repr(e))) - raise ConnectorError() - - except Exception as e: - logger.error(objname + ' Customer:%s Job:%s : Error %s - %s' % (logger.customer, logger.job, method, repr(e))) - raise e - - else: - return doc - - -def parse_json(logger, objname, globopts, buf, method): - try: - doc = json.loads(buf) - - except ValueError as e: - logger.error(objname + ' Customer:%s Job:%s : Error parsing JSON feed %s - %s' % (logger.customer, logger.job, method, repr(e))) - raise ConnectorError() - - except Exception as e: - logger.error(objname + ' Customer:%s Job:%s : Error %s - %s' % (logger.customer, logger.job, method, repr(e))) - raise e - - else: - return doc diff --git a/modules/io/__init__.py b/modules/io/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modules/io/avrowrite.py b/modules/io/avrowrite.py new file mode 100644 index 00000000..b6605318 --- /dev/null +++ b/modules/io/avrowrite.py @@ -0,0 +1,55 @@ +import avro.schema +from avro.datafile import DataFileWriter +from avro.io import DatumWriter, BinaryEncoder +import aiofiles + +from io import BytesIO + + +class AvroWriteException(Exception): + pass + + +def load_schema(schema): + try: + f = open(schema) + schema = avro.schema.parse(f.read()) + return schema + except Exception as e: + raise e + + +class AvroWriter(object): + """ AvroWriter """ + def __init__(self, schema, outfile): + self.schema = schema + self.outfile = outfile + self.datawrite = None + self.avrofile = None + self._load_datawriter() + + def _load_datawriter(self): + try: + lschema = load_schema(self.schema) + self.avrofile = open(self.outfile, 'w+b') + self.datawrite = DataFileWriter(self.avrofile, DatumWriter(), lschema) + except Exception: + return False + + return True + + def write(self, data): + try: + if (not self.datawrite or not self.avrofile): + raise AvroWriteException('AvroFileWriter not initalized') + + for elem in data: + self.datawrite.append(elem) + + self.datawrite.close() + self.avrofile.close() + + except Exception as e: + return False, e + + return True, None diff --git a/modules/io/http.py b/modules/io/http.py new file mode 100644 index 00000000..18eb0585 --- /dev/null +++ b/modules/io/http.py @@ -0,0 +1,200 @@ +import ssl +import asyncio +import aiohttp + +from aiohttp import client_exceptions, http_exceptions, ClientSession +from argo_egi_connectors.utils import module_class_name +from argo_egi_connectors.exceptions import ConnectorHttpError + + +def build_ssl_settings(globopts): + try: + sslcontext = ssl.create_default_context(capath=globopts['AuthenticationCAPath'.lower()], + cafile=globopts['AuthenticationCAFile'.lower()]) + sslcontext.load_cert_chain(globopts['AuthenticationHostCert'.lower()], + globopts['AuthenticationHostKey'.lower()]) + + return sslcontext + + except KeyError: + return None + + +def build_connection_retry_settings(globopts): + retry = int(globopts['ConnectionRetry'.lower()]) + timeout = int(globopts['ConnectionTimeout'.lower()]) + return (retry, timeout) + + +class SessionWithRetry(object): + def __init__(self, logger, msgprefix, globopts, token=None, custauth=None, + verbose_ret=False, handle_session_close=False): + self.ssl_context = build_ssl_settings(globopts) + n_try, client_timeout = build_connection_retry_settings(globopts) + client_timeout = aiohttp.ClientTimeout(total=client_timeout, + connect=client_timeout, sock_connect=client_timeout, + sock_read=client_timeout) + self.session = ClientSession(timeout=client_timeout) + self.n_try = n_try + self.logger = logger + self.token = token + if custauth: + self.custauth = aiohttp.BasicAuth( + custauth['AuthenticationHttpUser'.lower()], + custauth['AuthenticationHttpPass'.lower()] + ) + else: + self.custauth = None + self.verbose_ret = verbose_ret + self.handle_session_close = handle_session_close + self.globopts = globopts + self.erroneous_statuses = [404] + + async def _http_method(self, method, url, data=None, headers=None): + method_obj = getattr(self.session, method) + raised_exc = None + n = 1 + if self.token: + headers = headers or {} + headers.update({ + 'x-api-key': self.token, + 'Accept': 'application/json' + }) + try: + sleepsecs = float(self.globopts['ConnectionSleepRetry'.lower()]) + + while n <= self.n_try: + if n > 1: + if getattr(self.logger, 'job', False): + self.logger.info(f"{module_class_name(self)} Customer:{self.logger.customer} Job:{self.logger.job} : HTTP Connection try - {n} after sleep {sleepsecs} seconds") + else: + self.logger.info(f"{module_class_name(self)} Customer:{self.logger.customer} : HTTP Connection try - {n} after sleep {sleepsecs} seconds") + try: + async with method_obj(url, data=data, headers=headers, + ssl=self.ssl_context, auth=self.custauth) as response: + if response.status in self.erroneous_statuses: + if getattr(self.logger, 'job', False): + self.logger.error('{}.http_{}({}) Customer:{} Job:{} - Erroneous HTTP status: {} {}'.\ + format(module_class_name(self), + method, url, + self.logger.customer, + self.logger.job, + response.status, + response.reason)) + else: + self.logger.error('{}.http_{}({}) Customer:{} - Erroneus HTTP status: {} {}'.\ + format(module_class_name(self), + method, url, + self.logger.customer, + response.status, + response.reason)) + break + content = await response.text() + if content: + if self.verbose_ret: + return (content, response.headers, response.status) + return content + + if getattr(self.logger, 'job', False): + self.logger.warn("{} Customer:{} Job:{} : HTTP Empty response".format(module_class_name(self), + self.logger.customer, self.logger.job)) + else: + self.logger.warn("{} Customer:{} : HTTP Empty response".format(module_class_name(self), + self.logger.customer)) + + # do not retry on SSL errors + # raise exc that will be handled in outer try/except clause + except ssl.SSLError as exc: + raise exc + + # retry on client errors + except (client_exceptions.ClientError, + client_exceptions.ServerTimeoutError, + asyncio.TimeoutError) as exc: + if getattr(self.logger, 'job', False): + self.logger.error('{}.http_{}({}) Customer:{} Job:{} - {}'.format(module_class_name(self), + method, url, self.logger.customer, + self.logger.job, repr(exc))) + else: + self.logger.error('{}.http_{}({}) Customer:{} - {}'.format(module_class_name(self), + method, url, self.logger.customer, + repr(exc))) + raised_exc = exc + + # do not retry on HTTP protocol errors + # raise exc that will be handled in outer try/except clause + except (http_exceptions.HttpProcessingError) as exc: + if getattr(self.logger, 'job', False): + self.logger.error('{}.http_{}({}) Customer:{} Job:{} - {}'.format(module_class_name(self), + method, url, self.logger.customer, + self.logger.job, repr(exc))) + else: + self.logger.error('{}.http_{}({}) Customer:{} - {}'.format(module_class_name(self), + method, url, self.logger.customer, + repr(exc))) + raise exc + + await asyncio.sleep(sleepsecs) + n += 1 + + else: + if getattr(self.logger, 'job', False): + self.logger.info("{} Customer:{} Job:{} : HTTP Connection retry exhausted".format(module_class_name(self), + self.logger.customer, self.logger.job)) + else: + self.logger.info("{} Customer:{} : HTTP Connection retry exhausted".format(module_class_name(self), + self.logger.customer)) + raise raised_exc + + except Exception as exc: + if getattr(self.logger, 'job', False): + self.logger.error('{}.http_{}({}) Customer:{} Job:{} - {}'.format(module_class_name(self), + method, url, self.logger.customer, + self.logger.job, repr(exc))) + else: + self.logger.error('{}.http_{}({}) Customer:{} - {}'.format(module_class_name(self), + method, url, self.logger.customer, + repr(exc))) + raise exc + + finally: + if not self.handle_session_close: + await self.session.close() + + async def http_get(self, url, headers=None): + try: + content = await self._http_method('get', url, headers=headers) + return content + + except Exception as exc: + raise ConnectorHttpError(repr(exc)) from exc + + async def http_put(self, url, data, headers=None): + try: + content = await self._http_method('put', url, data=data, + headers=headers) + return content + + except Exception as exc: + raise ConnectorHttpError(repr(exc)) from exc + + async def http_post(self, url, data, headers=None): + try: + content = await self._http_method('post', url, data=data, + headers=headers) + return content + + except Exception as exc: + raise ConnectorHttpError(repr(exc)) from exc + + async def http_delete(self, url, headers=None): + try: + content = await self._http_method('delete', url, headers=headers) + return content + + except Exception as exc: + raise ConnectorHttpError(repr(exc)) from exc + + + async def close(self): + return await self.session.close() diff --git a/modules/io/ldap.py b/modules/io/ldap.py new file mode 100644 index 00000000..ca7bf092 --- /dev/null +++ b/modules/io/ldap.py @@ -0,0 +1,46 @@ +import asyncio + +import bonsai + +from argo_egi_connectors.utils import module_class_name +from argo_egi_connectors.exceptions import ConnectorHttpError + + +class LDAPSessionWithRetry(object): + def __init__(self, logger, retry_attempts, retry_sleep, connection_timeout): + self.n_try = retry_attempts + self.retry_sleep_list = [(i + 1) * retry_sleep for i in range(retry_attempts)] + self.timeout = connection_timeout + self.logger = logger + + + async def search(self, host, port, base, filter, attributes): + raised_exc = None + n = 1 + + try: + client = bonsai.LDAPClient('ldap://' + host + ':' + port + '/') + while n <= self.n_try: + try: + conn = await client.connect(True, timeout=float(self.timeout)) + res = await conn.search(base, + bonsai.LDAPSearchScope.SUB, filter, attributes, + timeout=float(self.timeout)) + + return res + + + except Exception as exc: + self.logger.error('from {}.search() - {}'.format(module_class_name(self), repr(exc))) + await asyncio.sleep(float(self.retry_sleep_list[n - 1])) + raised_exc = exc + + self.logger.info(f'LDAP Connection try - {n}') + n += 1 + + else: + self.logger.error('LDAP Connection retry exhausted') + + except Exception as exc: + self.logger.error('from {}.search() - {}'.format(module_class_name(self), repr(exc))) + raise ConnectorHttpError() diff --git a/modules/io/statewrite.py b/modules/io/statewrite.py new file mode 100644 index 00000000..4ee799d1 --- /dev/null +++ b/modules/io/statewrite.py @@ -0,0 +1,40 @@ +import datetime +import os +import aiofiles + +from argo_egi_connectors.utils import datestamp + + +daysback = 1 + + +async def state_write(caller, statedir, state, savedays, date=None): + filenamenew = '' + if 'topology' in caller: + filenamebase = 'topology-ok' + elif 'metricprofile' in caller: + filenamebase = 'metricprofile-ok' + elif 'weights' in caller: + filenamebase = 'weights-ok' + elif 'downtimes' in caller: + filenamebase = 'downtimes-ok' + + if date: + datebackstamp = date + else: + datebackstamp = datestamp(daysback) + + filenamenew = filenamebase + '_' + datebackstamp + db = datetime.datetime.strptime(datebackstamp, '%Y_%m_%d') + + datestart = db - datetime.timedelta(days=int(savedays)) + i = 0 + while i < int(savedays) * 2: + d = datestart - datetime.timedelta(days=i) + filenameold = filenamebase + '_' + d.strftime('%Y_%m_%d') + if os.path.exists(statedir + '/' + filenameold): + os.remove(statedir + '/' + filenameold) + i += 1 + + async with aiofiles.open(statedir + '/' + filenamenew, mode='w') as fp: + await fp.write(str(state)) diff --git a/modules/io/webapi.py b/modules/io/webapi.py new file mode 100644 index 00000000..a8b78ac0 --- /dev/null +++ b/modules/io/webapi.py @@ -0,0 +1,189 @@ +import datetime +import os +import json + +from argo_egi_connectors.utils import module_class_name +from argo_egi_connectors.io.http import SessionWithRetry + + +class WebAPI(object): + methods = { + 'downtimes-gocdb-connector.py': 'downtimes', + 'topology-gocdb-connector.py': 'topology', + 'topology-csv-connector.py': 'topology', + 'topology-json-connector.py': 'topology', + 'weights-vapor-connector.py': 'weights' + } + + def __init__(self, connector, host, token, logger, retry, + timeout=180, sleepretry=60, report=None, endpoints_group=None, + date=None): + self.connector = os.path.basename(connector) + self.webapi_method = self.methods[self.connector] + self.host = host + self.token = token + self.headers = { + 'x-api-key': self.token, + 'Accept': 'application/json' + } + self.report = report + self.logger = logger + self.retry = retry + self.timeout = timeout + self.sleepretry = sleepretry + self.retry_options = { + 'ConnectionRetry'.lower(): retry, + 'ConnectionTimeout'.lower(): timeout, + 'ConnectionSleepRetry'.lower(): sleepretry + } + self.endpoints_group = endpoints_group + self.date = date or self._construct_datenow() + self.session = SessionWithRetry(self.logger, module_class_name(self), + self.retry_options, verbose_ret=True, + handle_session_close=True) + + def _construct_datenow(self): + d = datetime.datetime.now() + + return d.strftime('%Y-%m-%d') + + def _format_downtimes(self, data): + formatted = dict() + + formatted['endpoints'] = data + + return formatted + + def _format_weights(self, data): + formatted = dict() + + if data: + formatted['weight_type'] = data[0]['type'] + groups = map(lambda s: {'name': s['site'], 'value': float(s['weight'])}, data) + else: + formatted['weight_type'] = '' + groups = [] + formatted['name'] = self.report + formatted['groups'] = list(groups) + formatted['name'] = self.report + formatted['group_type'] = self.endpoints_group + + return formatted + + async def _send(self, api, data_send, connector): + content, headers, status = await self.session.http_post(api, + data=json.dumps(data_send), + headers=self.headers) + if status != 201: + if connector.startswith('topology') or connector.startswith('downtimes'): + jsonret = json.loads(content) + msg = None + statusmsg = jsonret.get('status', False) + if statusmsg: + msg = jsonret['status']['message'] + else: + msg = jsonret['message'] + self.logger.error('%s %s() Customer:%s - HTTP POST %s' % (module_class_name(self), + '_send', + self.logger.customer, + msg)) + else: + errormsg = json.loads(content) + if 'errors' in errormsg: + errormsg = errormsg['errors'][0]['details'] + elif 'status' in errormsg: + errormsg = errormsg['status']['message'] + self.logger.error('%s %s() Customer:%s Job:%s - HTTP POST %s' % + (module_class_name(self), '_send', + self.logger.customer, self.logger.job, + errormsg)) + return status + + async def _get(self, api): + content, headers, status = await self.session.http_get(api, headers=self.headers) + return json.loads(content) + + async def _delete(self, api, id=None, date=None): + from urllib.parse import urlparse + loc = urlparse(api) + if id is not None: + loc = '{}://{}{}/{}'.format(loc.scheme, loc.hostname, loc.path, id) + else: + loc = '{}://{}{}'.format(loc.scheme, loc.hostname, loc.path) + if date is not None: + loc = '{}?date={}'.format(loc, date) + content, headers, status = await self.session.http_delete(loc, headers=self.headers) + return status + + async def _put(self, api, data_send, id): + from urllib.parse import urlparse + loc = urlparse(api) + loc = '{}://{}{}/{}?{}'.format(loc.scheme, loc.hostname, loc.path, id, loc.query) + content, headers, status = await self.session.http_put(loc, + data=json.dumps(data_send), + headers=self.headers) + return content, status + + async def _update(self, api, data_send): + content = await self._get(api) + target = list(filter(lambda w: w['name'] == data_send['name'], content['data'])) + if len(target) > 1: + self.logger.error('%s %s() Customer:%s Job:%s - HTTP PUT %s' % + (module_class_name(self), '_update', + self.logger.customer, self.logger.job, + 'Name of resource not unique on WEB-API, cannot proceed with update')) + else: + id = target[0]['id'] + content, status = await self._put(api, data_send, id) + if status == 200: + self.logger.info('Succesfully updated (HTTP PUT) resource') + else: + self.logger.error('%s %s() Customer:%s Job:%s - HTTP PUT %s' % + (module_class_name(self), '_update', + self.logger.customer, self.logger.job, + content)) + + async def _delete_and_resend(self, api, data_send, topo_component, downtimes_component): + id = None + content = await self._get(api) + if not topo_component and not downtimes_component: + id = content['data'][0]['id'] + status = await self._delete(api, id, self.date) + if status == 200: + await self._send(api, data_send, self.connector) + self.logger.info('Succesfully deleted and created new resource') + + async def send(self, data, topo_component=None, downtimes_component=None): + if topo_component: + # /topology/groups, /topology/endpoints + webapi_url = '{}/{}'.format(self.webapi_method, topo_component) + else: + webapi_url = self.webapi_method + + if self.date: + api = 'https://{}/api/v2/{}?date={}'.format(self.host, + webapi_url, + self.date) + else: + api = 'https://{}/api/v2/{}'.format(self.host, webapi_url) + + if topo_component: + data_send = data + else: + data_send = dict() + + if self.connector.startswith('downtimes'): + data_send = self._format_downtimes(data) + + if self.connector.startswith('weights'): + data_send = self._format_weights(data) + + status = await self._send(api, data_send, self.connector) + + # delete resource on WEB-API and resend + if status == 409 and topo_component or downtimes_component: + await self._delete_and_resend(api, data_send, topo_component, downtimes_component) + elif status == 409: + await self._update(api, data_send) + + await self.session.close() diff --git a/modules/log.py b/modules/log.py index 6214fadc..119ef899 100644 --- a/modules/log.py +++ b/modules/log.py @@ -1,37 +1,56 @@ -import logging, logging.handlers +import logging +import logging.handlers import sys import socket -logfile = "/var/log/argo-connectors/connectors.log" +LOGFILE = "/var/log/argo-connectors/connectors.log" + class Logger: def __init__(self, connector): lfs = '%(name)s[%(process)s]: %(levelname)s %(message)s' - lf = logging.Formatter(lfs) - lv = logging.INFO + logformat = logging.Formatter(lfs) + logverbose = logging.INFO + self.connector = connector logging.basicConfig(format=lfs, level=logging.INFO, stream=sys.stdout) self.logger = logging.getLogger(connector) try: - sh = logging.handlers.SysLogHandler('/dev/log', logging.handlers.SysLogHandler.LOG_USER) - except socket.error as e: - sh = logging.StreamHandler() - sh.setFormatter(lf) - sh.setLevel(lv) - self.logger.addHandler(sh) + sysloghandle = logging.handlers.SysLogHandler('/dev/log', logging.handlers.SysLogHandler.LOG_USER) + except socket.error: + sysloghandle = logging.StreamHandler() + sysloghandle.setFormatter(logformat) + sysloghandle.setLevel(logverbose) + self.logger.addHandler(sysloghandle) try: lffs = '%(asctime)s %(name)s[%(process)s]: %(levelname)s %(message)s' lff = logging.Formatter(lffs) - fh = logging.handlers.RotatingFileHandler(logfile, maxBytes=512*1024, backupCount=5) - fh.setFormatter(lff) - fh.setLevel(lv) - self.logger.addHandler(fh) + filehandle = logging.handlers.RotatingFileHandler(LOGFILE, maxBytes=512*1024, backupCount=5) + filehandle.setFormatter(lff) + filehandle.setLevel(logverbose) + self.logger.addHandler(filehandle) except Exception: pass - for func in ['warn', 'error', 'critical', 'info']: - code = """def %s(self, msg): - self.logger.%s(msg)""" % (func, func) - exec code + def __getstate__(self): + d = dict(self.__dict__) + del d['logger'] + return d + + def __setstate__(self, d): + self.__dict__.update(d) + self.logger = logging.getLogger(self.connector) + + def warn(self, msg): + self.logger.warning(msg) + + def critical(self, msg): + self.logger.critical(msg) + + def error(self, msg): + self.logger.error(msg) + + def info(self, msg): + self.logger.info(msg) diff --git a/modules/mesh/__init__.py b/modules/mesh/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modules/mesh/contacts.py b/modules/mesh/contacts.py new file mode 100644 index 00000000..29bab570 --- /dev/null +++ b/modules/mesh/contacts.py @@ -0,0 +1,92 @@ +def filter_dups_noemails(contact_list): + # does not preserve order needed for test + # no_dups = list(orderedset(contact_list)) + only_emails = list() + no_dups = list() + visited = set() + for contact in contact_list: + if contact in visited: + continue + else: + no_dups.append(contact) + visited.add(contact) + + for contact in no_dups: + if '@' not in contact: + continue + + if ',' in contact: + only_emails = only_emails + contact.split(',') + elif ';' in contact: + only_emails = only_emails + contact.split(';') + else: + only_emails.append(contact) + + return only_emails + + +def attach_contacts_topodata(logger, contacts, topodata): + updated_topodata = list() + found_contacts = None + + if len(contacts) == 0: + return topodata + + try: + for entity in topodata: + # group_groups topotype + if 'subgroup' in entity: + found_contacts = list( + filter(lambda contact: + contact['name'] == entity['subgroup'], + contacts) + ) + if found_contacts: + emails = list() + for contact in found_contacts[0]['contacts']: + if isinstance(contact, str): + filtered_emails = filter_dups_noemails(found_contacts[0]['contacts']) + if filtered_emails: + entity.update(notifications={ + 'contacts': filtered_emails, + 'enabled': True + }) + break + else: + emails.append(contact['email']) + if emails: + filtered_emails = filter_dups_noemails(emails) + if filtered_emails: + entity.update(notifications={ + 'contacts': filtered_emails, + 'enabled': True + }) + + # group_endpoints topotype + else: + for contact in contacts: + fqdn, servtype = contact['name'].split('+') + emails = filter_dups_noemails(contact['contacts']) + found_endpoints = list( + filter(lambda endpoint: + endpoint['hostname'] == fqdn \ + and endpoint['service'] == servtype, + topodata) + ) + if emails: + for endpoint in found_endpoints: + endpoint.update(notifications={ + 'contacts': emails, + 'enabled': True + }) + + updated_topodata.append(entity) + + except (KeyError, ValueError, TypeError) as exc: + logger.warn('Error joining contacts and topology data: %s' % repr(exc)) + if entity: + logger.warn('Topology entity: %s' % entity) + if found_contacts: + logger.warn('Found contacts: %s' % found_contacts) + + return updated_topodata diff --git a/modules/mesh/srm_port.py b/modules/mesh/srm_port.py new file mode 100644 index 00000000..2e033325 --- /dev/null +++ b/modules/mesh/srm_port.py @@ -0,0 +1,29 @@ +def load_srm_port_map(logger, ldap_data, attribute_name): + """ + Returnes a dictionary which maps hostnames to their respective ldap port if such exists + """ + port_dict = {} + for res in ldap_data: + try: + attribute = res[attribute_name][0] + start_index = attribute.index('//') + colon_index = attribute.index(':', start_index) + end_index = attribute.index('/', colon_index) + fqdn = attribute[start_index + 2:colon_index] + port = attribute[colon_index + 1:end_index] + + port_dict[fqdn] = port + + except ValueError: + logger.error('Exception happened while retrieving port from: %s' % res) + + return port_dict + +def attach_srmport_topodata(logger, attributes, topodata, group_endpoints): + """ + Get SRM ports from LDAP and put them under tags -> info_srm_port + """ + srm_port_map = load_srm_port_map(logger, topodata, attributes) + for endpoint in group_endpoints: + if endpoint['service'] == 'SRM' and srm_port_map.get(endpoint['hostname'], False): + endpoint['tags']['info_bdii_SRM2_PORT'] = srm_port_map[endpoint['hostname']] diff --git a/modules/mesh/storage_element_path.py b/modules/mesh/storage_element_path.py new file mode 100644 index 00000000..efd48558 --- /dev/null +++ b/modules/mesh/storage_element_path.py @@ -0,0 +1,90 @@ +def extract_value(key, entry): + if isinstance(entry, tuple): + for e in entry: + k, v = e[0] + if key == k: + return v + else: + return entry.get(key, None) + + +def update_map_entry(endpoint, mapping, sepath, voname): + if endpoint not in mapping: + mapping[endpoint] = list() + mapping[endpoint].append({ + 'voname': voname, + 'GlueVOInfoPath': sepath + }) + else: + mapping[endpoint].append({ + 'voname': voname, + 'GlueVOInfoPath': sepath + }) + + +def ispath_already_added(endpoint, mapping, sepath): + target = None + + if endpoint in mapping: + target = mapping[endpoint] + sepaths = set() + + for entry in target: + sepaths.add(entry['GlueVOInfoPath']) + + return sepath in sepaths + + else: + return False + + +def build_map_endpoint_path(logger, bdiidata): + mapping = dict() + + try: + for entry in bdiidata: + voname = extract_value('GlueVOInfoAccessControlBaseRule', entry) + if isinstance(voname, list): + voname = list(filter(lambda e: 'VO:' in e, voname)) + if voname: + voname = voname[0].split(':')[1] + else: + continue + + sepath = extract_value('GlueVOInfoPath', entry) + sepath = sepath[0] if isinstance(sepath, list) else None + endpoint = extract_value('GlueSEUniqueID', entry['dn'].rdns) + + if (voname and sepath and endpoint + and not ispath_already_added(endpoint, mapping, sepath) + and ' ' not in voname): + update_map_entry(endpoint, mapping, sepath, voname) + + elif (voname and sepath and endpoint + and not ispath_already_added(endpoint, mapping, sepath) + and ' ' in voname): + vonames = voname.split(' ') + for vo in vonames: + update_map_entry(endpoint, mapping, sepath, vo) + + except IndexError as exc: + logger.error('Error building map of endpoints and storage paths from BDII data: %s' % repr(exc)) + logger.error('LDAP entry: %s' % entry) + + return mapping + + +def attach_sepath_topodata(logger, bdii_opts, bdiidata, group_endpoints): + """ + Get SRM ports from LDAP and put them under tags -> info_srm_port + """ + endpoint_sepaths = build_map_endpoint_path(logger, bdiidata) + + for endpoint in group_endpoints: + if endpoint['hostname'] in endpoint_sepaths: + for paths in endpoint_sepaths[endpoint['hostname']]: + voname = paths['voname'] + sepath = paths['GlueVOInfoPath'] + endpoint['tags'].update({ + 'vo_{}_attr_SE_PATH'.format(voname): sepath + }) diff --git a/modules/output.py b/modules/output.py deleted file mode 100644 index 787cb88b..00000000 --- a/modules/output.py +++ /dev/null @@ -1,176 +0,0 @@ -import datetime -import os -import json - -import avro.schema -from avro.datafile import DataFileWriter -from avro.io import DatumWriter, BinaryEncoder - -from io import BytesIO - -from argo_egi_connectors.helpers import datestamp, retry, module_class_name -from argo_egi_connectors.log import Logger - -from argo_ams_library import AmsMessage, ArgoMessagingService, AmsException - - -daysback = 1 - - -class AvroWriter: - """ AvroWriter """ - def __init__(self, schema, outfile): - self.schema = schema - self.outfile = outfile - self.datawrite = None - self.avrofile = None - self._load_datawriter() - - def _load_datawriter(self): - try: - lschema = load_schema(self.schema) - self.avrofile = open(self.outfile, 'w+') - self.datawrite = DataFileWriter(self.avrofile, DatumWriter(), lschema) - except Exception: - return False - - return True - - def write(self, data): - try: - - if (not self.datawrite or - not self.avrofile): - raise ('AvroFileWriter not initalized') - - for elem in data: - self.datawrite.append(elem) - - self.datawrite.close() - self.avrofile.close() - - except Exception as e: - return False, e - - return True, None - - -class AmsPublish(object): - """ - Class represents interaction with AMS service - """ - def __init__(self, host, project, token, topic, report, bulk, packsingle, - logger, retry, timeout=180, sleepretry=60): - self.ams = ArgoMessagingService(host, token, project) - self.topic = topic - self.bulk = int(bulk) - self.report = report - self.timeout = int(timeout) - self.retry = int(retry) - self.sleepretry = int(sleepretry) - self.logger = logger - self.packsingle = eval(packsingle) - - @staticmethod - @retry - def _send(logger, msgprefix, retryopts, msgs, bulk, obj): - timeout = retryopts['ConnectionTimeout'.lower()] - try: - if bulk > 1: - q, r = divmod(len(msgs), bulk) - - if q: - s = 0 - e = bulk - 1 - - for i in range(q): - obj.ams.publish(obj.topic, msgs[s:e], timeout=timeout) - s += bulk - e += bulk - obj.ams.publish(obj.topic, msgs[s:], timeout=timeout) - - else: - obj.ams.publish(obj.topic, msgs, timeout=timeout) - - else: - obj.ams.publish(obj.topic, msgs, timeout=timeout) - - except AmsException as e: - raise e - - return True - - def send(self, schema, msgtype, date, msglist): - def _avro_serialize(msg): - opened_schema = load_schema(schema) - avro_writer = DatumWriter(opened_schema) - bytesio = BytesIO() - encoder = BinaryEncoder(bytesio) - if isinstance(msg, list): - for m in msg: - avro_writer.write(m, encoder) - else: - avro_writer.write(msg, encoder) - - return bytesio.getvalue() - - if self.packsingle: - self.bulk = 1 - msg = AmsMessage(attributes={'partition_date': date, - 'report': self.report, - 'type': msgtype}, - data=_avro_serialize(msglist)) - msgs = [msg] - - else: - msgs = map(lambda m: AmsMessage(attributes={'partition_date': date, - 'report': self.report, - 'type': msgtype}, - data=_avro_serialize(m)), msglist) - - if self._send(self.logger, module_class_name(self), - {'ConnectionRetry'.lower(): self.retry, - 'ConnectionTimeout'.lower(): self.timeout, - 'ConnectionSleepRetry'.lower(): self.sleepretry}, msgs, self.bulk, self): - return True - - -def load_schema(schema): - try: - f = open(schema) - schema = avro.schema.parse(f.read()) - return schema - except Exception as e: - raise e - - -def write_state(caller, statedir, state, savedays, date=None): - filenamenew = '' - if 'topology' in caller: - filenamebase = 'topology-ok' - elif 'metricprofile' in caller: - filenamebase = 'metricprofile-ok' - elif 'weights' in caller: - filenamebase = 'weights-ok' - elif 'downtimes' in caller: - filenamebase = 'downtimes-ok' - - if date: - datebackstamp = date - else: - datebackstamp = datestamp(daysback) - - filenamenew = filenamebase + '_' + datebackstamp - db = datetime.datetime.strptime(datebackstamp, '%Y_%m_%d') - - datestart = db - datetime.timedelta(days=int(savedays)) - i = 0 - while i < int(savedays)*2: - d = datestart - datetime.timedelta(days=i) - filenameold = filenamebase + '_' + d.strftime('%Y_%m_%d') - if os.path.exists(statedir + '/' + filenameold): - os.remove(statedir + '/' + filenameold) - i += 1 - - with open(statedir + '/' + filenamenew, 'w') as fp: - fp.write(str(state)) diff --git a/modules/parse/__init__.py b/modules/parse/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modules/parse/base.py b/modules/parse/base.py new file mode 100644 index 00000000..aba23951 --- /dev/null +++ b/modules/parse/base.py @@ -0,0 +1,131 @@ +import xml.dom.minidom + +from xml.parsers.expat import ExpatError +from io import StringIO + +from argo_egi_connectors.utils import module_class_name +from argo_egi_connectors.exceptions import ConnectorParseError + +import json +import csv + + +class ParseHelpers(object): + def __init__(self, logger, *args, **kwargs): + self.logger = logger + + def parse_xmltext(self, nodelist): + rc = [] + for node in nodelist: + if node.nodeType == node.TEXT_NODE: + rc.append(node.data) + return ''.join(rc) + + def parse_extensions(self, extensions_node): + extensions_dict = dict() + + for extension in extensions_node: + if extension.nodeName == 'EXTENSION': + key, value = None, None + for ext_node in extension.childNodes: + if ext_node.nodeName == 'KEY': + key = ext_node.childNodes[0].nodeValue + if ext_node.nodeName == 'VALUE': + value = ext_node.childNodes[0].nodeValue + if key and value: + extensions_dict.update({key: value}) + + return extensions_dict + + def parse_url_endpoints(self, endpoints_node): + endpoints_urls = list() + + for endpoint in endpoints_node: + if endpoint.nodeName == 'ENDPOINT': + url = None + for endpoint_node in endpoint.childNodes: + if endpoint_node.nodeName == 'ENDPOINT_MONITORED': + value = endpoint_node.childNodes[0].nodeValue + if value.lower() == 'y': + for url_node in endpoint.childNodes: + if url_node.nodeName == 'URL' and url_node.childNodes: + url = url_node.childNodes[0].nodeValue + endpoints_urls.append(url) + + if endpoints_urls: + return ', '.join(endpoints_urls) + else: + return None + + def parse_scopes(self, xml_node): + scopes = list() + + for elem in xml_node.childNodes: + if elem.nodeName == 'SCOPES': + for subelem in elem.childNodes: + if subelem.nodeName == 'SCOPE': + scopes.append(subelem.childNodes[0].nodeValue) + + return scopes + + def parse_json(self, data): + try: + if data is None: + if getattr(self.logger, 'job', False): + raise ConnectorParseError("{} Customer:{} Job:{} : No JSON data fetched".format(module_class_name(self), self.logger.customer, self.logger.job)) + else: + raise ConnectorParseError("{} Customer:{} : No JSON data fetched".format(module_class_name(self), self.logger.customer)) + + return json.loads(data) + + except ValueError as exc: + msg = '{} Customer:{} : Error parsing JSON feed - {}'.format(module_class_name(self), self.logger.customer, repr(exc)) + raise ConnectorParseError(msg) + + except Exception as exc: + msg = '{} Customer:{} : Error - {}'.format(module_class_name(self), self.logger.customer, repr(exc)) + raise ConnectorParseError(msg) + + def parse_xml(self, data): + try: + if data is None: + if getattr(self.logger, 'job', False): + raise ConnectorParseError("{} Customer:{} Job:{} : No XML data fetched".format(module_class_name(self), self.logger.customer, self.logger.job)) + else: + raise ConnectorParseError("{} Customer:{} : No XML data fetched".format(module_class_name(self), self.logger.customer)) + + + return xml.dom.minidom.parseString(data) + + except ExpatError as exc: + msg = '{} Customer:{} : Error parsing XML feed - {}'.format(module_class_name(self), self.logger.customer, repr(exc)) + raise ConnectorParseError(msg) + + except Exception as exc: + msg = '{} Customer:{} : Error - {}'.format(module_class_name(self), self.logger.customer, repr(exc)) + raise ConnectorParseError(msg) + + def csv_to_json(self, data): + data = StringIO(data) + reader = csv.reader(data, delimiter=',') + + num_row = 0 + results = [] + header = [] + for row in reader: + if num_row == 0: + header = row + num_row = num_row + 1 + continue + num_item = 0 + datum = {} + for item in header: + datum[item] = row[num_item] + num_item = num_item + 1 + results.append(datum) + + if not results: + msg = '{} Customer:{} : Error parsing CSV feed - empty data'.format(module_class_name(self), self.logger.customer) + raise ConnectorParseError(msg) + return results + diff --git a/modules/parse/flat_topology.py b/modules/parse/flat_topology.py new file mode 100644 index 00000000..8e9c0b63 --- /dev/null +++ b/modules/parse/flat_topology.py @@ -0,0 +1,111 @@ +from urllib.parse import urlparse +from argo_egi_connectors.utils import filename_date, module_class_name +from argo_egi_connectors.exceptions import ConnectorParseError +from argo_egi_connectors.parse.base import ParseHelpers + + +def construct_fqdn(http_endpoint): + return urlparse(http_endpoint).netloc + + +class ParseContacts(ParseHelpers): + def __init__(self, logger, data, uidservtype=False, is_csv=False): + self.logger = logger + self.uidservtype = uidservtype + if is_csv: + self.data = self.csv_to_json(data) + else: + self.data = self.parse_json(data) + + def get_contacts(self): + contacts = list() + + for entity in self.data: + if self.uidservtype: + key = '{}_{}+{}'.format(construct_fqdn(entity['URL']), entity['Service Unique ID'], entity['SERVICE_TYPE']) + else: + key = '{}+{}'.format(construct_fqdn(entity['URL']), entity['SERVICE_TYPE']) + + value = entity['CONTACT_EMAIL'] + contacts.append({ + 'name': key, + 'contacts': [value] if not type(value) == list else value + }) + + return contacts + + +class ParseFlatEndpoints(ParseHelpers): + def __init__(self, logger, data, project, uidservtype=False, + fetchtype='ServiceGroups', is_csv=False, scope=None): + self.uidservtype = uidservtype + self.fetchtype = fetchtype + self.logger = logger + self.project = project + self.is_csv = is_csv + self.scope = scope if scope else project + try: + if is_csv: + self.data = self.csv_to_json(data) + else: + self.data = self.parse_json(data) + + except ConnectorParseError as exc: + raise exc + + def get_groupgroups(self): + try: + groups = list() + already_added = list() + + for entity in self.data: + tmp_dict = dict() + + tmp_dict['type'] = 'PROJECT' + tmp_dict['group'] = self.project + tmp_dict['subgroup'] = entity['SITENAME-SERVICEGROUP'] + tmp_dict['tags'] = {'monitored': '1', 'scope': self.scope} + + if tmp_dict['subgroup'] in already_added: + continue + else: + groups.append(tmp_dict) + already_added.append(tmp_dict['subgroup']) + + return groups + + except (KeyError, IndexError, TypeError, AttributeError, AssertionError) as exc: + feedtype = 'CSV' if self.is_csv else 'JSON' + msg = 'Customer:%s : Error parsing %s feed - %s' % (self.logger.customer, feedtype, repr(exc).replace('\'', '').replace('\"', '')) + raise ConnectorParseError(msg) + + def get_groupendpoints(self): + try: + groups = list() + + for entity in self.data: + tmp_dict = dict() + + tmp_dict['type'] = self.fetchtype.upper() + tmp_dict['group'] = entity['SITENAME-SERVICEGROUP'] + tmp_dict['service'] = entity['SERVICE_TYPE'] + info_url = entity['URL'] + if self.uidservtype: + tmp_dict['hostname'] = '{1}_{0}'.format(entity['Service Unique ID'], construct_fqdn(info_url)) + else: + tmp_dict['hostname'] = construct_fqdn(entity['URL']) + + tmp_dict['tags'] = {'scope': self.project, + 'monitored': '1', + 'info_URL': info_url, + 'hostname': construct_fqdn(entity['URL'])} + + tmp_dict['tags'].update({'info_ID': str(entity['Service Unique ID'])}) + groups.append(tmp_dict) + + return groups + + except (KeyError, IndexError, TypeError, AttributeError, AssertionError) as exc: + feedtype = 'CSV' if self.is_csv else 'JSON' + msg = 'Customer:%s : Error parsing %s feed - %s' % (self.logger.customer, feedtype, repr(exc).replace('\'', '').replace('\"', '')) + raise ConnectorParseError(msg) diff --git a/modules/parse/gocdb_contacts.py b/modules/parse/gocdb_contacts.py new file mode 100644 index 00000000..dc2f3c0d --- /dev/null +++ b/modules/parse/gocdb_contacts.py @@ -0,0 +1,267 @@ +import xml.dom.minidom +from xml.parsers.expat import ExpatError + +from argo_egi_connectors.parse.base import ParseHelpers +from argo_egi_connectors.parse.gocdb_topology import ParseServiceEndpoints +from argo_egi_connectors.utils import filename_date, module_class_name +from argo_egi_connectors.exceptions import ConnectorParseError + + +class ParseContacts(ParseHelpers): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def _parse_contact(self, contacts_node, *attrs): + values = list() + for xml_attr in attrs: + value = contacts_node.getElementsByTagName(xml_attr) + if value and value[0].childNodes: + values.append(value[0].childNodes[0].nodeValue) + else: + values.append('') + return values + + def parse_contacts(self, data, root_node, child_node, topo_node): + interested = ('EMAIL', 'FORENAME', 'SURNAME', 'CERTDN', 'ROLE_NAME') + + try: + data = list() + xml_data = self.parse_xml(self.data) + entities = xml_data.getElementsByTagName(root_node) + for entity in entities: + if entity.nodeName == root_node: + emails = list() + for entity_node in entity.childNodes: + if entity_node.nodeName == child_node: + contact = entity_node + email, name, surname, certdn, role = self._parse_contact(contact, *interested) + emails.append({ + 'email': email, + 'forename': name, + 'surname': surname, + 'certdn': certdn, + 'role': role + }) + if entity_node.nodeName == topo_node: + entity_name = entity_node.childNodes[0].nodeValue + data.append({ + 'name': entity_name, + 'contacts': emails + }) + + return data + + except (KeyError, IndexError, TypeError, AttributeError, AssertionError) as exc: + self.logger.error(module_class_name(self) + ' Customer:%s : Error parsing - %s' % (self.logger.customer, repr(exc).replace('\'', '').replace('\"', ''))) + raise exc + + def parse_sites_with_contacts(self, data): + try: + sites_contacts = list() + xml_data = self.parse_xml(data) + elements = xml_data.getElementsByTagName('SITE') + for element in elements: + sitename, contact = None, None + for child in element.childNodes: + if child.nodeName == 'CONTACT_EMAIL' and child.childNodes: + contact = child.childNodes[0].nodeValue + if child.nodeName == 'SHORT_NAME' and child.childNodes: + sitename = child.childNodes[0].nodeValue + if contact: + if ';' in contact: + lcontacts = list() + for single_contact in contact.split(';'): + lcontacts.append(single_contact) + sites_contacts.append({ + 'name': sitename, + 'contacts': lcontacts + }) + else: + sites_contacts.append({ + 'name': sitename, + 'contacts': [contact] + }) + return sites_contacts + + except (KeyError, IndexError, TypeError, AttributeError, AssertionError) as exc: + self.logger.error(module_class_name(self) + ' Customer:%s : Error parsing - %s' % (self.logger.customer, repr(exc).replace('\'', '').replace('\"', ''))) + raise exc + + def parse_servicegroups_with_contacts(self, data): + return self.parse_servicegroup_contacts(data) + + def parse_servicegroup_contacts(self, data): + try: + endpoints_contacts = list() + xml_data = self.parse_xml(data) + elements = xml_data.getElementsByTagName('SERVICE_GROUP') + for element in elements: + name, contact = None, None + for child in element.childNodes: + if child.nodeName == 'NAME' and child.childNodes: + name = child.childNodes[0].nodeValue + if child.nodeName == 'CONTACT_EMAIL' and child.childNodes: + contact = child.childNodes[0].nodeValue + if contact and name: + endpoints_contacts.append({ + 'name': name, + 'contacts': [contact] + }) + return endpoints_contacts + + except (KeyError, IndexError, TypeError, AttributeError, AssertionError) as exc: + self.logger.error(module_class_name(self) + ' Customer:%s : Error parsing - %s' % (self.logger.customer, repr(exc).replace('\'', '').replace('\"', ''))) + raise exc + + def parse_serviceendpoint_contacts(self, data): + try: + endpoints_contacts = list() + xml_data = self.parse_xml(data) + elements = xml_data.getElementsByTagName('SERVICE_ENDPOINT') + for element in elements: + fqdn, contact, servtype = None, None, None + for child in element.childNodes: + if child.nodeName == 'HOSTNAME' and child.childNodes: + fqdn = child.childNodes[0].nodeValue + if child.nodeName == 'CONTACT_EMAIL' and child.childNodes: + contact = child.childNodes[0].nodeValue + if child.nodeName == 'SERVICE_TYPE' and child.childNodes: + servtype = child.childNodes[0].nodeValue + if contact: + if ';' in contact: + lcontacts = list() + for single_contact in contact.split(';'): + lcontacts.append(single_contact) + endpoints_contacts.append({ + 'name': '{}+{}'.format(fqdn, servtype), + 'contacts': lcontacts + }) + else: + endpoints_contacts.append({ + 'name': '{}+{}'.format(fqdn, servtype), + 'contacts': [contact] + }) + return endpoints_contacts + + except (KeyError, IndexError, TypeError, AttributeError, AssertionError) as exc: + self.logger.error(module_class_name(self) + ' Customer:%s : Error parsing - %s' % (self.logger.customer, repr(exc).replace('\'', '').replace('\"', ''))) + raise exc + + +class ParseRocContacts(ParseContacts): + def __init__(self, logger, data): + super().__init__(logger) + self.data = data + self.logger = logger + self.data = data + self._parse_data() + + def _parse_data(self): + try: + return self.parse_contacts(self.data, 'ROC', 'CONTACT', 'ROCNAME') + + except (KeyError, IndexError, TypeError, AttributeError, AssertionError, ExpatError) as exc: + raise ConnectorParseError + + def get_contacts(self): + return self._parse_data() + + +class ParseSiteContacts(ParseContacts): + def __init__(self, logger, data): + super().__init__(logger) + self.logger = logger + self.data = data + self._parse_data() + + def _parse_data(self): + try: + return self.parse_contacts(self.data, 'SITE', 'CONTACT', 'SHORT_NAME') + + except (KeyError, IndexError, TypeError, AttributeError, AssertionError, ExpatError) as exc: + raise ConnectorParseError + + def get_contacts(self): + return self._parse_data() + + +class ParseSitesWithContacts(ParseContacts): + def __init__(self, logger, data): + super().__init__(logger) + self.logger = logger + self.data = data + self._parse_data() + + def _parse_data(self): + try: + return self.parse_sites_with_contacts(self.data) + + except (KeyError, IndexError, TypeError, AttributeError, AssertionError, ExpatError) as exc: + raise ConnectorParseError + + def get_contacts(self): + return self._parse_data() + + +class ParseServiceEndpointContacts(ParseContacts): + def __init__(self, logger, data): + super().__init__(logger) + self.logger = logger + self.data = data + + def _parse_data(self): + try: + return self.parse_serviceendpoint_contacts(self.data) + + except (KeyError, IndexError, TypeError, AttributeError, AssertionError, ExpatError) as exc: + raise ConnectorParseError + + def get_contacts(self): + return self._parse_data() + + +class ParseProjectContacts(object): + def __init__(self, logger, data): + super().__init__(logger) + self.data = data + + def _parse_data(self): + pass + + def contacts(self): + pass + + +class ParseServiceGroupWithContacts(ParseContacts): + def __init__(self, logger, data): + super().__init__(logger) + self.data = data + self.logger = logger + + def _parse_data(self): + try: + return self.parse_servicegroups_with_contacts(self.data) + + except (KeyError, IndexError, TypeError, AttributeError, AssertionError, ExpatError) as exc: + raise ConnectorParseError + + + def get_contacts(self): + return self._parse_data() + + +class ParseServiceGroupRoles(ParseContacts): + def __init__(self, logger, data): + super().__init__(logger) + self.data = data + self.logger = logger + + def _parse_data(self): + try: + return self.parse_servicegroup_contacts(self.data) + + except (KeyError, IndexError, TypeError, AttributeError, AssertionError, ExpatError) as exc: + raise ConnectorParseError + + def get_contacts(self): + return self._parse_data() diff --git a/modules/parse/gocdb_downtimes.py b/modules/parse/gocdb_downtimes.py new file mode 100644 index 00000000..55722147 --- /dev/null +++ b/modules/parse/gocdb_downtimes.py @@ -0,0 +1,61 @@ +import datetime +import xml.dom.minidom + +from xml.parsers.expat import ExpatError +from argo_egi_connectors.utils import module_class_name +from argo_egi_connectors.exceptions import ConnectorParseError +from argo_egi_connectors.parse.base import ParseHelpers + + +class ParseDowntimes(ParseHelpers): + def __init__(self, logger, data, start, end, uid=False): + self.uid = uid + self.start = start + self.end = end + self.logger = logger + self.data = data + + def get_data(self): + filtered_downtimes = list() + + try: + downtimes = self.parse_xml(self.data).getElementsByTagName('DOWNTIME') + + for downtime in downtimes: + classification = downtime.getAttributeNode('CLASSIFICATION').nodeValue + hostname = self.parse_xmltext(downtime.getElementsByTagName('HOSTNAME')[0].childNodes) + service_type = self.parse_xmltext(downtime.getElementsByTagName('SERVICE_TYPE')[0].childNodes) + start_str = self.parse_xmltext(downtime.getElementsByTagName('FORMATED_START_DATE')[0].childNodes) + end_str = self.parse_xmltext(downtime.getElementsByTagName('FORMATED_END_DATE')[0].childNodes) + severity = self.parse_xmltext(downtime.getElementsByTagName('SEVERITY')[0].childNodes) + try: + service_id = self.parse_xmltext(downtime.getElementsByTagName('PRIMARY_KEY')[0].childNodes) + except IndexError: + service_id = downtime.getAttribute('PRIMARY_KEY') + start_time = datetime.datetime.strptime(start_str, "%Y-%m-%d %H:%M") + end_time = datetime.datetime.strptime(end_str, "%Y-%m-%d %H:%M") + + if start_time < self.start: + start_time = self.start + if end_time > self.end: + end_time = self.end + + if classification == 'SCHEDULED' and severity == 'OUTAGE': + downtime = dict() + if self.uid: + downtime['hostname'] = '{0}_{1}'.format(hostname, service_id) + else: + downtime['hostname'] = hostname + downtime['service'] = service_type + downtime['start_time'] = start_time.strftime('%Y-%m-%dT%H:%M:00Z') + downtime['end_time'] = end_time.strftime('%Y-%m-%dT%H:%M:00Z') + filtered_downtimes.append(downtime) + + return filtered_downtimes + + except (KeyError, IndexError, AttributeError, TypeError, AssertionError) as exc: + msg = '{} Customer:{} : Error parsing downtimes feed - {}'.format(module_class_name(self), self.logger.customer, repr(exc)) + raise ConnectorParseError(msg) + + except ConnectorParseError as exc: + raise exc diff --git a/modules/parse/gocdb_topology.py b/modules/parse/gocdb_topology.py new file mode 100644 index 00000000..db705eed --- /dev/null +++ b/modules/parse/gocdb_topology.py @@ -0,0 +1,267 @@ +from argo_egi_connectors.parse.base import ParseHelpers +from argo_egi_connectors.utils import module_class_name +from argo_egi_connectors.exceptions import ConnectorParseError + + +class ParseSites(ParseHelpers): + def __init__(self, logger, data, custname, uid=False, + pass_extensions=False): + super().__init__(logger) + self.logger = logger + self.data = data + self.uidservtype = uid + self.custname = custname + self.pass_extensions = pass_extensions + self._sites = dict() + self._parse_data() + + def _parse_data(self): + try: + xml_data = self.parse_xml(self.data) + sites = xml_data.getElementsByTagName('SITE') + for site in sites: + site_name = site.getAttribute('NAME') + if site_name not in self._sites: + self._sites[site_name] = {'site': site_name} + production_infra = site.getElementsByTagName('PRODUCTION_INFRASTRUCTURE') + if production_infra: + self._sites[site_name]['infrastructure'] = self.parse_xmltext(production_infra[0].childNodes) + certification_status = site.getElementsByTagName('CERTIFICATION_STATUS') + if certification_status: + self._sites[site_name]['certification'] = self.parse_xmltext(certification_status[0].childNodes) + try: + self._sites[site_name]['ngi'] = self.parse_xmltext(site.getElementsByTagName('ROC')[0].childNodes) + except IndexError: + self._sites[site_name]['ngi'] = site.getAttribute('ROC') + self._sites[site_name]['scope'] = ', '.join(self.parse_scopes(site)) + + # biomed feed does not have extensions + if self.pass_extensions: + try: + extensions = self.parse_extensions(site.getElementsByTagName('EXTENSIONS')[0].childNodes) + self._sites[site_name]['extensions'] = extensions + except IndexError: + pass + + except (KeyError, IndexError, TypeError, AttributeError, AssertionError) as exc: + msg = module_class_name(self) + ' Customer:%s : Error parsing sites feed - %s' % (self.logger.customer, repr(exc).replace('\'', '').replace('\"', '')) + raise ConnectorParseError(msg) + + except ConnectorParseError as exc: + raise exc + + def get_group_groups(self): + group_list, groupofgroups = list(), list() + group_list = group_list + sorted([value for _, value in self._sites.items()], key=lambda s: s['ngi']) + + for group in group_list: + tmpg = dict() + tmpg['type'] = 'NGI' + tmpg['group'] = group['ngi'] + tmpg['subgroup'] = group['site'] + tmpg['tags'] = {'certification': group.get('certification', ''), + 'scope': group.get('scope', ''), + 'infrastructure': group.get('infrastructure', '')} + + if self.pass_extensions and 'extensions' in group: + for key, value in group['extensions'].items(): + tmpg['tags'].update({ + 'info_ext_' + key: value + }) + + groupofgroups.append(tmpg) + + return groupofgroups + + +class ParseServiceEndpoints(ParseHelpers): + def __init__(self, logger, data=None, custname=None, uid=False, + pass_extensions=False): + super().__init__(logger) + self.data = data + self.__class__.fetched_data = data + self.uidservtype = uid + self.custname = custname + self.pass_extensions = pass_extensions + self._service_endpoints = dict() + self._parse_data() + self.maxDiff = None + + def _parse_data(self): + try: + xml_data = self.parse_xml(self.data) + services = xml_data.getElementsByTagName('SERVICE_ENDPOINT') + for service in services: + service_id = '' + if service.getAttributeNode('PRIMARY_KEY'): + service_id = str(service.attributes['PRIMARY_KEY'].value) + if service_id not in self._service_endpoints: + self._service_endpoints[service_id] = {} + self._service_endpoints[service_id]['hostname'] = self.parse_xmltext(service.getElementsByTagName('HOSTNAME')[0].childNodes) + self._service_endpoints[service_id]['type'] = self.parse_xmltext(service.getElementsByTagName('SERVICE_TYPE')[0].childNodes) + hostdn = service.getElementsByTagName('HOSTDN') + if hostdn: + self._service_endpoints[service_id]['hostdn'] = self.parse_xmltext(hostdn[0].childNodes) + self._service_endpoints[service_id]['monitored'] = self.parse_xmltext(service.getElementsByTagName('NODE_MONITORED')[0].childNodes) + self._service_endpoints[service_id]['production'] = self.parse_xmltext(service.getElementsByTagName('IN_PRODUCTION')[0].childNodes) + self._service_endpoints[service_id]['site'] = self.parse_xmltext(service.getElementsByTagName('SITENAME')[0].childNodes) + self._service_endpoints[service_id]['roc'] = self.parse_xmltext(service.getElementsByTagName('ROC_NAME')[0].childNodes) + self._service_endpoints[service_id]['service_id'] = service_id + self._service_endpoints[service_id]['scope'] = ', '.join(self.parse_scopes(service)) + self._service_endpoints[service_id]['sortId'] = self._service_endpoints[service_id]['hostname'] + '-' + self._service_endpoints[service_id]['type'] + '-' + self._service_endpoints[service_id]['site'] + self._service_endpoints[service_id]['url'] = self.parse_xmltext(service.getElementsByTagName('URL')[0].childNodes) + if self.pass_extensions: + extensions = self.parse_extensions(service.getElementsByTagName('EXTENSIONS')[0].childNodes) + self._service_endpoints[service_id]['extensions'] = extensions + self._service_endpoints[service_id]['endpoint_urls'] = self.parse_url_endpoints(service.getElementsByTagName('ENDPOINTS')[0].childNodes) + + except (KeyError, IndexError, TypeError, AttributeError, AssertionError) as exc: + msg = module_class_name(self) + ' Customer:%s : Error parsing topology service endpoint feed - %s' % (self.logger.customer, repr(exc).replace('\'', '').replace('\"', '')) + raise ConnectorParseError(msg) + + except ConnectorParseError as exc: + raise exc + + def get_group_endpoints(self): + group_list, groupofendpoints = list(), list() + group_list = group_list + sorted([value for _, value in self._service_endpoints.items()], key=lambda s: s['site']) + + for group in group_list: + tmpg = dict() + tmpg['type'] = 'SITES' + tmpg['group'] = group['site'] + tmpg['service'] = group['type'] + if self.uidservtype: + tmpg['hostname'] = '{1}_{0}'.format(group['service_id'], group['hostname']) + else: + tmpg['hostname'] = group['hostname'] + tmpg['tags'] = {'scope': group.get('scope', ''), + 'monitored': '1' if group['monitored'] == 'Y' or + group['monitored'] == 'True' else '0', + 'production': '1' if group['production'] == 'Y' or + group['production'] == 'True' else '0'} + tmpg['tags'].update({'info_ID': str(group['service_id'])}) + + if 'hostdn' in group: + tmpg['tags'].update({'info_HOSTDN': group['hostdn']}) + + if group['url']: + tmpg['tags'].update({ + 'info_URL': group['url'] + }) + if group['endpoint_urls']: + tmpg['tags'].update({ + 'info_service_endpoint_URL': group['endpoint_urls'] + }) + if self.pass_extensions: + for key, value in group['extensions'].items(): + tmpg['tags'].update({ + 'info_ext_' + key: value + }) + + groupofendpoints.append(tmpg) + + return groupofendpoints + + +class ParseServiceGroups(ParseHelpers): + def __init__(self, logger, data, custname, uid=False, + pass_extensions=False): + super().__init__(logger) + self.data = data + self.uidservtype = uid + self.custname = custname + self.pass_extensions = pass_extensions + # group_groups and group_endpoints components for ServiceGroup topology + self._service_groups = dict() + self._parse_data() + + def _parse_data(self): + try: + xml_data = self.parse_xml(self.data) + groups = xml_data.getElementsByTagName('SERVICE_GROUP') + for group in groups: + group_id = group.getAttribute('PRIMARY_KEY') + if group_id not in self._service_groups: + self._service_groups[group_id] = {} + self._service_groups[group_id]['name'] = self.parse_xmltext(group.getElementsByTagName('NAME')[0].childNodes) + self._service_groups[group_id]['monitored'] = self.parse_xmltext(group.getElementsByTagName('MONITORED')[0].childNodes) + + self._service_groups[group_id]['services'] = [] + services = group.getElementsByTagName('SERVICE_ENDPOINT') + self._service_groups[group_id]['scope'] = ', '.join(self.parse_scopes(group)) + + for service in services: + tmps = dict() + + tmps['hostname'] = self.parse_xmltext(service.getElementsByTagName('HOSTNAME')[0].childNodes) + try: + tmps['service_id'] = self.parse_xmltext(service.getElementsByTagName('PRIMARY_KEY')[0].childNodes) + except IndexError: + tmps['service_id'] = service.getAttribute('PRIMARY_KEY') + tmps['type'] = self.parse_xmltext(service.getElementsByTagName('SERVICE_TYPE')[0].childNodes) + tmps['monitored'] = self.parse_xmltext(service.getElementsByTagName('NODE_MONITORED')[0].childNodes) + tmps['production'] = self.parse_xmltext(service.getElementsByTagName('IN_PRODUCTION')[0].childNodes) + tmps['scope'] = ', '.join(self.parse_scopes(service)) + tmps['endpoint_urls'] = self.parse_url_endpoints(service.getElementsByTagName('ENDPOINTS')[0].childNodes) + if self.pass_extensions: + extensions = self.parse_extensions(service.getElementsByTagName('EXTENSIONS')[0].childNodes) + tmps['extensions'] = extensions + self._service_groups[group_id]['services'].append(tmps) + + except (KeyError, IndexError, TypeError, AttributeError, AssertionError) as exc: + msg = module_class_name(self) + ' Customer:%s : Error parsing service groups feed - %s' % (self.logger.customer, repr(exc).replace('\'', '').replace('\"', '')) + raise ConnectorParseError(msg) + + except ConnectorParseError as exc: + raise exc + + def get_group_endpoints(self): + group_list, groupofendpoints = list(), list() + group_list = group_list + [value for _, value in self._service_groups.items()] + + for group in group_list: + for service in group['services']: + tmpg = dict() + tmpg['type'] = 'SERVICEGROUPS' + tmpg['group'] = group['name'] + tmpg['service'] = service['type'] + if self.uidservtype: + tmpg['hostname'] = '{1}_{0}'.format(service['service_id'], service['hostname']) + else: + tmpg['hostname'] = service['hostname'] + tmpg['tags'] = {'scope': service.get('scope', ''), + 'monitored': '1' if service['monitored'].lower() == 'Y'.lower() or + service['monitored'].lower() == 'True'.lower() else '0', + 'production': '1' if service['production'].lower() == 'Y'.lower() or + service['production'].lower() == 'True'.lower() else '0'} + tmpg['tags'].update({'info_id': str(service['service_id'])}) + + if self.pass_extensions: + for key, value in service['extensions'].items(): + tmpg['tags'].update({ + 'info_ext_' + key: value + }) + if service['endpoint_urls']: + tmpg['tags'].update({ + 'info_service_endpoint_URL': service['endpoint_urls'] + }) + + groupofendpoints.append(tmpg) + + return groupofendpoints + + def get_group_groups(self): + group_list, groupofgroups = list(), list() + group_list = group_list + [value for _, value in self._service_groups.items()] + + for group in group_list: + tmpg = dict() + tmpg['type'] = 'PROJECT' + tmpg['group'] = self.custname + tmpg['subgroup'] = group['name'] + tmpg['tags'] = {'monitored': '1' if group['monitored'].lower() == 'Y'.lower() or + group['monitored'].lower() == 'True'.lower() else '0', 'scope': group.get('scope', '')} + groupofgroups.append(tmpg) + + return groupofgroups diff --git a/modules/parse/vapor.py b/modules/parse/vapor.py new file mode 100644 index 00000000..98e029c0 --- /dev/null +++ b/modules/parse/vapor.py @@ -0,0 +1,42 @@ +from argo_egi_connectors.utils import module_class_name +from argo_egi_connectors.log import Logger +from argo_egi_connectors.exceptions import ConnectorParseError +from argo_egi_connectors.parse.base import ParseHelpers + + +class ParseWeights(ParseHelpers): + def __init__(self, logger, data): + self.data = data + self.logger = logger + + def _reformat(self, data): + datawr = [] + for key in data: + w = data[key] + datawr.append({'type': 'computationpower', 'site': key, 'weight': w}) + return datawr + + def get_data(self): + try: + weights = dict() + for ngi in self.parse_json(self.data): + for site in ngi['site']: + key = site['id'] + if 'ComputationPower' in site: + val = site['ComputationPower'] + else: + self.logger.warn(module_class_name(self) + ': No ComputationPower value for NGI:%s Site:%s' % (ngi['ngi'], site['id'])) + val = '0' + weights[key] = val + + return self._reformat(weights) + + except (KeyError, IndexError, ValueError) as exc: + raise ConnectorParseError() + + except Exception as exc: + if getattr(self.logger, 'job', False): + self.logger.error('{} Customer:{} Job:{} : Error - {}'.format(module_class_name(self), self.logger.customer, self.logger.job, repr(exc))) + else: + self.logger.error('{} Customer:{} : Error - {}'.format(module_class_name(self), self.logger.customer, repr(exc))) + raise exc diff --git a/modules/parse/webapi_metricprofile.py b/modules/parse/webapi_metricprofile.py new file mode 100644 index 00000000..8f8f0b6f --- /dev/null +++ b/modules/parse/webapi_metricprofile.py @@ -0,0 +1,60 @@ +import json +from argo_egi_connectors.utils import module_class_name +from argo_egi_connectors.io.http import ConnectorHttpError +from argo_egi_connectors.exceptions import ConnectorParseError +from argo_egi_connectors.parse.base import ParseHelpers + + +class ParseMetricProfiles(ParseHelpers): + def __init__(self, logger, data, target_profiles, namespace): + self.logger = logger + self.data = data + self.target_profiles = target_profiles + self.namespace = namespace + + def get_data(self): + try: + fetched_profiles = self.parse_json(self.data)['data'] + target_profiles = list(filter(lambda profile: profile['name'] in self.target_profiles, fetched_profiles)) + profile_list = list() + + if len(target_profiles) == 0: + self.logger.error('Customer:' + self.logger.customer + ' Job:' + self.logger.job + ': No profiles {0} were found!'.format(', '.join(self.target_profiles))) + raise SystemExit(1) + + for profile in target_profiles: + for service in profile['services']: + for metric in service['metrics']: + if self.namespace: + profile_name = '{0}.{1}'.format(self.namespace, profile['name']) + else: + profile_name = profile['name'] + profile_list.append({ + 'profile': profile_name, + 'metric': metric, + 'service': service['service'] + }) + return profile_list + + except (KeyError, IndexError, ValueError) as exc: + self.logger.error(module_class_name(self) + ': Error parsing feed - %s' % (repr(exc).replace('\'', ''))) + raise ConnectorParseError() + + except Exception as exc: + if getattr(self.logger, 'job', False): + self.logger.error('{} Customer:{} Job:{} : Error - {}'.format(module_class_name(self), self.logger.customer, self.logger.job, repr(exc))) + else: + self.logger.error('{} Customer:{} : Error - {}'.format(module_class_name(self), self.logger.customer, repr(exc))) + raise exc + + def _format(self, profile_list): + profiles = [] + + for profile in profile_list: + tmpp = dict() + tmpp['metric'] = profile['metric'] + tmpp['profile'] = profile['profile'] + tmpp['service'] = profile['service'] + profiles.append(tmpp) + + return profiles diff --git a/modules/utils.py b/modules/utils.py new file mode 100644 index 00000000..2481e83d --- /dev/null +++ b/modules/utils.py @@ -0,0 +1,35 @@ +import datetime +import re + +strerr = '' +num_excp_expand = 0 +daysback = 1 + + +def date_check(arg): + if re.search("[0-9]{4}-[0-9]{2}-[0-9]{2}", arg): + return True + else: + return False + + +def datestamp(daysback=None): + if daysback: + dateback = datetime.datetime.now() - datetime.timedelta(days=daysback) + else: + dateback = datetime.datetime.now() + + return str(dateback.strftime('%Y_%m_%d')) + + +def filename_date(logger, option, path, stamp=None): + stamp = stamp if stamp else datestamp(daysback) + filename = path + re.sub(r'DATE(.\w+)$', r'%s\1' % stamp, option) + + return filename + + +def module_class_name(obj): + name = repr(obj.__class__.__name__) + + return name.replace("'", '') diff --git a/setup.py b/setup.py index 086063d5..a0bc38e3 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,8 @@ from distutils.core import setup import glob -NAME='argo-egi-connectors' +NAME = 'argo-egi-connectors' + def get_ver(): try: @@ -9,7 +10,7 @@ def get_ver(): if "Version:" in line: return line.split()[1] except IOError: - print "Make sure that %s is in directory" % (NAME + '.spec') + print("Make sure that {} is in directory".format(NAME + '.spec')) raise SystemExit(1) @@ -18,14 +19,26 @@ def get_ver(): author='SRCE', author_email='dvrcic@srce.hr, kzailac@srce.hr', description='Components generate input data for ARGO Compute Engine', + classifiers=[ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Apache Software License", + "Operating System :: POSIX", + "Programming Language :: Python :: 3.6", + "Intended Audience :: Developers", + ], url='http://argoeu.github.io/guides/sync/', - package_dir={'argo_egi_connectors': 'modules/'}, - packages=['argo_egi_connectors'], + package_dir={'argo_egi_connectors.io': 'modules/io', + 'argo_egi_connectors.parse': 'modules/parse', + 'argo_egi_connectors.mesh': 'modules/mesh', + 'argo_egi_connectors': 'modules/'}, + packages=['argo_egi_connectors', 'argo_egi_connectors.io', + 'argo_egi_connectors.parse', 'argo_egi_connectors.mesh'], data_files=[('/etc/argo-egi-connectors', glob.glob('etc/*.conf.template')), ('/usr/libexec/argo-egi-connectors', ['bin/downtimes-gocdb-connector.py', 'bin/metricprofile-webapi-connector.py', 'bin/topology-gocdb-connector.py', - 'bin/topology-eosc-connector.py', + 'bin/topology-json-connector.py', 'bin/weights-vapor-connector.py', + 'bin/topology-csv-connector.py', 'bin/replay-avro-data.py']), ('/etc/argo-egi-connectors/schemas', glob.glob('etc/schemas/*.avsc'))]) diff --git a/tests/customer.conf b/tests/customer.conf deleted file mode 100644 index 187a05e4..00000000 --- a/tests/customer.conf +++ /dev/null @@ -1,23 +0,0 @@ -[CUSTOMER_EGI] -Name = EGI -OutputDir = /var/lib/argo-connectors/EGI/ -Jobs = JOB_EGICritical, JOB_EGIFedcloud -AuthenticationHttpUser = user - - -[JOB_EGICritical] -Dirname = EGI_Critical -Profiles = ARGO_MON_CRITICAL -MetricProfileNamespace = ch.cern.SAM -TopoFetchType = Sites -TopoSelectGroupOfEndpoints = Production:Y, Monitored:Y, Scope:EGI -TopoSelectGroupOfGroups = NGI:EGI.eu, Certification:Certified, Infrastructure:Production, Scope:EGI -TopoFeedPaging = True - -[JOB_EGIFedcloud] -Dirname = EGI_Fedcloud -Profiles = FEDCLOUD -MetricProfileNamespace = ch.cern.SAM -TopoFetchType = Sites -TopoSelectGroupOfEndpoints = Monitored:Y, Scope:EGI -TopoSelectGroupOfGroups = Scope:EGI diff --git a/tests/global.conf b/tests/global.conf deleted file mode 100644 index a742d370..00000000 --- a/tests/global.conf +++ /dev/null @@ -1,52 +0,0 @@ -[DEFAULT] -SchemaDir = etc/schemas/ - -[General] -PublishAms = True -WriteAvro = True - -[AMS] -Host = localhost -Token = EGIKEY -Project = EGI -Topic = TOPIC -Bulk = 100 -PackSingleMsg = True - -[WebAPI] -Host = WebAPIHost -Token = WebAPIToken - -[Authentication] -VerifyServerCert = False -CAFile = /etc/pki/tls/certs/ca-bundle.crt -CAPath = /etc/grid-security/certificates -HostKey = /etc/grid-security/hostkey.pem -HostCert = /etc/grid-security/hostcert.pem -UsePlainHttpAuth = True -PoemToken = xxxx - -[Connection] -Timeout = 180 -Retry = 3 -SleepRetry = 60 - -[InputState] -SaveDir = /var/lib/argo-connectors/states/ -Days = 3 - -[AvroSchemas] -Downtimes = %(SchemaDir)s/downtimes.avsc -MetricProfile = %(SchemaDir)s/metric_profiles.avsc -Prefilter = %(SchemaDir)s/metric_data.avsc -TopologyGroupOfEndpoints = %(SchemaDir)s/group_endpoints.avsc -TopologyGroupOfGroups = %(SchemaDir)s/group_groups.avsc -Weights = %(SchemaDir)s/weight_sites.avsc - -[Output] -Downtimes = downtimes_DATE.avro -MetricProfile = poem_sync_DATE.avro -Prefilter = %(EGIDir)s/prefilter_DATE.avro -TopologyGroupOfEndpoints = group_endpoints_DATE.avro -TopologyGroupOfGroups = group_groups_DATE.avro -Weights = weights_DATE.avro diff --git a/tests/run-tests.sh b/tests/run-tests.sh deleted file mode 100755 index 90ae8a91..00000000 --- a/tests/run-tests.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -sitepack=$(python -c 'import sys; \ - sitedirs=[p for p in sys.path if p.endswith("site-packages")]; \ - print sitedirs[0]') - -declare -a files - -if [[ ! -L $sitepack/argo_egi_connectors ]] -then - ln -s $PWD/modules $sitepack/argo_egi_connectors - files[${#files[@]}]="$sitepack/argo_egi_connectors" -fi - -pushd bin/ > /dev/null - -for f in [a-z]*.py -do - link=${f//-/_} - if [[ ! -L $link ]] - then - ln -s $f $link - files[${#files[@]}]="bin/$link" - fi -done - -if [[ ! -e __init__.py ]] -then - touch __init__.py - files[${#files[@]}]="bin/__init__.py" -fi - -popd > /dev/null - -coverage run --source=tests -m unittest2 discover tests -v && coverage xml - -for f in ${files[@]} -do - rm -f $f* -done diff --git a/tests/sample-bdii_sepaths.json b/tests/sample-bdii_sepaths.json new file mode 100644 index 00000000..cbef2a29 --- /dev/null +++ b/tests/sample-bdii_sepaths.json @@ -0,0 +1,39 @@ +[ + { + "GlueVOInfoAccessControlBaseRule": ["ukqcd.vo.gridpp.ac.uk", "VO:ukqcd.vo.gridpp.ac.uk"], + "GlueVOInfoPath": ["/dpm/gla.scotgrid.ac.uk/home/ukqcd.vo.gridpp.ac.uk"], + "dn": "GlueVOInfoLocalID=ukqcd.vo.gridpp.ac.uk:generalPool,GlueSALocalID=generalPool:replica:online,GlueSEUniqueID=svr018.gla.scotgrid.ac.uk,Mds-Vo-name=UKI-SCOTGRID-GLASGOW,Mds-Vo-name=local,o=grid" + }, + { + "GlueVOInfoAccessControlBaseRule": ["VO:atlas"], + "GlueVOInfoPath": ["/atlas"], + "dn": "GlueVOInfoLocalID=atlas,GlueSALocalID=atlas:ATLASLOCALGROUPDISK,GlueSEUniqueID=lcg-se1.sfu.computecanada.ca,Mds-Vo-name=CA-SFU-T2,Mds-Vo-name=local,o=grid" + }, + { + "dn": "GlueSALocalID=tape-,GlueSEUniqueID=srm.pic.es,Mds-Vo-name=pic,Mds-Vo-name=local,o=grid" + }, + { + "dn": "GlueVOInfoLocalID=vo.cta.in2p3.fr:CTA,GlueSALocalID=CTA:SR:replica:online,GlueSEUniqueID=atlasse.lnf.infn.it,Mds-Vo-name=INFN-FRASCATI,Mds-Vo-name=local,o=grid", + "GlueVOInfoAccessControlBaseRule": ["VOMS:/vo.cta.in2p3.fr/Role=production", "VOMS:/vo.cta.in2p3.fr/Role=users"], "GlueVOInfoPath": ["/dpm/lnf.infn.it/home/vo.cta.in2p3.fr"] + }, + { + "dn": "GlueVOInfoLocalID=default-store-ops,GlueSALocalID=nas-complex-7a759b03,GlueSEUniqueID=grid-se.physik.uni-wuppertal.de,Mds-Vo-name=wuppertalprod,Mds-Vo-name=local,o=grid", + "GlueVOInfoAccessControlBaseRule": ["VO:ops"], + "GlueVOInfoPath" : ["/pnfs/physik.uni-wuppertal.de/data/ops"] + }, + { + "dn": "GlueVOInfoLocalID=default-pool-dteam,GlueSALocalID=nas-complex-7a759b03,GlueSEUniqueID=grid-se.physik.uni-wuppertal.de,Mds-Vo-name=wuppertalprod,Mds-Vo-name=local,o=grid", + "GlueVOInfoAccessControlBaseRule": ["VO:dteam"], + "GlueVOInfoPath": "/pnfs/physik.uni-wuppertal.de/data/dteam" + }, + { + "dn": "GlueVOInfoLocalID=default-store-dteam,GlueSALocalID=nas-complex-7a759b03,GlueSEUniqueID=grid-se.physik.uni-wuppertal.de,Mds-Vo-name=wuppertalprod,Mds-Vo-name=local,o=grid", + "GlueVOInfoAccessControlBaseRule": ["VO:dteam"], + "GlueVOInfoPath": "/pnfs/physik.uni-wuppertal.de/data/dteam" + }, + { + "dn": "GlueVOInfoLocalID=ops dteam:INFO-TOKEN,GlueSALocalID=info:replica:online,GlueSEUniqueID=se02.esc.qmul.ac.uk,Mds-Vo-name=UKI-LT2-QMUL,Mds-Vo-name=local,o=grid", + "GlueVOInfoAccessControlBaseRule": ["VO:ops dteam"], + "GlueVOInfoPath": "/info" + } +] diff --git a/tests/sample-roc_contacts.xml b/tests/sample-roc_contacts.xml new file mode 100644 index 00000000..2c63e96d --- /dev/null +++ b/tests/sample-roc_contacts.xml @@ -0,0 +1,58 @@ + + + + CERN + roc-contact@example.com + https://url.com + + Name1 + Surname1 + + <EMAIL>Name1.Surname1@example.com</EMAIL> + <TEL>+11 11 11 11111</TEL> + <CERTDN>/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=Name1/CN=11111/CN=Name1 Surname1</CERTDN> + <ROLE_NAME>NGI Operations Manager</ROLE_NAME> + </CONTACT> + <CONTACT USER_ID="204G0" PRIMARY_KEY="204G0"> + <FORENAME>Name2</FORENAME> + <SURNAME>Surname2</SURNAME> + <TITLE>Dr. + Name2.Surname2@example.com + +22 22 222 2222 + /DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=Name2/CN=111111/CN=Name2 Surname2 + NGI Security Officer + + + Name3 + Surname3 + + <EMAIL>Name3.Surname3@example.com</EMAIL> + <TEL>3333333333333</TEL> + <CERTDN>/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=Name3/CN=222222/CN=Name3 Surname3</CERTDN> + <ROLE_NAME>NGI Operations Deputy Manager</ROLE_NAME> + </CONTACT> + </ROC> + <ROC ID="4" ROC_NAME="EGI.eu"> + <ROCNAME>EGI.eu</ROCNAME> + <MAIL_CONTACT>operations@egi.eu</MAIL_CONTACT> + <GOCDB_PORTAL_URL>https://url.com</GOCDB_PORTAL_URL> + <CONTACT USER_ID="406G0" PRIMARY_KEY="406G0"> + <FORENAME>Name1</FORENAME> + <SURNAME>Surname1</SURNAME> + <TITLE/> + <EMAIL>Name1.Surname1@email.com</EMAIL> + <TEL>555555555555</TEL> + <CERTDN>/C=HR/O=edu/OU=srce/CN=Name1 Surname1</CERTDN> + <ROLE_NAME>Regional Staff (ROD)</ROLE_NAME> + </CONTACT> + <CONTACT USER_ID="6079G0" PRIMARY_KEY="6079G0"> + <FORENAME>Name2</FORENAME> + <SURNAME>Surname2</SURNAME> + <TITLE>Dr + Name2.Surname2@email.com + + 1111111@egi.eu + Regional Staff (ROD) + + + diff --git a/tests/sample-service_endpoint.xml b/tests/sample-service_endpoint.xml new file mode 100644 index 00000000..2c605530 --- /dev/null +++ b/tests/sample-service_endpoint.xml @@ -0,0 +1,98 @@ + + + + 14G0 + grid13.gsi.de + https://goc.egi.eu/portal/index.php?Page_Type=Service&id=14 + N + CE + 140.181.76.82 + + N + N + N + GSI-LCG2 + Germany + DE + NGI_DE + + + + EGI + wlcg + tier2 + alice + + + + + 782G0 + arc-ce01.gridpp.rl.ac.uk + https://goc.egi.eu/portal/index.php?Page_Type=Service&id=782 + /C=UK/O=eScience/OU=CLRC/L=RAL/CN=arc-ce01.gridpp.rl.ac.uk + N + gLite-APEL + + Y + Y + N + RAL-LCG2 + United Kingdom + GB + NGI_UK + + + + EGI + wlcg + tier1 + alice + atlas + cms + lhcb + + + + 494 + InformationSystem + https://www.gridpp.rl.ac.uk/RAL-LCG2/RAL-LCG2_CE.json + + + + + 1555G0 + ce.physics.science.az + https://goc.egi.eu/portal/index.php?Page_Type=Service&id=1555 + /DC=ORG/DC=SEE-GRID/O=Hosts/O=Institute of Physics of ANAS/CN=ce.physics.science.az + Scientific Linux release 6.8 (Carbon) + x86_64 + N + CREAM-CE + 185.147.24.193 + + Y + Y + N + AZ-IFAN + Azerbaijan + AZ + NGI_TR + ce.physics.science.az:8443/cream-pbs-ops + + + 7056 + ce.physics.science.az + + ce.physics.science.az:8443/cream-pbs-ops + CREAM-CE + Y + + + + EGI + wlcg + atlas + + + + diff --git a/tests/sample-service_endpoint_biomed.xml b/tests/sample-service_endpoint_biomed.xml new file mode 100644 index 00000000..625160c2 --- /dev/null +++ b/tests/sample-service_endpoint_biomed.xml @@ -0,0 +1,50 @@ + + + 375G0 + kalkan1.ulakbim.gov.tr + https://goc.egi.eu/portal/index.php?Page_Type=Service&id=375 + /C=TR/O=TRGrid/OU=TUBITAK-ULAKBIM/CN=kalkan1.ulakbim.gov.tr + Scientific Linux 6.2 + N + APEL + 193.140.99.15 + + Y + Y + N + TR-10-ULAKBIM + Turkey + TR + NGI_TR + + + + EGI + wlcg + tier2 + atlas + + + + + 451G0 + cream-ce01.marie.hellasgrid.gr + https://goc.egi.eu/portal/index.php?Page_Type=Service&id=451 + N + APEL + + Y + Y + N + HG-02-IASA + Greece + GR + NGI_GRNET + + + + EGI + + + + diff --git a/tests/sample-service_endpoint_with_contacts.xml b/tests/sample-service_endpoint_with_contacts.xml new file mode 100644 index 00000000..f42bb354 --- /dev/null +++ b/tests/sample-service_endpoint_with_contacts.xml @@ -0,0 +1,75 @@ + + + + 19G0 + some.fqdn.com + https://gocdb.com + contact@email.com + N + service.type + + Y + Y + Y + SITE1 + Germany + DE + NGI1 + https://service.endpoint.com/url + + + SCOPE1 + + + + + 19G1 + some.fqdn1.com + https://gocdb.com + contact1@email.com + N + service.type1 + + Y + Y + Y + SITE1 + Germany + DE + NGI1 + https://service.endpoint1.com/url + + + SCOPE1 + + + + + 142G0 + some.fqdn2.com + https://gocdb.com + contact1@email.com;contact2@email.com;contact3@email.com + N + service.type2 + + Y + Y + Y + SITE2 + Italy + IT + NGI2 + https://service.endpoint2.com/url + + + SCOPE2 + + + + 115 + version + 1.0.4 + + + + diff --git a/tests/sample-service_group_contacts.xml b/tests/sample-service_group_contacts.xml new file mode 100644 index 00000000..01c4ae74 --- /dev/null +++ b/tests/sample-service_group_contacts.xml @@ -0,0 +1,24 @@ + + + + GROUP1 + Group1 Core Services + Y + grid-admin@example.com + https://gocdb.url.com/path + + Name1 + Surnam1 + /C=HR/O=SRCE/OU=SRCE/CN=Name1 Surname1 + https://gocdb.url.com/path + Service Group Administrator + + + + GROUP2 + Core infrastructure services for GROUP2 + Y + grid-meteo@example1.com + https://gocdb.url.com/path + + diff --git a/tests/sample-service_group_with_contacts.xml b/tests/sample-service_group_with_contacts.xml new file mode 100644 index 00000000..7c4bd12d --- /dev/null +++ b/tests/sample-service_group_with_contacts.xml @@ -0,0 +1,109 @@ + + + + B2FIND-Askeladden + + False + name1.surname1@email.com + https://dp.eudat.eu/projects/proj-askeladden-b2find/b2find-askeladden + No 'creg_id' found + + 830865d4204443428f7172dfc666707a + b2find.eudat.eu + https://dp.eudat.eu/providers/DKRZ/b2find.eudat.eu_38 + https://creg.eudat.eu/view_portal/index.php?Page_Type=Service&id=38 + + b2find.ckan + 123.456.789.012 + + + True + eudat-monitoring@dkrz.de + DKRZ + Germany + DE + EUDAT_REGISTRY + http://b2find.eudat.eu + + + + undefined + creg_id + 38 + + + undefined + email + eudat-monitoring@dkrz.de + + + undefined + servicetype_id + 46 + + + undefined + state + production + + + + + EUDAT + + + + + B2FIND-UHH + + False + name2.surname2@email.com + https://dp.eudat.eu/projects/uhh-in-b2find/b2find-uhh + No creg_id found + + 830865d4204443428f7172dfc666707a + b2find.eudat.eu + https://dp.eudat.eu/providers/DKRZ/b2find.eudat.eu_38 + https://creg.eudat.eu/view_portal/index.php?Page_Type=Service&id=38 + + b2find.ckan + 123.456.789.012 + + + True + eudat-monitoring@dkrz.de + DKRZ + Germany + DE + EUDAT_REGISTRY + http://b2find.eudat.eu + + + + undefined + creg_id + 38 + + + undefined + email + eudat-monitoring@dkrz.de + + + undefined + servicetype_id + 46 + + + undefined + state + production + + + + + EUDAT + + + + diff --git a/tests/sample-site_contacts.xml b/tests/sample-site_contacts.xml new file mode 100644 index 00000000..64ab0898 --- /dev/null +++ b/tests/sample-site_contacts.xml @@ -0,0 +1,100 @@ + + + + 73G0 + Site1 + + Name1 + Surname1 + + <EMAIL>Name1.Surname1@email.hr</EMAIL> + <TEL>+111 11 111 1111</TEL> + <CERTDN>/C=HR/O=CROGRID/O=SRCE/CN=Name1 Surname1</CERTDN> + <ROLE_NAME>Site Security Officer</ROLE_NAME> + </CONTACT> + <CONTACT USER_ID="217G0" PRIMARY_KEY="217G0"> + <FORENAME>Name1</FORENAME> + <SURNAME>Surname1</SURNAME> + <TITLE/> + <EMAIL>Name1.Surname1@email.hr</EMAIL> + <TEL>+111 11 111 1111</TEL> + <CERTDN>/C=HR/O=CROGRID/O=SRCE/CN=Name1 Surname1</CERTDN> + <ROLE_NAME>Site Operations Manager</ROLE_NAME> + </CONTACT> + <CONTACT USER_ID="256G0" PRIMARY_KEY="256G0"> + <FORENAME>Name2</FORENAME> + <SURNAME>Surname2</SURNAME> + <TITLE>Mr. + Name2.Surname2@email.hr + +2222 22222222 + /C=HR/O=CROGRID/O=SRCE/CN=Name2 Surname2 + Site Operations Manager + + + + 201G0 + Site2 + + Name1 + Surname1 + + <EMAIL>Name1.Surname1@email.hr</EMAIL> + <TEL>+111 11 111 1111</TEL> + <CERTDN>/C=HR/O=CROGRID/O=SRCE/CN=Name1 Surname1</CERTDN> + <ROLE_NAME>Site Operations Manager</ROLE_NAME> + </CONTACT> + <CONTACT USER_ID="256G0" PRIMARY_KEY="256G0"> + <FORENAME>Name2</FORENAME> + <SURNAME>Surname2</SURNAME> + <TITLE>Mr. + Name2.Surname2@email.hr + +2222 22222222 + /C=HR/O=CROGRID/O=SRCE/CN=Name2 Surname2 + Site Operations Manager + + + Name2 + Surname2 + Mr. + Name2.Surname2@email.hr + +2222 22222222 + /C=HR/O=CROGRID/O=SRCE/CN=Name2 Surname2 + Site Security Officer + + + Name1 + Surname1 + + <EMAIL>Name1.Surname1@email.hr</EMAIL> + <TEL>+111 11 111 1111</TEL> + <CERTDN>/C=HR/O=CROGRID/O=SRCE/CN=Name1 Surname1</CERTDN> + <ROLE_NAME>Site Security Officer</ROLE_NAME> + </CONTACT> + <CONTACT USER_ID="217G0" PRIMARY_KEY="217G0"> + <FORENAME>Name1</FORENAME> + <SURNAME>Surname1</SURNAME> + <TITLE/> + <EMAIL>Name1.Surname1@email.hr</EMAIL> + <TEL>+111 11 111 1111</TEL> + <CERTDN>/C=HR/O=CROGRID/O=SRCE/CN=Name1 Surname1</CERTDN> + <ROLE_NAME>Site Administrator</ROLE_NAME> + </CONTACT> + <CONTACT USER_ID="256G0" PRIMARY_KEY="256G0"> + <FORENAME>Name2</FORENAME> + <SURNAME>Surname2</SURNAME> + <TITLE>Mr. + Name2.Surname2@email.hr + +2222 22222222 + /C=HR/O=CROGRID/O=SRCE/CN=Name2 Surname2 + Site Administrator + + + Name3 + Mr. + Name3.Surname3@email.hr + +3333 33333333 + /C=HR/O=CROGRID/O=SRCE/CN=Name3 Surname3 + Site Administrator + + + diff --git a/tests/sample-sites_biomed.xml b/tests/sample-sites_biomed.xml new file mode 100644 index 00000000..01a4269e --- /dev/null +++ b/tests/sample-sites_biomed.xml @@ -0,0 +1,4 @@ + + + + diff --git a/tests/sample-sites_with_contacts.xml b/tests/sample-sites_with_contacts.xml new file mode 100644 index 00000000..364fc8f1 --- /dev/null +++ b/tests/sample-sites_with_contacts.xml @@ -0,0 +1,61 @@ + + + + 28aa65bd20e047cabb0d474427dadfc2 + INFN + Istituto Nazionale di Fisica Nucleare + https://dp.eudat.eu/providers/infn + No 'creg_id' found + https://www.infn.it/ + name1.surname1@ba.infn.it + not set + IT + Italy + EUDAT_REGISTRY + DICE + Candidate + + + + not set + + infn.it + + + + undefined + state + internal + + + + + ea9a749d796f4cfcab23764eec0d897a + DATACITE + DataCite + https://dp.eudat.eu/providers/DataCite + No 'creg_id' found + http://datacite.org + name2.surname2@email.com + not set + DE + Germany + EUDAT_REGISTRY + DICE + Candidate + + + + not set + + + + + + undefined + state + internal + + + + diff --git a/tests/sample-topo.csv b/tests/sample-topo.csv new file mode 100644 index 00000000..c1c2a35a --- /dev/null +++ b/tests/sample-topo.csv @@ -0,0 +1,4 @@ +Service Unique ID,URL,SERVICE_TYPE,Service Description (Alphanumeric and basic punctuation),SITENAME-SERVICEGROUP,COUNTRY_NAME,CONTACT_EMAIL,notification flag?,Status +tenant_1,https://files.dev.tenant.eu,nextcloud,tenant project Data sharing service,NextCloud,Country,name.surname@country.com,Yes,Development +tenant_2,https://files.tenant.eu,nextcloud,tenant project Data sharing service,NextCloud,Country,name.surname@country.com,Yes,Development +tenant_3,https://sso.tenant.eu,aai,tenant project SSO service,AAI,Greece,name.surname@country.com,Yes,Production diff --git a/tests/sample-topo.json b/tests/sample-topo.json new file mode 100644 index 00000000..cb6c64f1 --- /dev/null +++ b/tests/sample-topo.json @@ -0,0 +1,16 @@ +[ + { + "CONTACT_EMAIL": ["info@t-systems.com"], + "SERVICE_TYPE": "eu.eosc.portal.services.url", + "SITENAME-SERVICEGROUP": "Open Telekom Cloud", + "Service Unique ID": 227, + "URL": "https://open-telekom-cloud.com/en" + }, + { + "CONTACT_EMAIL": ["infn-eosc@lists.infn.it"], + "SERVICE_TYPE": "eu.eosc.portal.services.url", + "SITENAME-SERVICEGROUP": "PaaS Orchestrator ", + "Service Unique ID": 243, + "URL": "https://indigo-paas.cloud.ba.infn.it" + } +] diff --git a/tests/test_bdiisepath.py b/tests/test_bdiisepath.py new file mode 100644 index 00000000..03df8ddb --- /dev/null +++ b/tests/test_bdiisepath.py @@ -0,0 +1,175 @@ +import unittest +import json + +from argo_egi_connectors.log import Logger +from argo_egi_connectors.mesh.storage_element_path import attach_sepath_topodata + +from bonsai import LDAPEntry + + +logger = Logger('test_contactfeed.py') +CUSTOMER_NAME = 'CUSTOMERFOO' + + +class MeshSePathAndTopodata(unittest.TestCase): + def setUp(self): + logger.customer = CUSTOMER_NAME + with open('tests/sample-bdii_sepaths.json') as fh: + content = fh.read() + self.sample_ldap = json.loads(content) + self.maxDiff = None + self.bdiiopts = { + 'bdii': 'True', 'bdiihost': 'bdii.egi.cro-ngi.hr', + 'bdiiport': '2170', + 'bdiiqueryattributessepath': 'GlueVOInfoAccessControlBaseRule GlueVOInfoPath', + 'bdiiqueryattributessrm': 'GlueServiceEndpoint', + 'bdiiquerybase': 'o=grid', + 'bdiiqueryfiltersepath': '(objectClass=GlueSATop)', + 'bdiiqueryfiltersrm': '(&(objectClass=GlueService)(|(GlueServiceType=srm_v1)(GlueServiceType=srm)))' + } + + + self.sample_storage_endpoints = [ + { + "type": "SITES", "group": "UKI-SCOTGRID-GLASGOW", "service": + "eu.egi.storage.accounting", "hostname": + "svr018.gla.scotgrid.ac.uk", "notifications": None, "tags": + { + "scope": "EGI, wlcg, tier2, atlas, cms, lhcb", + "monitored": "1", + "production": "1" + } + }, + { + "type": "SITES", "group": "CA-WATERLOO-T2", "service": "SRM", + "hostname": "lcg-se1.uw.computecanada.ca", "notifications": None, + "tags": { + "scope": "EGI", + "monitored": "1", + "production": "1", + "info_URL": "httpg://lcg-se1.uw.computecanada.ca:8443/srm/managerv2", + "info_SRM_port": "8443" + } + }, + { + "type": "SITES", "group": "CA-SFU-T2", "service": "SRM", + "hostname": "lcg-se1.sfu.computecanada.ca", "notifications": None, + "tags": {"scope": "EGI", "monitored": "1", "production": "1", + "info_URL": + "httpg://lcg-se1.sfu.computecanada.ca:8443/srm/managerv2", + "info_SRM_port": "8443" + } + }, + { + "type": "SITES", "group": "wuppertalprod", "service": "SRM", + "hostname": "grid-se.physik.uni-wuppertal.de", + "notifications": None, + "tags": {"scope": "EGI, wlcg, tier2, atlas", "monitored": "1", + "production": "1", "info_id": "3077G0", + "info_SRM_port": "8443" + } + }, + { + "type": "SITES", + "group": "UKI-LT2-QMUL", + "service": "SRM", + "hostname": "se02.esc.qmul.ac.uk", + "notifications": None, + "tags": {"scope": "EGI, wlcg, tier2, atlas, lhcb", "monitored": "1", + "production": "0", "info_id": "8458G0", + "info_SRM_port": "8444" + } + } + ] + self._construct_ldap_entries() + + def _construct_ldap_entries(self): + tmp = list() + for entry in self.sample_ldap: + new_entry = LDAPEntry(entry['dn']) + for key, value in entry.items(): + if key == 'dn': + continue + new_entry[key] = value + tmp.append(new_entry) + self.sample_ldap = tmp + + def test_meshSePathTopo(self): + attach_sepath_topodata(logger, self.bdiiopts, self.sample_ldap, self.sample_storage_endpoints) + self.assertEqual(self.sample_storage_endpoints, [ + { + 'group': 'UKI-SCOTGRID-GLASGOW', + 'hostname': 'svr018.gla.scotgrid.ac.uk', + 'notifications': None, + 'service': 'eu.egi.storage.accounting', + 'tags': { + 'monitored': '1', + 'production': '1', + 'scope': 'EGI, wlcg, tier2, atlas, cms, lhcb', + 'vo_ukqcd.vo.gridpp.ac.uk_attr_SE_PATH': '/dpm/gla.scotgrid.ac.uk/home/ukqcd.vo.gridpp.ac.uk' + }, + 'type': 'SITES' + }, + { + 'group': 'CA-WATERLOO-T2', + 'hostname': 'lcg-se1.uw.computecanada.ca', + 'notifications': None, + 'service': 'SRM', + 'tags': { + 'info_SRM_port': '8443', + 'info_URL': 'httpg://lcg-se1.uw.computecanada.ca:8443/srm/managerv2', + 'monitored': '1', + 'production': '1', + 'scope': 'EGI' + }, + 'type': 'SITES'}, + { + 'group': 'CA-SFU-T2', + 'hostname': 'lcg-se1.sfu.computecanada.ca', + 'notifications': None, + 'service': 'SRM', + 'tags': { + 'info_SRM_port': '8443', + 'info_URL': 'httpg://lcg-se1.sfu.computecanada.ca:8443/srm/managerv2', + 'monitored': '1', + 'production': '1', + 'scope': 'EGI', + 'vo_atlas_attr_SE_PATH': '/atlas' + }, + 'type': 'SITES' + }, + { + 'group': 'wuppertalprod', + 'hostname': 'grid-se.physik.uni-wuppertal.de', + 'notifications': None, + 'service': 'SRM', + 'tags': { + 'info_SRM_port': '8443', + 'info_id': '3077G0', + 'monitored': '1', + 'production': '1', + 'scope': 'EGI, wlcg, tier2, atlas', + 'vo_dteam_attr_SE_PATH': '/pnfs/physik.uni-wuppertal.de/data/dteam', + 'vo_ops_attr_SE_PATH': '/pnfs/physik.uni-wuppertal.de/data/ops' + }, + 'type': 'SITES' + }, + { + 'group': 'UKI-LT2-QMUL', + 'hostname': 'se02.esc.qmul.ac.uk', + 'notifications': None, + 'service': 'SRM', + 'tags': {'info_SRM_port': '8444', + 'info_id': '8458G0', + 'monitored': '1', + 'production': '0', + 'scope': 'EGI, wlcg, tier2, atlas, lhcb', + 'vo_dteam_attr_SE_PATH': '/info', + 'vo_ops_attr_SE_PATH': '/info' + }, + 'type': 'SITES' + }]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_config.py b/tests/test_config.py deleted file mode 100644 index 50e4ce07..00000000 --- a/tests/test_config.py +++ /dev/null @@ -1,91 +0,0 @@ -import unittest2 as unittest -import modules.config - -class TestConfig(unittest.TestCase): - def setUp(self): - self.globalconfig = modules.config.Global('topology-gocdb-connector.py', 'tests/global.conf') - self.customerconfig = modules.config.CustomerConf('topology-gocdb-connector.py', 'tests/customer.conf') - self.globalconfig_nocaller = modules.config.Global(None, 'tests/global.conf') - - def testGlobalParse(self): - opts = self.globalconfig.parse() - self.assertTrue(isinstance(opts, dict)) - self.assertEqual(opts['outputtopologygroupofendpoints'], 'group_endpoints_DATE.avro') - self.assertEqual(opts['outputtopologygroupofgroups'], 'group_groups_DATE.avro') - self.assertEqual(opts['avroschemastopologygroupofendpoints'], 'etc/schemas//group_endpoints.avsc') - self.assertEqual(opts['avroschemastopologygroupofgroups'], 'etc/schemas//group_groups.avsc') - - opts_nocall = self.globalconfig_nocaller.parse() - self.assertEqual(opts_nocall['amshost'],'localhost') - self.assertEqual(opts_nocall['authenticationverifyservercert'], 'False') - self.assertEqual(opts_nocall['authenticationcafile'], '/etc/pki/tls/certs/ca-bundle.crt') - self.assertEqual(opts_nocall['inputstatedays'], '3') - self.assertEqual(opts_nocall['amstoken'], 'EGIKEY') - self.assertEqual(opts_nocall['authenticationcapath'], '/etc/grid-security/certificates') - self.assertEqual(opts_nocall['inputstatesavedir'], '/var/lib/argo-connectors/states/') - self.assertEqual(opts_nocall['connectionretry'], '3') - self.assertEqual(opts_nocall['amsproject'], 'EGI') - self.assertEqual(opts_nocall['generalpublishams'], 'True') - self.assertEqual(opts_nocall['generalwriteavro'], 'True') - self.assertEqual(opts_nocall['authenticationhostcert'], '/etc/grid-security/hostcert.pem') - self.assertEqual(opts_nocall['connectiontimeout'], '180') - self.assertEqual(opts_nocall['authenticationhostkey'], '/etc/grid-security/hostkey.pem') - self.assertEqual(opts_nocall['amstopic'], 'TOPIC') - self.assertEqual(opts_nocall['amsbulk'], '100') - self.assertEqual(opts_nocall['amspacksinglemsg'], 'True') - - def testAmsOpts(self): - opts = self.globalconfig.parse() - ams_incomplete = dict(amshost='host', amstoken='token') - complete, missing = self.globalconfig.is_complete(ams_incomplete, 'ams') - self.assertFalse(complete) - self.assertEqual(missing, set(['amsproject', 'amstopic', 'amsbulk', 'amspacksinglemsg'])) - merged = self.globalconfig.merge_opts(ams_incomplete, 'ams') - self.assertEqual(merged, dict(amshost='host', amsproject='EGI', - amstoken='token', amstopic='TOPIC', - amsbulk='100', amspacksinglemsg='True')) - - def testHttpAuthOpts(self): - globalopts = self.globalconfig.parse() - feeds = self.customerconfig.get_mapfeedjobs('topology-gocdb-connector.py', 'GOCDB', deffeed='https://goc.egi.eu/gocdbpi/') - jobcust = feeds.items()[0][1] - auth_custopts = self.customerconfig.get_authopts('foofeed', jobcust) - auth_opts = self.globalconfig.merge_opts(auth_custopts, 'authentication') - complete, missing = self.globalconfig.is_complete(auth_opts, 'authentication') - self.assertEqual(complete, False) - self.assertEqual(missing, set(['authenticationhttppass'])) - - def testCustomerParse(self): - opts = self.customerconfig.parse() - customers = self.customerconfig.get_customers() - self.assertEqual(customers, ['CUSTOMER_EGI']) - jobs = self.customerconfig.get_jobs(customers[0]) - self.assertEqual(jobs, ['JOB_EGICritical', 'JOB_EGIFedcloud']) - custdir = self.customerconfig.get_custdir(customers[0]) - self.assertEqual(custdir, '/var/lib/argo-connectors/EGI/') - ggtags = self.customerconfig.get_gocdb_ggtags(jobs[0]) - self.assertEqual(ggtags, {'NGI': 'EGI.eu', 'Infrastructure': - 'Production', 'Certification': 'Certified', - 'Scope': 'EGI'}) - getags = self.customerconfig.get_gocdb_getags(jobs[0]) - self.assertEqual(getags, {'Scope': 'EGI', 'Production': 'Y', 'Monitored': 'Y'}) - profiles = self.customerconfig.get_profiles(jobs[0]) - self.assertEqual(profiles, ['ARGO_MON_CRITICAL']) - feedjobs = self.customerconfig.get_mapfeedjobs('topology-gocdb-connector.py', - 'GOCDB', - deffeed='https://goc.egi.eu/gocdbpi/') - self.assertEqual(feedjobs, {'https://goc.egi.eu/gocdbpi/': - [('JOB_EGICritical', 'CUSTOMER_EGI'), - ('JOB_EGIFedcloud', 'CUSTOMER_EGI')]}) - - def testMetricProfileOpts(self): - customerconfig = modules.config.CustomerConf('metricprofile-webapi-connector.py', 'tests/customer.conf') - opts = customerconfig.parse() - customers = self.customerconfig.get_customers() - jobs = self.customerconfig.get_jobs(customers[0]) - namespace = self.customerconfig.get_namespace(jobs[0]) - self.assertEqual(namespace, 'ch.cern.SAM') - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_contactfeed.py b/tests/test_contactfeed.py new file mode 100644 index 00000000..10863940 --- /dev/null +++ b/tests/test_contactfeed.py @@ -0,0 +1,228 @@ +import unittest + +from argo_egi_connectors.log import Logger +from argo_egi_connectors.parse.gocdb_contacts import ParseSiteContacts, ParseSitesWithContacts, \ + ParseRocContacts, ParseServiceEndpointContacts, \ + ParseServiceGroupRoles, ParseServiceGroupWithContacts, ConnectorParseError +from argo_egi_connectors.parse.gocdb_topology import ParseServiceEndpoints +from argo_egi_connectors.parse.flat_topology import ParseContacts + + +logger = Logger('test_contactfeed.py') +CUSTOMER_NAME = 'CUSTOMERFOO' + + +class ParseRocContactTest(unittest.TestCase): + def setUp(self): + with open('tests/sample-roc_contacts.xml') as feed_file: + self.content = feed_file.read() + logger.customer = CUSTOMER_NAME + parse_roc_contacts = ParseRocContacts(logger, self.content) + self.roc_contacts = parse_roc_contacts.get_contacts() + + def test_formatContacts(self): + self.assertEqual(self.roc_contacts[0], + { + 'name': 'CERN', + 'contacts': [ + { + 'certdn': '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=Name1/CN=11111/CN=Name1 Surname1', + 'email': 'Name1.Surname1@example.com', + 'forename': 'Name1', + 'role': 'NGI Operations Manager', + 'surname': 'Surname1' + }, + { + 'certdn': '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=Name2/CN=111111/CN=Name2 Surname2', + 'email': 'Name2.Surname2@example.com', + 'forename': 'Name2', + 'role': 'NGI Security Officer', + 'surname': 'Surname2' + }, + { + 'certdn': '/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=Name3/CN=222222/CN=Name3 Surname3', + 'email': 'Name3.Surname3@example.com', + 'forename': 'Name3', + 'role': 'NGI Operations Deputy Manager', + 'surname': 'Surname3' + } + ] + } + ) + + +class ParseSitesContactTest(unittest.TestCase): + def setUp(self): + with open('tests/sample-site_contacts.xml') as feed_file: + self.content = feed_file.read() + logger.customer = CUSTOMER_NAME + parse_sites_contacts = ParseSiteContacts(logger, self.content) + self.site_contacts = parse_sites_contacts.get_contacts() + + def test_lenContacts(self): + self.assertEqual(len(self.site_contacts), 2) + site_1 = len(self.site_contacts[0]['contacts']) + site_2 = len(self.site_contacts[1]['contacts']) + self.assertEqual(10, site_1 + site_2) + + def test_malformedContacts(self): + self.assertRaises(ConnectorParseError, ParseSiteContacts, logger, 'wrong mocked data') + + def test_formatContacts(self): + self.assertEqual(self.site_contacts[0], + { + 'name': 'Site1', + 'contacts': [ + { + 'certdn': '/C=HR/O=CROGRID/O=SRCE/CN=Name1 Surname1', + 'email': 'Name1.Surname1@email.hr', + 'forename': 'Name1', + 'role': 'Site Security Officer', + 'surname': 'Surname1' + }, + { + 'certdn': '/C=HR/O=CROGRID/O=SRCE/CN=Name1 Surname1', + 'email': 'Name1.Surname1@email.hr', + 'forename': 'Name1', + 'role': 'Site Operations Manager', + 'surname': 'Surname1' + }, + { + 'certdn': '/C=HR/O=CROGRID/O=SRCE/CN=Name2 Surname2', + 'email': 'Name2.Surname2@email.hr', + 'forename': 'Name2', + 'role': 'Site Operations Manager', + 'surname': 'Surname2' + } + ], + } + ) + # contact without surname + self.assertEqual( + self.site_contacts[1]['contacts'][6], + { + 'certdn': '/C=HR/O=CROGRID/O=SRCE/CN=Name3 Surname3', + 'email': 'Name3.Surname3@email.hr', + 'forename': 'Name3', + 'role': 'Site Administrator', + 'surname': '' + } + ) + + +class ParseSitesWithContactTest(unittest.TestCase): + def setUp(self): + with open('tests/sample-sites_with_contacts.xml') as feed_file: + self.content = feed_file.read() + self.maxDiff = None + logger.customer = CUSTOMER_NAME + parse_sites_contacts = ParseSitesWithContacts(logger, self.content) + self.site_contacts = parse_sites_contacts.get_contacts() + + def test_formatContacts(self): + self.assertEqual(self.site_contacts[0], + { + 'name': 'INFN', + 'contacts': ['name1.surname1@ba.infn.it'] + } + ) + + +class ParseServiceEndpointsWithContactsTest(unittest.TestCase): + def setUp(self): + with open('tests/sample-service_endpoint_with_contacts.xml') as feed_file: + self.content = feed_file.read() + logger.customer = CUSTOMER_NAME + + serviceendpoint_contacts = ParseServiceEndpointContacts(logger, self.content) + self.serviceendpoint_contacts = serviceendpoint_contacts.get_contacts() + + with open('tests/sample-service_endpoint.xml') as feed_file: + self.content = feed_file.read() + serviceendpoint_nocontacts = ParseServiceEndpointContacts(logger, self.content) + self.serviceendpoint_nocontacts = serviceendpoint_nocontacts.get_contacts() + + def test_formatContacts(self): + self.assertEqual(self.serviceendpoint_contacts[0], + { + 'contacts': ['contact@email.com'], + 'name': 'some.fqdn.com+service.type' + } + ) + self.assertEqual(self.serviceendpoint_contacts[2], + { + 'contacts': ['contact1@email.com', 'contact2@email.com', + 'contact3@email.com'], + 'name': 'some.fqdn2.com+service.type2' + } + ) + + def test_formatNoContacts(self): + self.assertEqual(self.serviceendpoint_nocontacts, []) + + +class ParseServiceGroupRolesTest(unittest.TestCase): + def setUp(self): + with open('tests/sample-service_group_contacts.xml') as feed_file: + self.content = feed_file.read() + logger.customer = CUSTOMER_NAME + + servicegroup_contacts = ParseServiceGroupRoles(logger, self.content) + self.servicegroup_contacts = servicegroup_contacts.get_contacts() + + def test_formatContacts(self): + self.assertEqual(self.servicegroup_contacts[0], + { + 'contacts': ['grid-admin@example.com'], + 'name': 'GROUP1' + } + ) + + +class ParseServiceGroupWithContactsTest(unittest.TestCase): + def setUp(self): + with open('tests/sample-service_group_with_contacts.xml') as feed_file: + self.content = feed_file.read() + logger.customer = CUSTOMER_NAME + + servicegroup_contacts = ParseServiceGroupWithContacts(logger, self.content) + self.servicegroup_contacts = servicegroup_contacts.get_contacts() + + def test_formatContacts(self): + self.assertEqual(self.servicegroup_contacts[0], + { + 'contacts': ['name1.surname1@email.com'], + 'name': 'B2FIND-Askeladden' + } + ) + + +class ParseCsvServiceEndpointsWithContacts(unittest.TestCase): + def setUp(self): + with open('tests/sample-topo.csv') as feed_file: + self.content = feed_file.read() + logger.customer = CUSTOMER_NAME + + self.contacts = ParseContacts(logger, self.content, uidservtype=True, is_csv=True).get_contacts() + + def test_FormatContacts(self): + self.assertEqual(self.contacts, + [ + { + 'contacts': ['name.surname@country.com'], + 'name': 'files.dev.tenant.eu_tenant_1+nextcloud' + }, + { + 'contacts': ['name.surname@country.com'], + 'name': 'files.tenant.eu_tenant_2+nextcloud' + }, + { + 'contacts': ['name.surname@country.com'], + 'name': 'sso.tenant.eu_tenant_3+aai' + } + ] + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_input.py b/tests/test_input.py deleted file mode 100644 index e1cbbb1d..00000000 --- a/tests/test_input.py +++ /dev/null @@ -1,842 +0,0 @@ -import datetime -import httplib -import mock -import modules.config -import unittest2 as unittest - -from bin.downtimes_gocdb_connector import GOCDBReader as DowntimesGOCDBReader -from bin.downtimes_gocdb_connector import main as downtimes_main -from bin.topology_gocdb_connector import GOCDBReader, TopoFilter -from bin.weights_vapor_connector import Vapor as VaporReader -from modules.log import Logger - - -class ConnectorSetup(object): - downtimes_feed = \ - """\n\n - \n - 100728G0\n - nagios.c4.csir.co.za\n - ngi.SAM\n - nagios.c4.csir.co.zangi.SAM\n - ZA-MERAKA\n - https://goc.egi.eu/portal/index.php?Page_Type=Downtime&id=22154\n - \n - OUTAGE\n - Preparation for decommissioning of the service.\n - 1481808624\n - 1482105600\n - 1488240000\n - 2016-12-19 00:00\n - 2017-02-28 00:00\n - \n - \n - 100784G0\n - ce1.grid.lebedev.ru\n - CE\n - ce1.grid.lebedev.ruCE\n - ru-Moscow-FIAN-LCG2\n - https://goc.egi.eu/portal/index.php?Page_Type=Downtime&id=22209\n - \n - OUTAGE\n - Problems with hosting room (hack for ATLAS site status script that does not currently handle site status and works only on DTs)\n - 1482748113\n - 1482882540\n - 1485215940\n - 2016-12-27 23:49\n - 2017-01-23 23:59\n - \n - \n - 100784G0\n - ce1.grid.lebedev.ru\n - APEL\n - ce1.grid.lebedev.ruAPEL\n - ru-Moscow-FIAN-LCG2\n - https://goc.egi.eu/portal/index.php?Page_Type=Downtime&id=22209\n - \n - OUTAGE\n - Problems with hosting room (hack for ATLAS site status script that does not currently handle site status and works only on DTs)\n - 1482748113\n - 1482882540\n - 1485215940\n - 2016-12-27 23:49\n - 2017-01-23 23:59\n - \n - \n""" - - weights_feed = \ - """[ {"ngi": "NGI_FRANCE", "site": - [ - { "codeCountry":"FR", - "country":"france", - "description":"Grille au service de la Recherche en Ile de France", - "latitude":"48.699262", - "longitude":"2.170686", - "ngi":"NGI_FRANCE", - "subregioncode":"155", - "url":"http://grif.fr/", - "id":"GRIF-LLR", - "Tier":"NA", - "SI2000":"0", - "specint2000":"0", - "HEPSPEC2006":"0", - "ExecutionEnvironmentLogicalCPUs":"0", - "ExecutionEnvironmentPhysicalCPUs":"0", - "ExecutionEnvironmentTotalInstances":"0", - "ComputingManagerLogicalCPUs":"0", - "ComputingManagerPhysicalCPUs":"0", - "ComputationPower":"0", - "WaitingJobs":"0", - "RunningJobs":"410", - "TotalJobs":"410", - "DataStoreTotalSize":"0", - "DataStoreFreeSize":"0", - "DataStoreUsedSize":"0", - "ServiceOnlineTotalSize":"1537617", - "ServiceOnlineFreeSize":"103181", - "ServiceOnlineUsedSize":"1412339", - "ServiceNearlineTotalSize":"0", - "ServiceNearlineFreeSize":"0", - "ServiceNearlineUsedSize":"0", - "TotalSize":"1537615", - "FreeSize":"125269", - "UsedSize":"1412344", - "TotalSizeOnline":"1537615", - "FreeSizeOnline":"125269", - "UsedSizeOnline":"1412344", - "TotalSizeNearline":"0", - "FreeSizeNearline":"0", - "UsedSizeNearline":"0" - } - ] - }, - {"ngi": "NGI_FRANCE", "site": - [ - { "id":"IN2P3-IRES", - "description":"EGI Site", - "url":"http://www.iphc.cnrs.fr/", - "Tier":"2", - "ngi":"NGI_FRANCE", - "country":"france", - "codeCountry":"FR", - "longitude":"7.7095", - "latitude":"48.6056", - "logCpu":"2420", - "PhyCpu":"272", - "subregioncode":"155", - "SI2000":"3142", - "specint2000":"3142", - "HEPSPEC2006":"13", - "ExecutionEnvironmentLogicalCPUs":"2420", - "ExecutionEnvironmentPhysicalCPUs":"272", - "ExecutionEnvironmentTotalInstances":"302", - "ComputingManagerLogicalCPUs":"0", - "ComputingManagerPhysicalCPUs":"0", - "ComputationPower":"30414.559999999998", - "WaitingJobs":"691", - "RunningJobs":"431", - "TotalJobs":"1122", - "DataStoreTotalSize":"0", - "DataStoreFreeSize":"0", - "DataStoreUsedSize":"0", - "ServiceOnlineTotalSize":"1616089", - "ServiceOnlineFreeSize":"255735", - "ServiceOnlineUsedSize":"1360353", - "ServiceNearlineTotalSize":"0", - "ServiceNearlineFreeSize":"0", - "ServiceNearlineUsedSize":"0", - "TotalSize":"1604423", - "FreeSize":"255752", - "UsedSize":"1348671", - "TotalSizeOnline":"1604423", - "FreeSizeOnline":"255752", - "UsedSizeOnline":"1348671", - "TotalSizeNearline":"0", - "FreeSizeNearline":"0", - "UsedSizeNearline":"0" - } - ] - }, - {"ngi": "NGI_DE", "site": - [ - { "id":"FZK-LCG2", - "description":"DE-KIT, Forschungszentrum Karlsruhe (FZK), GridKa", - "url":"http://www.gridka.de/", - "Tier":"1", - "ngi":"NGI_DE", - "country":"germany", - "codeCountry":"DE", - "longitude":"8.4321", - "latitude":"49.0963", - "logCpu":"20964", - "PhyCpu":"0", - "subregioncode":"155", - "SI2000":"0", - "specint2000":"0", - "HEPSPEC2006":"0", - "ExecutionEnvironmentLogicalCPUs":"0", - "ExecutionEnvironmentPhysicalCPUs":"0", - "ExecutionEnvironmentTotalInstances":"0", - "ComputingManagerLogicalCPUs":"20964", - "ComputingManagerPhysicalCPUs":"0", - "ComputationPower":"0", - "WaitingJobs":"12158", - "RunningJobs":"-239724", - "TotalJobs":"-227464", - "DataStoreTotalSize":"0", - "DataStoreFreeSize":"0", - "DataStoreUsedSize":"0", - "ServiceOnlineTotalSize":"0", - "ServiceOnlineFreeSize":"0", - "ServiceOnlineUsedSize":"0", - "ServiceNearlineTotalSize":"0", - "ServiceNearlineFreeSize":"0", - "ServiceNearlineUsedSize":"0", - "TotalSize":"0", - "FreeSize":"0", - "UsedSize":"0", - "TotalSizeOnline":"0", - "FreeSizeOnline":"0", - "UsedSizeOnline":"0", - "TotalSizeNearline":"0", - "FreeSizeNearline":"0", - "UsedSizeNearline":"0" - } - ] - } - ]""" - - group_groups_feed = \ - """ - - - 73G0 - TU-Kosice - Technical University of Kosice - https://goc.egi.eu/portal/index.php?Page_Type=Site&id=40 - http://www.tuke.sk - grid-adm@lists.tuke.sk - +421 55 602 5123 - ldap://mon.grid.tuke.sk:2170/Mds-Vo-name=TU-Kosice,o=grid - SK - Slovakia - NGI_SK - Production - Certified - Europe/Bratislava - 48.73 - 21.24 - grid-adm@lists.tuke.sk - FALSE - - tuke.sk - - 0.0.0.0/255.255.255.255 - - EGI - - - - - 201G0 - IISAS-Bratislava - Institute of Informatics, Slovak Academy of Sciences, Bratislava - IISAS Bratislava production EGI site - https://goc.egi.eu/portal/index.php?Page_Type=Site&id=41 - grid-admin.ui@sav.sk - +421-2-5941-1289 - ldap://sbdii.ui.savba.sk:2170/Mds-Vo-name=IISAS-Bratislava,o=grid - SK - Slovakia - NGI_SK - Production - Certified - Europe/Bratislava - 48.17 - 17.07 - grid-admin.ui@sav.sk - FALSE - - ui.savba.sk - - 0.0.0.0/255.255.255.255 - - EGI - - - - - 8G0 - prague_cesnet_lcg2_cert - Prague Cesnet - https://goc.egi.eu/portal/index.php?Page_Type=Site&id=42 - www.cesnet.cz - lcg-admin@fzu.cz - +420 2 6605 2145 - ldap://skurut16.cesnet.cz:2170/mds-vo-name=prague_cesnet_lcg2_cert,o=grid - CZ - Czech Republic - NGI_CZ - Production - Closed - Europe/Prague - lcg-admin@fzu.cz - FALSE - - cesnet.cz - - 0.0.0.0/255.255.255.255 - - EGI - - - - - """ - - group_endpoints_feed = \ - """\n - \n - \n - 8253G0\n - occi-api.100percentit.com\n - https://goc.egi.eu/portal/index.php?Page_Type=Service&id=8253\n - N\n - eu.egi.cloud.vm-management.occi\n - \n - Y\n - Y\n - 100IT\n - United Kingdom\n - GB\n - NGI_UK\n - https://occi-api.100percentit.com:8787/occi1.1/?image=53d9172f-599f-4340-86a2-a52b425f80a3&platform=openstack&resource=1\n - \n - \n - EGI\n - FedCloud\n - \n - \n - \n - \n - 4495G0\n - egi-cloud-accounting.100percentit.com\n - https://goc.egi.eu/portal/index.php?Page_Type=Service&id=4495\n - N\n - eu.egi.cloud.accounting\n - \n - Y\n - Y\n - 100IT\n - United Kingdom\n - GB\n - NGI_UK\n - 100IT\n - \n - \n - EGI\n - FedCloud\n - \n - \n - \n - \n - 4588G0\n - occi-api.100percentit.com\n - https://goc.egi.eu/portal/index.php?Page_Type=Service&id=4588\n - N\n - eu.egi.cloud.information.bdii\n - \n - Y\n - Y\n - 100IT\n - United Kingdom\n - GB\n - NGI_UK\n - ldap://site-bdii.100percentit.com:2170\n - \n - \n - EGI\n - FedCloud\n - \n - \n - \n - \n""" - - poem_feed = """ - {"name": "FEDCLOUD", - "description": "Profile for Fedcloud CentOS 7 instance", - "vo": "ops", - "metric_instances": - [ - {"service_flavour": "eu.egi.cloud.vm-management.occi", - "metric": "eu.egi.cloud.OCCI-AppDB-Sync"}, - {"service_flavour": "eu.egi.cloud.vm-management.occi", - "metric": "eu.egi.cloud.OCCI-Categories"}, - {"service_flavour": "eu.egi.cloud.vm-management.occi", - "metric": "eu.egi.cloud.OCCI-Context"}, - {"service_flavour": "eu.egi.cloud.vm-management.occi", - "metric": "eu.egi.cloud.OCCI-VM-OIDC"}, - {"service_flavour": "org.openstack.nova", - "metric": "eu.egi.cloud.OpenStack-VM-OIDC"}, - {"service_flavour": "org.openstack.nova", - "metric": "eu.egi.cloud.OpenStack-VM-VOMS-OIDC"} - ] - } - """ - - erroneous_poem_feed = """ - {"name": "FEDCLOUD", - "description": "Profile for Fedcloud CentOS 7 instance", - "vo": "ops", - "metrics": - [ - {"service_flavour": "eu.egi.cloud.vm-management.occi", - "metric": "eu.egi.cloud.OCCI-AppDB-Sync"}, - {"service_flavour": "eu.egi.cloud.vm-management.occi", - "metric": "eu.egi.cloud.OCCI-Categories"}, - {"service_flavour": "eu.egi.cloud.vm-management.occi", - "metric": "eu.egi.cloud.OCCI-Context"}, - {"service_flavour": "eu.egi.cloud.vm-management.occi", - "metric": "eu.egi.cloud.OCCI-VM-OIDC"}, - {"service_flavour": "org.openstack.nova", - "metric": "eu.egi.cloud.OpenStack-VM-OIDC"}, - {"service_flavour": "org.openstack.nova", - "metric": "eu.egi.cloud.OpenStack-VM-VOMS-OIDC"} - ] - } - """ - - - poem = [{'metric': u'eu.egi.cloud.OCCI-AppDB-Sync', - 'profile': u'ch.cern.SAM.FEDCLOUD', - 'service': u'eu.egi.cloud.vm-management.occi', - 'vo': 'ops'}, - {'metric': u'eu.egi.cloud.OCCI-Categories', - 'profile': u'ch.cern.SAM.FEDCLOUD', - 'service': u'eu.egi.cloud.vm-management.occi', - 'vo': 'ops'}, - {'metric': u'eu.egi.cloud.OCCI-Context', - 'profile': u'ch.cern.SAM.FEDCLOUD', - 'service': u'eu.egi.cloud.vm-management.occi', - 'vo': 'ops'}, - {'metric': u'eu.egi.cloud.OCCI-VM-OIDC', - 'profile': u'ch.cern.SAM.FEDCLOUD', - 'service': u'eu.egi.cloud.vm-management.occi', - 'vo': 'ops'}, - {'metric': u'eu.egi.cloud.OpenStack-VM-OIDC', - 'profile': u'ch.cern.SAM.FEDCLOUD', - 'service': u'org.openstack.nova', - 'vo': 'ops'}, - {'metric': u'eu.egi.cloud.OpenStack-VM-VOMS-OIDC', - 'profile': u'ch.cern.SAM.FEDCLOUD', - 'service': u'org.openstack.nova', - 'vo': 'ops'}] - - downtimes = [{'end_time': '2017-01-19T23:59:00Z', - 'hostname': u'nagios.c4.csir.co.za', - 'service': u'ngi.SAM', - 'start_time': '2017-01-19T00:00:00Z'}, - {'end_time': '2017-01-19T23:59:00Z', - 'hostname': u'ce1.grid.lebedev.ru', - 'service': u'CE', 'start_time': - '2017-01-19T00:00:00Z'}, - {'end_time': '2017-01-19T23:59:00Z', - 'hostname': u'ce1.grid.lebedev.ru', - 'service': u'APEL', - 'start_time': '2017-01-19T00:00:00Z'}] - - weights = {u'FZK-LCG2': u'0', u'IN2P3-IRES': u'30414.559999999998', u'GRIF-LLR': u'0'} - - group_groups = [{'group': u'NGI_SK', 'subgroup': u'IISAS-Bratislava', - 'tags': {'certification': u'Certified', - 'infrastructure': u'Production', - 'scope': 'EGI'}, - 'type': 'NGI'}, - {'group': u'NGI_SK', 'subgroup': u'TU-Kosice', - 'tags': {'certification': u'Certified', - 'infrastructure': u'Production', - 'scope': 'EGI'}, - 'type': 'NGI'}, - {'group': u'NGI_CZ', 'subgroup': u'prague_cesnet_lcg2_cert', - 'tags': {'certification': u'Closed', - 'infrastructure': u'Production', - 'scope': 'EGI'}, - 'type': 'NGI'}] - - group_groups_servicegroup_filter = [ - {'group': 'EGI', 'subgroup': u'NGI_CH_SERVICES', - 'tags': {'monitored': '1', 'scope': 'EGI'}, - 'type': 'PROJECT'}, - {'group': 'EGI', 'subgroup': u'SLA_TEST_B', - 'tags': {'monitored': '1', 'scope': 'EGI'}, - 'type': 'PROJECT'}, - {'group': 'EGI', 'subgroup': u'NGI_HU_SERVICES', - 'tags': {'monitored': '1', 'scope': 'EGI'}, - 'type': 'PROJECT'}, - {'group': 'EGI', 'subgroup': u'SLA_TEST', - 'tags': {'monitored': '1', 'scope': 'EGI'}, - 'type': 'PROJECT'}, - ] - - group_groups_sites_filter = [ - {"group": "NGI_HR", "tags": {"scope": "EGI", "infrastructure": - "Production", "certification": - "Certified"}, - "type": "NGI", "subgroup": "egee.irb.hr"}, - {"group": "NGI_HR", "tags": {"scope": "EGI", "infrastructure": - "Production", "certification": - "Certified"}, - "type": "NGI", "subgroup": "egee.srce.hr"} - ] - - group_endpoints_sites_filter = [ - {"group": "egee.irb.hr", "hostname": "lorienmaster.irb.hr", - "type": "SITES", "service": "Site-BDII", - "tags": {"scope": "EGI", "production": "1", "monitored": "1"}}, - {"group": "egee.srce.hr", "hostname": "se.srce.egi.cro-ngi.hr", - "type": "SITES", "service": "gLite-APEL", "tags": {"scope": "EGI", - "production": "1", - "monitored": "1"}} - ] - - group_endpoints = [{'group': u'100IT', - 'hostname': u'occi-api.100percentit.com', - 'service': u'eu.egi.cloud.vm-management.occi', - 'tags': {'monitored': '1', - 'production': '1', - 'scope': 'EGI'}, - 'type': 'SITES'}, - {'group': u'100IT', - 'hostname': u'egi-cloud-accounting.100percentit.com', - 'service': u'eu.egi.cloud.accounting', - 'tags': {'monitored': '1', - 'production': '1', - 'scope': 'EGI'}, - 'type': 'SITES'}, - {'group': u'100IT', - 'hostname': u'occi-api.100percentit.com', - 'service': u'eu.egi.cloud.information.bdii', - 'tags': {'monitored': '1', - 'production': '1', - 'scope': 'EGI'}, - 'type': 'SITES'}] - - group_endpoints_uid = [{'group': u'100IT', - 'hostname': u'occi-api.100percentit.com_4497G0', - 'service': u'eu.egi.cloud.vm-management.occi', - 'tags': {'monitored': '1', - 'production': '1', - 'scope': 'EGI'}, - 'type': 'SITES'}, - {'group': u'100IT', - 'hostname': u'egi-cloud-accounting.100percentit.com_4495G0', - 'service': u'eu.egi.cloud.accounting', - 'tags': {'monitored': '1', - 'production': '1', - 'scope': 'EGI'}, - 'type': 'SITES'}, - {'group': u'100IT', - 'hostname': u'occi-api.100percentit.com_4588G0', - 'service': u'eu.egi.cloud.information.bdii', - 'tags': {'monitored': '1', - 'production': '1', - 'scope': 'EGI'}, - 'type': 'SITES'}] - - group_endpoints_servicegroup_filter = [ - {'group': u'SLA_TEST_B', - 'group_monitored': u'Y', - 'hostname': u'snf-189278.vm.okeanos.grnet.gr', - 'service': u'eu.egi.MPI', - 'tags': {'monitored': '1', 'production': '1', 'scope': 'EGI'}, - 'type': 'SERVICEGROUPS'}, - {'group': u'SLA_TEST', - 'group_monitored': u'Y', - 'hostname': u'se01.marie.hellasgrid.gr', - 'service': u'SRM', - 'tags': {'monitored': '1', 'production': '1', 'scope': 'EGI'}, - 'type': 'SERVICEGROUPS'}, - {'group': u'NGI_HU_SERVICES', - 'group_monitored': u'Y', - 'hostname': u'grid153.kfki.hu', - 'service': u'MyProxy', - 'tags': {'monitored': '1', 'production': '1', 'scope': 'EGI'}, - 'type': 'SERVICEGROUPS'}, - {'group': u'NGI_HU_SERVICES', - 'group_monitored': u'Y', - 'hostname': u'grid146.kfki.hu', - 'service': u'ngi.ARGUS', - 'tags': {'monitored': '1', 'production': '1', 'scope': 'EGI'}, - 'type': 'SERVICEGROUPS'} - ] - - def __init__(self, connector, gconf, cconf): - self.globalconfig = modules.config.Global(connector, gconf) - self.customerconfig = modules.config.CustomerConf(connector, cconf) - self.globopts = self.globalconfig.parse() - self.customerconfig.parse() - customers = self.customerconfig.get_customers() - self.custname = self.customerconfig.get_custname(customers[0]) - self.jobs = self.customerconfig.get_jobs(customers[0]) - self.jobdir = self.customerconfig.get_fulldir(customers[0], self.jobs[0]) - - -class TopologyXml(unittest.TestCase): - def setUp(self): - self.connset = ConnectorSetup('topology-gocdb-connector.py', - 'tests/global.conf', - 'tests/customer.conf') - for c in ['globalconfig', 'customerconfig', 'globopts', - 'group_endpoints', 'group_endpoints_uid', 'group_groups', 'group_endpoints_feed', - 'group_groups_feed', 'group_groups_servicegroup_filter', - 'group_endpoints_servicegroup_filter', - 'group_endpoints_sites_filter', 'group_groups_sites_filter']: - code = """self.%s = self.connset.%s""" % (c, c) - exec code - - feedjobs = self.customerconfig.get_mapfeedjobs('topology-gocdb-connector.py', - 'GOCDB', - deffeed='https://localhost/gocdbpi/') - feed = feedjobs.keys()[0] - jobcust = feedjobs.values()[0] - scopes = self.customerconfig.get_feedscopes(feed, jobcust) - self.gocdbreader = GOCDBReader(feed, scopes) - self.orig_get_xmldata = self.gocdbreader._get_xmldata - self.gocdbreader._get_xmldata = self.wrap_get_xmldata - - def wrap_get_xmldata(self, scope, pi): - globopts = self.globalconfig.parse() - self.orig_get_xmldata.im_func.func_globals['globopts'] = globopts - self.orig_get_xmldata.im_func.func_globals['input'].connection.func = self.mock_conn - return self.orig_get_xmldata(scope, pi) - - @mock.patch('modules.input.connection') - def testServiceEndpoints(self, mock_conn): - servicelist = dict() - mock_conn.__name__ = 'mock_conn' - mock_conn.return_value = self.group_endpoints_feed - self.mock_conn = mock_conn - self.gocdbreader.getServiceEndpoints(servicelist, '&scope=EGI') - self.gocdbreader.serviceListEGI = servicelist - self.gocdbreader.getGroupOfEndpoints.im_func.func_globals['fetchtype'] = 'SITES' - sge = sorted(self.group_endpoints, key=lambda e: e['service']) - obj_sge = sorted(self.gocdbreader.getGroupOfEndpoints(), - key=lambda e: e['service']) - self.assertEqual(sge, obj_sge) - - @mock.patch('modules.input.connection') - def testUIDServiceEndpoints(self, mock_conn): - servicelist = dict() - mock_conn.__name__ = 'mock_conn' - mock_conn.return_value = self.group_endpoints_feed - self.mock_conn = mock_conn - self.gocdbreader.getServiceEndpoints(servicelist, '&scope=EGI') - self.gocdbreader.serviceListEGI = servicelist - self.gocdbreader.getGroupOfEndpoints.im_func.func_globals['fetchtype'] = 'SITES' - sge = sorted(self.group_endpoints_uid, key=lambda e: e['service']) - obj_sge = sorted(self.gocdbreader.getGroupOfEndpoints(True), - key=lambda e: e['service']) - self.assertEqual(sge, obj_sge) - - @mock.patch('modules.input.connection') - def testSites(self, mock_conn): - siteslist = dict() - mock_conn.__name__ = 'mock_conn' - mock_conn.return_value = self.group_groups_feed - self.mock_conn = mock_conn - self.gocdbreader.getSitesInternal(siteslist, '&scope=EGI') - self.gocdbreader.siteListEGI = siteslist - self.gocdbreader.getGroupOfGroups.im_func.func_globals['fetchtype'] = 'SITES' - sgg = sorted(self.group_groups, key=lambda e: e['subgroup']) - obj_sgg = sorted(self.gocdbreader.getGroupOfGroups(), - key=lambda e: e['subgroup']) - self.assertEqual(sgg, obj_sgg) - - def testTopoFilter(self): - groupfilter = {'Monitored': 'Y', - 'Scope': 'EGI', - 'ServiceGroup': 'SLA_TEST'} - subgroupfilter = {'Monitored': 'Y', - 'Production': 'Y', - 'Scope': 'EGI'} - tf = TopoFilter(self.group_groups_servicegroup_filter, - self.group_endpoints_servicegroup_filter, - groupfilter, - subgroupfilter) - self.assertEqual(tf.gg, [{'group': 'EGI', 'subgroup': u'SLA_TEST', - 'tags': {'monitored': '1', 'scope': 'EGI'}, - 'type': 'PROJECT'}]) - self.assertEqual(tf.ge, [{'group': u'SLA_TEST', - 'group_monitored': 'Y', - 'hostname': 'se01.marie.hellasgrid.gr', - 'service': 'SRM', - 'tags': {'monitored': '1', - 'production': '1', - 'scope': 'EGI'}, - 'type': 'SERVICEGROUPS'}]) - groupfilter = {'Infrastructure': 'Production', - 'Scope': 'EGI', - 'Certification': 'Certified', - 'Site': 'egee.srce.hr'} - subgroupfilter = {'Monitored': 'Y', - 'Production': 'Y', - 'Scope': 'EGI'} - tf = TopoFilter(self.group_groups_sites_filter, - self.group_endpoints_sites_filter, - groupfilter, - subgroupfilter) - self.assertEqual(tf.gg, [{'group': 'NGI_HR', 'subgroup': 'egee.srce.hr', - 'tags': {'certification': 'Certified', - 'infrastructure': 'Production', - 'scope': 'EGI'}, - 'type': 'NGI'}]) - self.assertEqual(tf.ge, [{'group': 'egee.srce.hr', - 'hostname': 'se.srce.egi.cro-ngi.hr', - 'service': 'gLite-APEL', - 'tags': {'monitored': '1', - 'production': '1', - 'scope': 'EGI'}, - 'type': 'SITES'}]) - - -class WeightsJson(unittest.TestCase): - def setUp(self): - self.connset = ConnectorSetup('weights-vapor-connector.py', - 'tests/global.conf', - 'tests/customer.conf') - for c in ['globalconfig', 'customerconfig', 'globopts', 'jobs', - 'jobdir', 'weights', 'weights_feed']: - code = """self.%s = self.connset.%s""" % (c, c) - exec code - - def wrap_get_weights(self, mock_conn): - logger = Logger('weights-vapor-connector.py') - logger.customer = 'EGI' - logger.job = self.jobs[0] - self.orig_get_weights.im_func.func_globals['globopts'] = self.globopts - self.orig_get_weights.im_func.func_globals['input'].connection.func = mock_conn - self.orig_get_weights.im_func.func_globals['logger'] = logger - return self.orig_get_weights() - - @mock.patch('modules.input.connection') - def testJson(self, mock_conn): - feeds = self.customerconfig.get_mapfeedjobs('weights-vapor-connector.py', - deffeed= 'https://operations-portal.egi.eu/vapor/downloadLavoisier/option/json/view/VAPOR_Ngi_Sites_Info') - vapor = VaporReader(feeds.keys()[0]) - datestamp = datetime.datetime.strptime('2017-01-19', '%Y-%m-%d') - self.orig_get_weights = vapor.getWeights - mock_conn.__name__ = 'mock_conn' - mock_conn.return_value = 'Erroneous JSON feed' - vapor.getWeights = self.wrap_get_weights - self.assertEqual(vapor.getWeights(mock_conn), []) - - mock_conn.return_value = self.weights_feed - vapor.getWeights = self.wrap_get_weights - self.assertEqual(vapor.getWeights(mock_conn), self.weights) - - -class DowntimesXml(unittest.TestCase): - def setUp(self): - self.connset = ConnectorSetup('downtimes-gocdb-connector.py', - 'tests/global.conf', - 'tests/customer.conf') - for c in ['globalconfig', 'customerconfig', 'globopts', 'jobs', - 'jobdir', 'downtimes', 'downtimes_feed']: - code = """self.%s = self.connset.%s""" % (c, c) - exec code - - def wrap_get_downtimes(self, start, end, mock_conn): - logger = Logger('downtimes-gocdb-connector.py') - logger.customer = 'EGI' - logger.job = self.jobs[0] - self.orig_get_downtimes.im_func.func_globals['globopts'] = self.globopts - self.orig_get_downtimes.im_func.func_globals['input'].connection.func = mock_conn - self.orig_get_downtimes.im_func.func_globals['logger'] = logger - return self.orig_get_downtimes(start, end) - - @mock.patch('modules.helpers.time.sleep') - @mock.patch('modules.input.connection') - def testRetryConnection(self, mock_conn, mock_sleep): - feeds = self.customerconfig.get_mapfeedjobs('downtimes-gocdb-connector.py', deffeed='https://goc.egi.eu/gocdbpi/') - gocdb = DowntimesGOCDBReader(feeds.keys()[0]) - datestamp = datetime.datetime.strptime('2017-01-19', '%Y-%m-%d') - start = datestamp.replace(hour=0, minute=0, second=0) - end = datestamp.replace(hour=23, minute=59, second=59) - self.orig_get_downtimes = gocdb.getDowntimes - gocdb.getDowntimes = self.wrap_get_downtimes - mock_sleep.return_value = True - mock_conn.__name__ = 'mock_conn' - mock_conn.side_effect = [httplib.HTTPException('Bogus'), - httplib.HTTPException('Bogus'), - httplib.HTTPException('Bogus')] - self.assertEqual(gocdb.getDowntimes(start, end, mock_conn), []) - self.assertEqual(mock_conn.call_count, int(self.globopts['ConnectionRetry'.lower()]) + 1) - self.assertTrue(mock_sleep.called) - self.assertEqual(mock_sleep.call_count, int(self.globopts['ConnectionRetry'.lower()])) - sleepretry = int(self.globopts['ConnectionSleepRetry'.lower()]) - self.assertEqual(mock_sleep.call_args_list, [mock.call(sleepretry), - mock.call(sleepretry), - mock.call(sleepretry)]) - - @mock.patch('modules.input.connection') - def testXml(self, mock_conn): - feeds = self.customerconfig.get_mapfeedjobs('downtimes-gocdb-connector.py', deffeed='https://goc.egi.eu/gocdbpi/') - gocdb = DowntimesGOCDBReader(feeds.keys()[0]) - datestamp = datetime.datetime.strptime('2017-01-19', '%Y-%m-%d') - start = datestamp.replace(hour=0, minute=0, second=0) - end = datestamp.replace(hour=23, minute=59, second=59) - self.orig_get_downtimes = gocdb.getDowntimes - mock_conn.__name__ = 'mock_conn' - mock_conn.return_value = 'Erroneous XML feed' - gocdb.getDowntimes = self.wrap_get_downtimes - self.assertEqual(gocdb.getDowntimes(start, end, mock_conn), []) - mock_conn.return_value = self.downtimes_feed - gocdb.getDowntimes = self.wrap_get_downtimes - self.assertEqual(gocdb.getDowntimes(start, end, mock_conn), self.downtimes) - - @mock.patch('bin.downtimes_gocdb_connector.sys') - @mock.patch('modules.output.write_state') - @mock.patch('bin.downtimes_gocdb_connector.CustomerConf', autospec=True) - @mock.patch('bin.downtimes_gocdb_connector.argparse.ArgumentParser.parse_args') - @mock.patch('bin.downtimes_gocdb_connector.Global') - @mock.patch('bin.downtimes_gocdb_connector.GOCDBReader') - def testStateFile(self, gocdbreader, glob, parse_args, customerconf, write_state, mock_sys): - argmock = mock.Mock() - argmock.date = ['2017-01-19'] - argmock.gloconf = ['tests/global.conf'] - argmock.custconf = ['tests/customer.conf'] - parse_args.return_value = argmock - customerconf.get_mapfeedjobs.return_value = self.customerconfig.get_mapfeedjobs('downtimes-gocdb-connector.py', - deffeed='https://goc.egi.eu/gocdbpi/') - customerconf.get_fullstatedir.side_effect = ['/var/lib/argo-connectors/states//EGI/EGI_Critical', '/var/lib/argo-connectors/states//EGI/EGI_Cloudmon', '/var/lib/argo-connectors/states//EGI/EGI_Critical', '/var/lib/argo-connectors/states//EGI/EGI_Cloudmon'] - self.globopts['generalwriteavro'] = 'False' - self.globopts['generalpublishams'] = 'False' - mock_sys.argv = ['downtimes-gocdb-connector.py'] - downtimes_main.func_globals['output'].write_state = write_state - customerconf.side_effect = [customerconf, customerconf] - gocdbreader.side_effect = [gocdbreader, gocdbreader] - glob.side_effect = [glob, glob] - glob.is_complete.return_value = (True, []) - - gocdbreader.state = True - gocdbreader.getDowntimes.return_value = self.downtimes - glob.parse.return_value = self.globopts - downtimes_main() - self.assertTrue(write_state.called) - self.assertEqual(write_state.call_count, len(self.jobs)) - for call in write_state.call_args_list: - self.assertTrue(gocdbreader.state in call[0]) - self.assertTrue('2017_01_19' in call[0]) - - gocdbreader.state = False - gocdbreader.getDowntimes.return_value = [] - downtimes_main() - self.assertTrue(write_state.called) - self.assertEqual(write_state.call_count, 2*len(self.jobs)) - for call in write_state.call_args_list[2:]: - self.assertTrue(gocdbreader.state in call[0]) - self.assertTrue('2017_01_19' in call[0]) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_output.py b/tests/test_output.py deleted file mode 100644 index 6133e277..00000000 --- a/tests/test_output.py +++ /dev/null @@ -1,604 +0,0 @@ -import datetime -import httplib -import json -import mock -import modules.config -import unittest2 as unittest - -from httmock import urlmatch, HTTMock, response - -from bin.topology_gocdb_connector import logger -from modules import output -from modules import input -from modules.helpers import datestamp, filename_date, retry - - -class ConnectorSetup(object): - poem = [{'metric': u'org.nordugrid.ARC-CE-ARIS', - 'profile': u'ch.cern.sam.ARGO_MON_CRITICAL', - 'service': u'ARC-CE', - 'tags': {'fqan': u'', 'vo': 'ops'}}, - {'metric': u'org.nordugrid.ARC-CE-IGTF', - 'profile': u'ch.cern.sam.ARGO_MON_CRITICAL', - 'service': u'ARC-CE', - 'tags': {'fqan': u'', 'vo': 'ops'}}, - {'metric': u'org.nordugrid.ARC-CE-result', - 'profile': u'ch.cern.sam.ARGO_MON_CRITICAL', - 'service': u'ARC-CE', - 'tags': {'fqan': u'', 'vo': 'ops'}}] - - downtimes = [{'end_time': '2017-01-19T23:59:00Z', - 'hostname': u'nagios.c4.csir.co.za', - 'service': u'ngi.SAM', - 'start_time': '2017-01-19T00:00:00Z'}, - {'end_time': '2017-01-19T23:59:00Z', - 'hostname': u'ce1.grid.lebedev.ru', - 'service': u'CE', 'start_time': - '2017-01-19T00:00:00Z'}, - {'end_time': '2017-01-19T23:59:00Z', - 'hostname': u'ce1.grid.lebedev.ru', - 'service': u'APEL', - 'start_time': '2017-01-19T00:00:00Z'}] - - weights = [{'site': u'FZK-LCG2', 'type': 'hepspec', 'weight': u'0'}, - {'site': u'IN2P3-IRES', 'type': 'hepspec', 'weight': u'13'}, - {'site': u'GRIF-LLR', 'type': 'hepspec', 'weight': u'0'}] - - group_groups = [{'group': u'AfricaArabia', 'subgroup': u'MA-01-CNRST', - 'tags': {'certification': u'Certified', - 'infrastructure': u'Production', - 'scope': 'EGI'}, - 'type': 'NGI'}, - {'group': u'AfricaArabia', 'subgroup': u'MA-04-CNRST-ATLAS', - 'tags': {'certification': u'Certified', - 'infrastructure': u'Production', - 'scope': 'EGI'}, - 'type': 'NGI'}, - {'group': u'AfricaArabia', 'subgroup': u'ZA-UCT-ICTS', - 'tags': {'certification': u'Suspended', - 'infrastructure': u'Production', - 'scope': 'EGI'}, - 'type': 'NGI'}] - - group_endpoints = [{'group': u'100IT', - 'hostname': u'occi-api.100percentit.com', - 'service': u'eu.egi.cloud.vm-management.occi', - 'tags': {'monitored': '1', - 'production': '1', - 'scope': 'EGI'}, - 'type': 'SITES'}, - {'group': u'100IT', - 'hostname': u'egi-cloud-accounting.100percentit.com', - 'service': u'eu.egi.cloud.accounting', - 'tags': {'monitored': '1', - 'production': '1', - 'scope': 'EGI'}, - 'type': 'SITES'}, - {'group': u'100IT', - 'hostname': u'occi-api.100percentit.com', - 'service': u'eu.egi.cloud.information.bdii', - 'tags': {'monitored': '1', - 'production': '1', - 'scope': 'EGI'}, - 'type': 'SITES'}] - - def __init__(self, connector, gconf, cconf): - self.globalconfig = modules.config.Global(connector, gconf) - self.customerconfig = modules.config.CustomerConf(connector, cconf) - self.globopts = self.globalconfig.parse() - self.customerconfig.parse() - customers = self.customerconfig.get_customers() - self.jobs = self.customerconfig.get_jobs(customers[0]) - self.jobdir = self.customerconfig.get_fulldir(customers[0], self.jobs[0]) - - -class TopologyAvro(unittest.TestCase): - def setUp(self): - self.connset = ConnectorSetup('topology-gocdb-connector.py', - 'tests/global.conf', - 'tests/customer.conf') - for c in ['globalconfig', 'customerconfig', 'globopts', 'jobs', - 'jobdir', 'group_groups', 'group_endpoints']: - code = """self.%s = self.connset.%s""" % (c, c) - exec code - - @mock.patch('modules.output.load_schema') - @mock.patch('modules.output.open') - def testGroupGroups(self, mock_open, mock_lschema): - mock_avrofile = mock.create_autospec(output.DataFileWriter) - filename = filename_date(logger, self.globopts['OutputTopologyGroupOfGroups'.lower()], self.jobdir) - m = output.AvroWriter(self.globopts['AvroSchemasTopologyGroupOfGroups'.lower()], filename) - m.datawrite = mock_avrofile - m.write(self.group_groups) - mock_open.assert_called_with(filename, 'w+') - mock_lschema.assert_called_with(self.globopts['AvroSchemasTopologyGroupOfGroups'.lower()]) - self.assertTrue(mock_avrofile.append.called) - self.assertEqual(mock_avrofile.append.call_count, 3) - self.assertEqual(mock_avrofile.append.mock_calls.index(mock.call(self.group_groups[0])), 0) - self.assertEqual(mock_avrofile.append.mock_calls.index(mock.call(self.group_groups[1])), 1) - self.assertEqual(mock_avrofile.append.mock_calls.index(mock.call(self.group_groups[2])), 2) - - @mock.patch('modules.output.load_schema') - @mock.patch('modules.output.open') - def testGroupEndpoints(self, mock_open, mock_lschema): - mock_avrofile = mock.create_autospec(output.DataFileWriter) - filename = filename_date(logger, self.globopts['OutputTopologyGroupOfEndpoints'.lower()], self.jobdir) - m = output.AvroWriter(self.globopts['AvroSchemasTopologyGroupOfEndpoints'.lower()], filename) - m.datawrite = mock_avrofile - m.write(self.group_endpoints) - mock_open.assert_called_with(filename, 'w+') - mock_lschema.assert_called_with(self.globopts['AvroSchemasTopologyGroupOfEndpoints'.lower()]) - self.assertTrue(mock_avrofile.append.called) - self.assertEqual(mock_avrofile.append.call_count, 3) - self.assertEqual(mock_avrofile.append.mock_calls.index(mock.call(self.group_endpoints[0])), 0) - self.assertEqual(mock_avrofile.append.mock_calls.index(mock.call(self.group_endpoints[1])), 1) - self.assertEqual(mock_avrofile.append.mock_calls.index(mock.call(self.group_endpoints[2])), 2) - - -class DowntimesAvro(unittest.TestCase): - def setUp(self): - self.connset = ConnectorSetup('downtimes-gocdb-connector.py', - 'tests/global.conf', - 'tests/customer.conf') - for c in ['globalconfig', 'customerconfig', 'globopts', 'jobs', - 'jobdir', 'downtimes']: - code = """self.%s = self.connset.%s""" % (c, c) - exec code - - @mock.patch('modules.output.load_schema') - @mock.patch('modules.output.open') - def testDowntimes(self, mock_open, mock_lschema): - mock_avrofile = mock.create_autospec(output.DataFileWriter) - filename = filename_date(logger, self.globopts['OutputDowntimes'.lower()], self.jobdir) - m = output.AvroWriter(self.globopts['AvroSchemasDowntimes'.lower()], filename) - m.datawrite = mock_avrofile - m.write(self.downtimes) - mock_open.assert_called_with(filename, 'w+') - mock_lschema.assert_called_with(self.globopts['AvroSchemasDowntimes'.lower()]) - self.assertTrue(mock_avrofile.append.called) - self.assertEqual(mock_avrofile.append.call_count, 3) - self.assertEqual(mock_avrofile.append.mock_calls.index(mock.call(self.downtimes[0])), 0) - self.assertEqual(mock_avrofile.append.mock_calls.index(mock.call(self.downtimes[1])), 1) - self.assertEqual(mock_avrofile.append.mock_calls.index(mock.call(self.downtimes[2])), 2) - - -class WeightsAvro(unittest.TestCase): - def setUp(self): - self.connset = ConnectorSetup('weights-vapor-connector.py', - 'tests/global.conf', - 'tests/customer.conf') - for c in ['globalconfig', 'customerconfig', 'globopts', 'jobs', - 'jobdir', 'weights']: - code = """self.%s = self.connset.%s""" % (c, c) - exec code - - @mock.patch('modules.output.load_schema') - @mock.patch('modules.output.open') - def testWeights(self, mock_open, mock_lschema): - mock_avrofile = mock.create_autospec(output.DataFileWriter) - filename = filename_date(logger, self.globopts['OutputWeights'.lower()], self.jobdir) - m = output.AvroWriter(self.globopts['AvroSchemasWeights'.lower()], filename) - m.datawrite = mock_avrofile - m.write(self.weights) - mock_open.assert_called_with(filename, 'w+') - mock_lschema.assert_called_with(self.globopts['AvroSchemasWeights'.lower()]) - self.assertTrue(mock_avrofile.append.called) - self.assertEqual(mock_avrofile.append.call_count, 3) - self.assertEqual(mock_avrofile.append.mock_calls.index(mock.call(self.weights[0])), 0) - self.assertEqual(mock_avrofile.append.mock_calls.index(mock.call(self.weights[1])), 1) - self.assertEqual(mock_avrofile.append.mock_calls.index(mock.call(self.weights[2])), 2) - - -class WeightsAvro(unittest.TestCase): - def setUp(self): - self.connset = ConnectorSetup('weights-vapor-connector.py', - 'tests/global.conf', - 'tests/customer.conf') - for c in ['globalconfig', 'customerconfig', 'globopts', 'jobs', - 'jobdir', 'poem']: - code = """self.%s = self.connset.%s""" % (c, c) - exec code - - @mock.patch('modules.output.load_schema') - @mock.patch('modules.output.open') - def testWeights(self, mock_open, mock_lschema): - mock_avrofile = mock.create_autospec(output.DataFileWriter) - filename = filename_date(logger, self.globopts['OutputWeights'.lower()], self.jobdir) - m = output.AvroWriter(self.globopts['AvroSchemasWeights'.lower()], filename) - m.datawrite = mock_avrofile - m.write(self.poem) - mock_open.assert_called_with(filename, 'w+') - mock_lschema.assert_called_with(self.globopts['AvroSchemasWeights'.lower()]) - self.assertTrue(mock_avrofile.append.called) - self.assertEqual(mock_avrofile.append.call_count, 3) - self.assertEqual(mock_avrofile.append.mock_calls.index(mock.call(self.poem[0])), 0) - self.assertEqual(mock_avrofile.append.mock_calls.index(mock.call(self.poem[1])), 1) - self.assertEqual(mock_avrofile.append.mock_calls.index(mock.call(self.poem[2])), 2) - - -class MetricProfileAms(unittest.TestCase): - publish_topic_urlmatch = dict(netloc='localhost', - path='/v1/projects/EGI/topics/TOPIC:publish', - method='POST') - - def setUp(self): - self.connset = ConnectorSetup('metricprofile-webapi-connector.py', - 'tests/global.conf', - 'tests/customer.conf') - for c in ['globalconfig', 'customerconfig', 'globopts', 'jobs', - 'jobdir', 'poem']: - code = """self.%s = self.connset.%s""" % (c, c) - exec code - - self.globopts['amspacksinglemsg'] = 'False' - self.amspublish = output.AmsPublish(self.globopts['amshost'], - self.globopts['amsproject'], - self.globopts['amstoken'], - self.globopts['amstopic'], - self.customerconfig.get_jobdir(self.jobs[0]), - self.globopts['amsbulk'], - self.globopts['amspacksinglemsg'], - logger, - int(self.globopts['connectionretry']), - int(self.globopts['connectiontimeout']), - int(self.globopts['connectionsleepretry'])) - - self.globopts['amspacksinglemsg'] = 'True' - self.amspublish_pack = output.AmsPublish(self.globopts['amshost'], - self.globopts['amsproject'], - self.globopts['amstoken'], - self.globopts['amstopic'], - self.customerconfig.get_jobdir(self.jobs[0]), - self.globopts['amsbulk'], - self.globopts['amspacksinglemsg'], - logger, - int(self.globopts['connectionretry']), - int(self.globopts['connectiontimeout'])) - def testMetricProfile(self): - @urlmatch(**self.publish_topic_urlmatch) - def publish_bulk_mock(url, request): - assert url.path == "/v1/projects/EGI/topics/TOPIC:publish" - # Check request produced by ams client - req_body = json.loads(request.body) - self.assertEqual(req_body["messages"][0]["data"], "OmNoLmNlcm4uc2FtLkFSR09fTU9OX0NSSVRJQ0FMDEFSQy1DRTJvcmcubm9yZHVncmlkLkFSQy1DRS1BUklTAgQIZnFhbgAEdm8Gb3BzAA==") - self.assertEqual(req_body["messages"][0]["attributes"]["type"], "poem") - self.assertEqual(req_body["messages"][0]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][0]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - self.assertEqual(req_body["messages"][1]["data"], "OmNoLmNlcm4uc2FtLkFSR09fTU9OX0NSSVRJQ0FMDEFSQy1DRTJvcmcubm9yZHVncmlkLkFSQy1DRS1JR1RGAgQIZnFhbgAEdm8Gb3BzAA==") - self.assertEqual(req_body["messages"][1]["attributes"]["type"], "poem") - self.assertEqual(req_body["messages"][1]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][1]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - self.assertEqual(req_body["messages"][2]["data"], "OmNoLmNlcm4uc2FtLkFSR09fTU9OX0NSSVRJQ0FMDEFSQy1DRTZvcmcubm9yZHVncmlkLkFSQy1DRS1yZXN1bHQCBAhmcWFuAAR2bwZvcHMA") - self.assertEqual(req_body["messages"][2]["attributes"]["type"], "poem") - self.assertEqual(req_body["messages"][2]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][2]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - return '{"msgIds": ["1", "2", "3"]}' - - with HTTMock(publish_bulk_mock): - ret = self.amspublish.send(self.globopts['AvroSchemasMetricProfile'.lower()], - 'poem', datestamp().replace('_', '-'), - self.poem) - self.assertTrue(ret) - - @urlmatch(**self.publish_topic_urlmatch) - def publish_pack_mock(url, request): - assert url.path == "/v1/projects/EGI/topics/TOPIC:publish" - # Check request produced by ams client - req_body = json.loads(request.body) - self.assertEqual(req_body["messages"][0]["data"], "OmNoLmNlcm4uc2FtLkFSR09fTU9OX0NSSVRJQ0FMDEFSQy1DRTJvcmcubm9yZHVncmlkLkFSQy1DRS1BUklTAgQIZnFhbgAEdm8Gb3BzADpjaC5jZXJuLnNhbS5BUkdPX01PTl9DUklUSUNBTAxBUkMtQ0Uyb3JnLm5vcmR1Z3JpZC5BUkMtQ0UtSUdURgIECGZxYW4ABHZvBm9wcwA6Y2guY2Vybi5zYW0uQVJHT19NT05fQ1JJVElDQUwMQVJDLUNFNm9yZy5ub3JkdWdyaWQuQVJDLUNFLXJlc3VsdAIECGZxYW4ABHZvBm9wcwA=") - self.assertEqual(req_body["messages"][0]["attributes"]["type"], "poem") - self.assertEqual(req_body["messages"][0]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][0]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - return '{"msgIds": ["1"]}' - - - with HTTMock(publish_pack_mock): - ret = self.amspublish_pack.send(self.globopts['AvroSchemasMetricProfile'.lower()], - 'poem', datestamp().replace('_', '-'), - self.poem) - self.assertTrue(ret) - -class WeightsAms(unittest.TestCase): - publish_topic_urlmatch = dict(netloc='localhost', - path='/v1/projects/EGI/topics/TOPIC:publish', - method='POST') - - def setUp(self): - self.connset = ConnectorSetup('weights-vapor-connector.py', - 'tests/global.conf', - 'tests/customer.conf') - for c in ['globalconfig', 'customerconfig', 'globopts', 'jobs', - 'jobdir', 'weights']: - code = """self.%s = self.connset.%s""" % (c, c) - exec code - - self.globopts['amspacksinglemsg'] = 'False' - self.amspublish = output.AmsPublish(self.globopts['amshost'], - self.globopts['amsproject'], - self.globopts['amstoken'], - self.globopts['amstopic'], - self.customerconfig.get_jobdir(self.jobs[0]), - self.globopts['amsbulk'], - self.globopts['amspacksinglemsg'], - logger, - int(self.globopts['connectionretry']), - int(self.globopts['connectiontimeout'])) - - self.globopts['amspacksinglemsg'] = 'True' - self.amspublish_pack = output.AmsPublish(self.globopts['amshost'], - self.globopts['amsproject'], - self.globopts['amstoken'], - self.globopts['amstopic'], - self.customerconfig.get_jobdir(self.jobs[0]), - self.globopts['amsbulk'], - self.globopts['amspacksinglemsg'], - logger, - int(self.globopts['connectionretry']), - int(self.globopts['connectiontimeout'])) - - def testWeights(self): - @urlmatch(**self.publish_topic_urlmatch) - def publish_bulk_mock(url, request): - assert url.path == "/v1/projects/EGI/topics/TOPIC:publish" - # Check request produced by ams client - req_body = json.loads(request.body) - self.assertEqual(req_body["messages"][0]["data"], "DmhlcHNwZWMQRlpLLUxDRzICMA==") - self.assertEqual(req_body["messages"][0]["attributes"]["type"], "weights") - self.assertEqual(req_body["messages"][0]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][0]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - self.assertEqual(req_body["messages"][1]["data"], "DmhlcHNwZWMUSU4yUDMtSVJFUwQxMw==") - self.assertEqual(req_body["messages"][1]["attributes"]["type"], "weights") - self.assertEqual(req_body["messages"][1]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][1]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - self.assertEqual(req_body["messages"][2]["data"], "DmhlcHNwZWMQR1JJRi1MTFICMA==") - self.assertEqual(req_body["messages"][2]["attributes"]["type"], "weights") - self.assertEqual(req_body["messages"][2]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][2]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - return '{"msgIds": ["1", "2", "3"]}' - - - with HTTMock(publish_bulk_mock): - ret = self.amspublish.send(self.globopts['AvroSchemasWeights'.lower()], - 'weights', datestamp().replace('_', '-'), - self.weights) - self.assertTrue(ret) - - @urlmatch(**self.publish_topic_urlmatch) - def publish_pack_mock(url, request): - assert url.path == "/v1/projects/EGI/topics/TOPIC:publish" - # Check request produced by ams client - req_body = json.loads(request.body) - self.assertEqual(req_body["messages"][0]["data"], "DmhlcHNwZWMQRlpLLUxDRzICMA5oZXBzcGVjFElOMlAzLUlSRVMEMTMOaGVwc3BlYxBHUklGLUxMUgIw") - self.assertEqual(req_body["messages"][0]["attributes"]["type"], "weights") - self.assertEqual(req_body["messages"][0]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][0]["attributes"]["partition_date"], datestamp().replace('_', '-')) - return '{"msgIds": ["1"]}' - - with HTTMock(publish_pack_mock): - ret = self.amspublish_pack.send(self.globopts['AvroSchemasWeights'.lower()], - 'weights', datestamp().replace('_', '-'), - self.weights) - self.assertTrue(ret) - -class DowntimesAms(unittest.TestCase): - publish_topic_urlmatch = dict(netloc='localhost', - path='/v1/projects/EGI/topics/TOPIC:publish', - method='POST') - - def setUp(self): - self.connset = ConnectorSetup('downtimes-gocdb-connector.py', - 'tests/global.conf', - 'tests/customer.conf') - for c in ['globalconfig', 'customerconfig', 'globopts', 'jobs', - 'jobdir', 'downtimes']: - code = """self.%s = self.connset.%s""" % (c, c) - exec code - - self.globopts['amspacksinglemsg'] = 'False' - self.amspublish = output.AmsPublish(self.globopts['amshost'], - self.globopts['amsproject'], - self.globopts['amstoken'], - self.globopts['amstopic'], - self.customerconfig.get_jobdir(self.jobs[0]), - self.globopts['amsbulk'], - self.globopts['amspacksinglemsg'], - logger, - int(self.globopts['connectionretry']), - int(self.globopts['connectiontimeout'])) - - self.globopts['amspacksinglemsg'] = 'True' - self.amspublish_pack = output.AmsPublish(self.globopts['amshost'], - self.globopts['amsproject'], - self.globopts['amstoken'], - self.globopts['amstopic'], - self.customerconfig.get_jobdir(self.jobs[0]), - self.globopts['amsbulk'], - self.globopts['amspacksinglemsg'], - logger, - int(self.globopts['connectionretry']), - int(self.globopts['connectiontimeout'])) - - def testDowntimes(self): - @urlmatch(**self.publish_topic_urlmatch) - def publish_bulk_mock(url, request): - assert url.path == "/v1/projects/EGI/topics/TOPIC:publish" - # Check request produced by ams client - req_body = json.loads(request.body) - self.assertEqual(req_body["messages"][0]["data"], "KG5hZ2lvcy5jNC5jc2lyLmNvLnphDm5naS5TQU0oMjAxNy0wMS0xOVQwMDowMDowMFooMjAxNy0wMS0xOVQyMzo1OTowMFo=") - self.assertEqual(req_body["messages"][0]["attributes"]["type"], "downtimes") - self.assertEqual(req_body["messages"][0]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][0]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - self.assertEqual(req_body["messages"][1]["data"], "JmNlMS5ncmlkLmxlYmVkZXYucnUEQ0UoMjAxNy0wMS0xOVQwMDowMDowMFooMjAxNy0wMS0xOVQyMzo1OTowMFo=") - self.assertEqual(req_body["messages"][1]["attributes"]["type"], "downtimes") - self.assertEqual(req_body["messages"][1]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][1]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - self.assertEqual(req_body["messages"][2]["data"], "JmNlMS5ncmlkLmxlYmVkZXYucnUIQVBFTCgyMDE3LTAxLTE5VDAwOjAwOjAwWigyMDE3LTAxLTE5VDIzOjU5OjAwWg==") - self.assertEqual(req_body["messages"][2]["attributes"]["type"], "downtimes") - self.assertEqual(req_body["messages"][2]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][2]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - return '{"msgIds": ["1", "2", "3"]}' - - with HTTMock(publish_bulk_mock): - ret = self.amspublish.send(self.globopts['AvroSchemasDowntimes'.lower()], - 'downtimes', datestamp().replace('_', '-'), self.downtimes) - self.assertTrue(ret) - - @urlmatch(**self.publish_topic_urlmatch) - def publish_pack_mock(url, request): - assert url.path == "/v1/projects/EGI/topics/TOPIC:publish" - req_body = json.loads(request.body) - self.assertEqual(req_body["messages"][0]["data"], u'KG5hZ2lvcy5jNC5jc2lyLmNvLnphDm5naS5TQU0oMjAxNy0wMS0xOVQwMDowMDowMFooMjAxNy0wMS0xOVQyMzo1OTowMFomY2UxLmdyaWQubGViZWRldi5ydQRDRSgyMDE3LTAxLTE5VDAwOjAwOjAwWigyMDE3LTAxLTE5VDIzOjU5OjAwWiZjZTEuZ3JpZC5sZWJlZGV2LnJ1CEFQRUwoMjAxNy0wMS0xOVQwMDowMDowMFooMjAxNy0wMS0xOVQyMzo1OTowMFo=') - self.assertEqual(req_body["messages"][0]["attributes"]["type"], "downtimes") - self.assertEqual(req_body["messages"][0]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][0]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - return '{"msgIds": ["1"]}' - - with HTTMock(publish_pack_mock): - ret = self.amspublish_pack.send(self.globopts['AvroSchemasDowntimes'.lower()], - 'downtimes', datestamp().replace('_', '-'), - self.downtimes) - self.assertTrue(ret) - - -class TopologyAms(unittest.TestCase): - publish_topic_urlmatch = dict(netloc='localhost', - path='/v1/projects/EGI/topics/TOPIC:publish', - method='POST') - - def setUp(self): - self.connset = ConnectorSetup('topology-gocdb-connector.py', - 'tests/global.conf', - 'tests/customer.conf') - for c in ['globalconfig', 'customerconfig', 'globopts', 'jobs', - 'jobdir', 'group_groups', 'group_endpoints']: - code = """self.%s = self.connset.%s""" % (c, c) - exec code - - self.globopts['amspacksinglemsg'] = 'False' - self.amspublish = output.AmsPublish(self.globopts['amshost'], - self.globopts['amsproject'], - self.globopts['amstoken'], - self.globopts['amstopic'], - self.customerconfig.get_jobdir(self.jobs[0]), - self.globopts['amsbulk'], - self.globopts['amspacksinglemsg'], - logger, - int(self.globopts['connectionretry']), - int(self.globopts['connectiontimeout'])) - - self.globopts['amspacksinglemsg'] = 'True' - self.amspublish_pack = output.AmsPublish(self.globopts['amshost'], - self.globopts['amsproject'], - self.globopts['amstoken'], - self.globopts['amstopic'], - self.customerconfig.get_jobdir(self.jobs[0]), - self.globopts['amsbulk'], - self.globopts['amspacksinglemsg'], - logger, - int(self.globopts['connectionretry']), - int(self.globopts['connectiontimeout'])) - - def testGroupGroups(self): - @urlmatch(**self.publish_topic_urlmatch) - def publish_bulk_mock(url, request): - assert url.path == "/v1/projects/EGI/topics/TOPIC:publish" - # Check request produced by ams client - req_body = json.loads(request.body) - self.assertEqual(req_body["messages"][0]["data"], "Bk5HSRhBZnJpY2FBcmFiaWEWTUEtMDEtQ05SU1QCBgpzY29wZQZFR0kcaW5mcmFzdHJ1Y3R1cmUUUHJvZHVjdGlvbhpjZXJ0aWZpY2F0aW9uEkNlcnRpZmllZAA=") - self.assertEqual(req_body["messages"][0]["attributes"]["type"], "group_groups") - self.assertEqual(req_body["messages"][0]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][0]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - self.assertEqual(req_body["messages"][1]["data"], "Bk5HSRhBZnJpY2FBcmFiaWEiTUEtMDQtQ05SU1QtQVRMQVMCBgpzY29wZQZFR0kcaW5mcmFzdHJ1Y3R1cmUUUHJvZHVjdGlvbhpjZXJ0aWZpY2F0aW9uEkNlcnRpZmllZAA=") - self.assertEqual(req_body["messages"][1]["attributes"]["type"], "group_groups") - self.assertEqual(req_body["messages"][1]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][1]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - self.assertEqual(req_body["messages"][2]["data"], "Bk5HSRhBZnJpY2FBcmFiaWEWWkEtVUNULUlDVFMCBgpzY29wZQZFR0kcaW5mcmFzdHJ1Y3R1cmUUUHJvZHVjdGlvbhpjZXJ0aWZpY2F0aW9uElN1c3BlbmRlZAA=") - self.assertEqual(req_body["messages"][2]["attributes"]["type"], "group_groups") - self.assertEqual(req_body["messages"][2]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][2]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - return '{"msgIds": ["1", "2", "3"]}' - - - with HTTMock(publish_bulk_mock): - ret = self.amspublish.send(self.globopts['AvroSchemasTopologyGroupOfGroups'.lower()], - 'group_groups', datestamp().replace('_', '-'), self.group_groups) - self.assertTrue(ret) - - @urlmatch(**self.publish_topic_urlmatch) - def publish_pack_mock(url, request): - assert url.path == "/v1/projects/EGI/topics/TOPIC:publish" - # Check request produced by ams client - req_body = json.loads(request.body) - self.assertEqual(req_body["messages"][0]["data"], "Bk5HSRhBZnJpY2FBcmFiaWEWTUEtMDEtQ05SU1QCBgpzY29wZQZFR0kcaW5mcmFzdHJ1Y3R1cmUUUHJvZHVjdGlvbhpjZXJ0aWZpY2F0aW9uEkNlcnRpZmllZAAGTkdJGEFmcmljYUFyYWJpYSJNQS0wNC1DTlJTVC1BVExBUwIGCnNjb3BlBkVHSRxpbmZyYXN0cnVjdHVyZRRQcm9kdWN0aW9uGmNlcnRpZmljYXRpb24SQ2VydGlmaWVkAAZOR0kYQWZyaWNhQXJhYmlhFlpBLVVDVC1JQ1RTAgYKc2NvcGUGRUdJHGluZnJhc3RydWN0dXJlFFByb2R1Y3Rpb24aY2VydGlmaWNhdGlvbhJTdXNwZW5kZWQA") - self.assertEqual(req_body["messages"][0]["attributes"]["type"], "group_groups") - self.assertEqual(req_body["messages"][0]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][0]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - return '{"msgIds": ["1"]}' - - - with HTTMock(publish_pack_mock): - ret = self.amspublish_pack.send(self.globopts['AvroSchemasTopologyGroupOfGroups'.lower()], - 'group_groups', datestamp().replace('_', '-'), self.group_groups) - self.assertTrue(ret) - - def testGroupEndpoints(self): - @urlmatch(**self.publish_topic_urlmatch) - def publish_bulk_mock(url, request): - assert url.path == "/v1/projects/EGI/topics/TOPIC:publish" - # Check request produced by ams client - req_body = json.loads(request.body) - self.assertEqual(req_body["messages"][0]["data"], "ClNJVEVTCjEwMElUPmV1LmVnaS5jbG91ZC52bS1tYW5hZ2VtZW50Lm9jY2kyb2NjaS1hcGkuMTAwcGVyY2VudGl0LmNvbQIGCnNjb3BlBkVHSRRwcm9kdWN0aW9uAjESbW9uaXRvcmVkAjEA") - self.assertEqual(req_body["messages"][0]["attributes"]["type"], "group_endpoints") - self.assertEqual(req_body["messages"][0]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][0]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - self.assertEqual(req_body["messages"][1]["data"], "ClNJVEVTCjEwMElULmV1LmVnaS5jbG91ZC5hY2NvdW50aW5nSmVnaS1jbG91ZC1hY2NvdW50aW5nLjEwMHBlcmNlbnRpdC5jb20CBgpzY29wZQZFR0kUcHJvZHVjdGlvbgIxEm1vbml0b3JlZAIxAA==") - self.assertEqual(req_body["messages"][1]["attributes"]["type"], "group_endpoints") - self.assertEqual(req_body["messages"][1]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][1]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - self.assertEqual(req_body["messages"][2]["data"], "ClNJVEVTCjEwMElUOmV1LmVnaS5jbG91ZC5pbmZvcm1hdGlvbi5iZGlpMm9jY2ktYXBpLjEwMHBlcmNlbnRpdC5jb20CBgpzY29wZQZFR0kUcHJvZHVjdGlvbgIxEm1vbml0b3JlZAIxAA==") - self.assertEqual(req_body["messages"][2]["attributes"]["type"], "group_endpoints") - self.assertEqual(req_body["messages"][2]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][2]["attributes"]["partition_date"], datestamp().replace('_', '-')) - - return '{"msgIds": ["1", "2", "3"]}' - - - with HTTMock(publish_bulk_mock): - ret = self.amspublish.send(self.globopts['AvroSchemasTopologyGroupOfEndpoints'.lower()], - 'group_endpoints', datestamp().replace('_', '-'), self.group_endpoints) - self.assertTrue(ret) - - @urlmatch(**self.publish_topic_urlmatch) - def publish_pack_mock(url, request): - assert url.path == "/v1/projects/EGI/topics/TOPIC:publish" - # Check request produced by ams client - req_body = json.loads(request.body) - self.assertEqual(req_body["messages"][0]["data"], "ClNJVEVTCjEwMElUPmV1LmVnaS5jbG91ZC52bS1tYW5hZ2VtZW50Lm9jY2kyb2NjaS1hcGkuMTAwcGVyY2VudGl0LmNvbQIGCnNjb3BlBkVHSRRwcm9kdWN0aW9uAjESbW9uaXRvcmVkAjEAClNJVEVTCjEwMElULmV1LmVnaS5jbG91ZC5hY2NvdW50aW5nSmVnaS1jbG91ZC1hY2NvdW50aW5nLjEwMHBlcmNlbnRpdC5jb20CBgpzY29wZQZFR0kUcHJvZHVjdGlvbgIxEm1vbml0b3JlZAIxAApTSVRFUwoxMDBJVDpldS5lZ2kuY2xvdWQuaW5mb3JtYXRpb24uYmRpaTJvY2NpLWFwaS4xMDBwZXJjZW50aXQuY29tAgYKc2NvcGUGRUdJFHByb2R1Y3Rpb24CMRJtb25pdG9yZWQCMQA=") - self.assertEqual(req_body["messages"][0]["attributes"]["type"], "group_endpoints") - self.assertEqual(req_body["messages"][0]["attributes"]["report"], "EGI_Critical") - self.assertEqual(req_body["messages"][0]["attributes"]["partition_date"], datestamp().replace('_', '-')) - return '{"msgIds": ["1"]}' - - - with HTTMock(publish_pack_mock): - ret = self.amspublish_pack.send(self.globopts['AvroSchemasTopologyGroupOfEndpoints'.lower()], - 'group_endpoints', datestamp().replace('_', '-'), self.group_endpoints) - self.assertTrue(ret) - diff --git a/tests/test_retry.py b/tests/test_retry.py new file mode 100644 index 00000000..cd317943 --- /dev/null +++ b/tests/test_retry.py @@ -0,0 +1,159 @@ +import unittest +import mock +import asyncio + +from aiohttp import client_exceptions +from aiohttp import http_exceptions + +from argo_egi_connectors.io.http import SessionWithRetry +from argo_egi_connectors.log import Logger +from argo_egi_connectors.exceptions import ConnectorHttpError + +logger = Logger('test_topofeed.py') +CUSTOMER_NAME = 'CUSTOMERFOO' + + +class async_test(object): + """ + Decorator to create asyncio context for asyncio methods or functions. + """ + def __init__(self, test_method): + self.test_method = test_method + + def __call__(self, *args, **kwargs): + test_obj = args[0] + test_obj.loop.run_until_complete(self.test_method(*args, **kwargs)) + + +class mockHttpGetEmpty(mock.AsyncMock): + async def __aenter__(self, *args, **kwargs): + mock_obj = mock.AsyncMock() + mock_obj.text.return_value = '' + return mock_obj + async def __aexit__(self, *args, **kwargs): + pass + + +class mockConnectionProblem(mock.AsyncMock): + async def __aenter__(self, *args, **kwargs): + mock_obj = mock.AsyncMock() + mock_oserror = mock.create_autospec(OSError) + mock_obj.text.side_effect = client_exceptions.ClientConnectorError('mocked key', mock_oserror) + return mock_obj + async def __aexit__(self, *args, **kwargs): + pass + + +class mockProtocolProblem(mock.AsyncMock): + async def __aenter__(self, *args, **kwargs): + mock_obj = mock.AsyncMock() + mock_obj.text.side_effect = http_exceptions.HttpBadRequest('mocked bad HTTP request') + return mock_obj + async def __aexit__(self, *args, **kwargs): + pass + + +class mockHttpAcceptableStatuses(mock.AsyncMock): + async def __aenter__(self, *args, **kwargs): + mock_obj = mock.AsyncMock() + mock_obj.text.return_value = 'mocked response data' + mock_obj.status.return_value = 202 + return mock_obj + async def __aexit__(self, *args, **kwargs): + pass + + +class mockHttpErroneousStatuses(mock.AsyncMock): + async def __aenter__(self, *args, **kwargs): + mock_obj = mock.AsyncMock() + mock_obj.text.return_value = 'mocked failed response data' + mock_obj_status = mock.Mock() + mock_obj.status = mock_obj_status.return_value = 404 + return mock_obj + async def __aexit__(self, *args, **kwargs): + pass + + +class ConnectorsHttpRetry(unittest.TestCase): + def setUp(self): + self.loop = asyncio.get_event_loop() + logger.customer = CUSTOMER_NAME + self.globopts = { + 'authenticationcafile': 'fakeca', + 'authenticationcapath': 'fakepath', + 'authenticationhostcert': 'fakehostcert', + 'authenticationhostkey': 'fakehostkey', + 'authenticationhttppass': 'xxxx', + 'authenticationhttpuser': 'xxxx', + 'authenticationuseplainhttpauth': 'False', + 'authenticationverifyservercert': 'True', + 'avroschemasweights': 'fakeavroschema', + 'connectionretry': '3', 'connectionsleepretry': '1', + 'connectiontimeout': '180', 'generalpassextensions': 'True', + 'generalpublishwebapi': 'False', 'generalwriteavro': 'True', + 'inputstatedays': '3', 'inputstatesavedir': 'fakestate', + 'outputweights': 'fakeoutput.avro', + 'webapihost': 'api.devel.argo.grnet.gr' + } + async def setsession(): + with mock.patch('argo_egi_connectors.io.http.build_ssl_settings'): + self.session = SessionWithRetry(logger, 'test_retry.py', self.globopts, verbose_ret=True) + self.loop.run_until_complete(setsession()) + + # @unittest.skip("skipping") + @mock.patch('aiohttp.ClientSession.get', side_effect=mockHttpGetEmpty) + @async_test + async def test_ConnectorEmptyRetry(self, mocked_get): + path='/url_path' + with self.assertRaises(ConnectorHttpError) as cm: + res = await self.session.http_get('{}://{}{}'.format('http', 'localhost', path)) + self.assertTrue(mocked_get.called) + # defined in connectionretry + self.assertEqual(mocked_get.call_count, 3) + self.assertEqual(mocked_get.call_args[0][0], 'http://localhost/url_path') + + # @unittest.skip("skipping") + @mock.patch('aiohttp.ClientSession.get', side_effect=mockConnectionProblem) + @async_test + async def test_ConnectorConnectionRetry(self, mocked_get): + path='/url_path' + with self.assertRaises(ConnectorHttpError) as cm: + res = await self.session.http_get('{}://{}{}'.format('http', 'localhost', path)) + self.assertTrue(mocked_get.called) + self.assertEqual(mocked_get.call_count, 3) + self.assertEqual(mocked_get.call_args[0][0], 'http://localhost/url_path') + + # @unittest.skip("skipping") + @mock.patch('aiohttp.ClientSession.get', side_effect=mockProtocolProblem) + @async_test + async def test_ConnectorProtocolError(self, mocked_protocolerror): + path='/url_path' + with self.assertRaises(ConnectorHttpError) as cm: + res = await self.session.http_get('{}://{}{}'.format('http', 'localhost', path)) + excep = cm.exception + self.assertIsInstance(excep, ConnectorHttpError) + self.assertTrue(mocked_protocolerror.called) + self.assertEqual(mocked_protocolerror.call_count, 1) + + # @unittest.skip("skipping") + @mock.patch('aiohttp.ClientSession.get', side_effect=mockHttpAcceptableStatuses) + @async_test + async def test_ConnectorHttpAcceptable(self, mocked_httpstatuses): + path='/url_path' + res = await self.session.http_get('{}://{}{}'.format('http', 'localhost', path)) + self.assertTrue(mocked_httpstatuses.called) + + # @unittest.skip("demonstrating skipping") + @mock.patch('aiohttp.ClientSession.get', side_effect=mockHttpErroneousStatuses) + @async_test + async def test_ConnectorHttpErroneous(self, mocked_httperrorstatuses): + path='/url_path' + res = await self.session.http_get('{}://{}{}'.format('http', 'localhost', path)) + self.assertTrue(mocked_httperrorstatuses.called) + self.assertEqual(mocked_httperrorstatuses.call_count, 1) + self.assertFalse(mocked_httperrorstatuses.text.called) + + def tearDown(self): + async def run(): + await self.session.close() + self.loop.run_until_complete(run()) diff --git a/tests/test_topofeed.py b/tests/test_topofeed.py new file mode 100644 index 00000000..5ff2c480 --- /dev/null +++ b/tests/test_topofeed.py @@ -0,0 +1,552 @@ +import unittest + +from argo_egi_connectors.log import Logger +from argo_egi_connectors.parse.gocdb_topology import ParseServiceGroups, ParseServiceEndpoints, ParseSites +from argo_egi_connectors.parse.flat_topology import ParseFlatEndpoints +from argo_egi_connectors.exceptions import ConnectorParseError +from argo_egi_connectors.mesh.contacts import attach_contacts_topodata + +logger = Logger('test_topofeed.py') +CUSTOMER_NAME = 'CUSTOMERFOO' + +# Help function - check if any of endpoints contains extensions +# Used for checking if pass_extensions is working properly +def endpoints_have_extension(group_endpoints): + for gren in group_endpoints: + for key in gren['tags'].keys(): + if key.startswith('info_ext_'): + return True + return False + +# Returns element of group_endpoints with given group_name or None +def get_group(group_endpoints, group_name): + for group in group_endpoints: + if group['group'] == group_name: + return group + + return None + + +class ParseServiceEndpointsTest(unittest.TestCase): + def setUp(self): + with open('tests/sample-service_endpoint.xml') as feed_file: + self.content = feed_file.read() + logger.customer = CUSTOMER_NAME + parse_service_endpoints = ParseServiceEndpoints(logger, self.content, CUSTOMER_NAME) + self.group_endpoints = parse_service_endpoints.get_group_endpoints() + + parse_service_endpoints_ext = ParseServiceEndpoints(logger, self.content, 'CUSTOMERFOO', uid=True, pass_extensions=True) + self.group_endpoints_ext = parse_service_endpoints_ext.get_group_endpoints() + + def test_LenEndpoints(self): + self.assertEqual(len(self.group_endpoints), 3) # Parsed correct number of endpoint groups + + def test_DataEndpoints(self): + self.assertEqual(self.group_endpoints[0], + { + 'group': 'AZ-IFAN', + 'hostname': 'ce.physics.science.az', + 'service': 'CREAM-CE', + 'tags': {'info_HOSTDN': '/DC=ORG/DC=SEE-GRID/O=Hosts/O=Institute of Physics of ANAS/CN=ce.physics.science.az', + 'info_ID': '1555G0', + 'info_URL': 'ce.physics.science.az:8443/cream-pbs-ops', + 'info_service_endpoint_URL': 'ce.physics.science.az:8443/cream-pbs-ops', + 'monitored': '1', + 'production': '1', + 'scope': 'EGI, wlcg, atlas'}, + 'type': 'SITES' + }, + { + 'group': 'RAL-LCG2', + 'hostname': 'arc-ce01.gridpp.rl.ac.uk', + 'service': 'gLite-APEL', + 'tags': {'info_HOSTDN': '/C=UK/O=eScience/OU=CLRC/L=RAL/CN=arc-ce01.gridpp.rl.ac.uk', + 'info_ID': '782G0', + 'monitored': '1', + 'production': '1', + 'scope': 'EGI, wlcg, tier1, alice, atlas, cms, lhcb'}, + 'type': 'SITES' + } + + ) + + def test_HaveExtensions(self): + # Assert pass_extensions=False is working + self.assertFalse(endpoints_have_extension(self.group_endpoints)) + + def test_EnabledExtensions(self): + # Assert pass_extensions=True is working + self.assertTrue(endpoints_have_extension(self.group_endpoints_ext)) + + def test_SuffixUid(self): + # Assert uid=True is working + temp_group = get_group(self.group_endpoints_ext, 'GSI-LCG2') + self.assertIsNotNone(temp_group) + self.assertEqual(temp_group['hostname'], 'grid13.gsi.de_14G0') + + temp_group = get_group(self.group_endpoints_ext, 'RAL-LCG2') + self.assertIsNotNone(temp_group) + self.assertEqual(temp_group['hostname'], 'arc-ce01.gridpp.rl.ac.uk_782G0') + + temp_group = get_group(self.group_endpoints_ext, 'AZ-IFAN') + self.assertIsNotNone(temp_group) + self.assertEqual(temp_group['hostname'], 'ce.physics.science.az_1555G0') + + def test_ConnectorParseErrorException(self): + # Assert proper exception is thrown if empty xml is given to the function + with self.assertRaises(ConnectorParseError) as cm: + ParseServiceEndpoints(logger, '', 'CUSTOMERFOO', uid=True, pass_extensions=True) + excep = cm.exception + self.assertTrue('XML feed' in excep.msg) + self.assertTrue('ExpatError' in excep.msg) + + +class MeshSitesAndContacts(unittest.TestCase): + def setUp(self): + logger.customer = CUSTOMER_NAME + self.maxDiff = None + self.sample_sites_data = [ + { + 'group': 'iris.ac.uk', + 'subgroup': 'dirac-durham', + 'tags': { + 'certification': 'Certified', 'infrastructure': + 'Production', 'scope': 'iris.ac.uk' + }, + 'type': 'NGI' + }, + { + 'group': 'Russia', + 'subgroup': 'RU-SARFTI', + 'tags': { + 'certification': 'Certified', 'infrastructure': + 'Production', 'scope': 'EGI' + }, + 'type': 'NGI' + }, + ] + self.sample_sites_contacts = [ + { + 'contacts': [ + { + 'certdn': 'certdn1-dirac-durham', + 'email': 'name1.surname1@durham.ac.uk', + 'forename': 'Name1', + 'role': 'Site Administrator', + 'surname': 'Surname1' + }, + { + 'certdn': 'certdn2-dirac-durham', + 'email': 'name2.surname2@durham.ac.uk', + 'forename': 'Name2', + 'role': 'Site Operations Manager', + 'surname': 'Surname2' + } + ], + 'name': 'dirac-durham' + }, + { + 'contacts': [ + { + 'certdn': 'certdn1-ru-sarfti', + 'email': 'name1.surname1@gmail.com', + 'forename': 'Name1', + 'role': 'Site Administrator', + 'surname': 'Surname1' + }, + { + 'certdn': 'certdn2-ru-sarfti', + 'email': 'name2.surname2@gmail.com', + 'forename': 'Name2', + 'role': 'Site Administrator', + 'surname': 'Surname2' + }, + ], + 'name': 'RU-SARFTI' + }, + ] + + def test_SitesAndContacts(self): + attach_contacts_topodata(logger, self.sample_sites_contacts, self.sample_sites_data) + self.assertEqual(self.sample_sites_data[0], + { + 'group': 'iris.ac.uk', + 'notifications': {'contacts': ['name1.surname1@durham.ac.uk', + 'name2.surname2@durham.ac.uk'], + 'enabled': True}, + 'subgroup': 'dirac-durham', + 'tags': {'certification': 'Certified', 'infrastructure': + 'Production', 'scope': 'iris.ac.uk'}, + 'type': 'NGI' + } + ) + self.assertEqual(self.sample_sites_data[1], + { + 'group': 'Russia', + 'notifications': {'contacts': ['name1.surname1@gmail.com', + 'name2.surname2@gmail.com'], + 'enabled': True}, + 'subgroup': 'RU-SARFTI', + 'tags': {'certification': 'Certified', 'infrastructure': + 'Production', 'scope': 'EGI'}, + 'type': 'NGI' + } + ) + + +class MeshServiceGroupsAndContacts(unittest.TestCase): + def setUp(self): + logger.customer = CUSTOMER_NAME + self.maxDiff = None + self.sample_servicegroups_data = [ + { + 'group': 'EGI', + 'subgroup': 'NGI_ARMGRID_SERVICES', + 'tags': { + 'monitored': '1', + 'scope': 'EGI' + }, + 'type': 'PROJECT' + }, + { + 'group': 'EGI', + 'subgroup': 'NGI_CYGRID_SERVICES', + 'tags': { + 'monitored': '1', + 'scope': 'EGI' + }, + 'type': 'PROJECT' + }, + ] + self.sample_servicegroup_contacts = [ + { + 'contacts': ['Name1.Surname1@email.com', 'Name2.Surname2@email.com'], + 'name': 'NGI_ARMGRID_SERVICES' + }, + { + 'contacts': ['Name3.Surname3@email.com', 'Name4.Surname4@email.com'], + 'name': 'NGI_CYGRID_SERVICES' + }, + + ] + + def test_ServiceGroupsAndContacts(self): + attach_contacts_topodata(logger, self.sample_servicegroup_contacts, + self.sample_servicegroups_data) + self.assertEqual(self.sample_servicegroups_data[0], + { + 'group': 'EGI', + 'subgroup': 'NGI_ARMGRID_SERVICES', + 'notifications': { + 'contacts': ['Name1.Surname1@email.com', 'Name2.Surname2@email.com'], + 'enabled': True + }, + 'tags': { + 'monitored': '1', + 'scope': 'EGI' + }, + 'type': 'PROJECT' + } + ) + + +class MeshServiceEndpointsAndContacts(unittest.TestCase): + def setUp(self): + logger.customer = CUSTOMER_NAME + self.maxDiff = None + self.sample_serviceendpoints_data = [ + { + 'group': 'GROUP1', + 'hostname': 'fqdn1.com', + 'service': 'service1', + 'tags': { + 'monitored': '1', + 'production': '0', + 'scope': '' + }, + 'type': 'SERVICEGROUPS' + }, + { + 'group': 'GROUP2', + 'hostname': 'fqdn2.com', + 'service': 'service2', + 'tags': { + 'monitored': '1', + 'production': '0', + 'scope': '' + }, + 'type': 'SERVICEGROUPS' + } + ] + self.sample_serviceendpoints_contacts = [ + { + 'contacts': ['Name1.Surname1@email.com', 'Name2.Surname2@email.com'], + 'name': 'fqdn1.com+service1' + }, + { + 'contacts': ['Name3.Surname3@email.com', 'Name4.Surname4@email.com'], + 'name': 'fqdn2.com+service2' + } + ] + + def test_ServiceEndpointsAndContacts(self): + attach_contacts_topodata(logger, self.sample_serviceendpoints_contacts, + self.sample_serviceendpoints_data) + self.assertEqual(self.sample_serviceendpoints_data[0], + { + 'group': 'GROUP1', + 'hostname': 'fqdn1.com', + 'service': 'service1', + 'notifications': { + 'contacts': ['Name1.Surname1@email.com', 'Name2.Surname2@email.com'], + 'enabled': True + }, + 'tags': { + 'monitored': '1', + 'production': '0', + 'scope': '' + }, + 'type': 'SERVICEGROUPS' + } + ) + self.assertEqual(self.sample_serviceendpoints_data[1], + { + 'group': 'GROUP2', + 'hostname': 'fqdn2.com', + 'service': 'service2', + 'notifications': { + 'contacts': ['Name3.Surname3@email.com', 'Name4.Surname4@email.com'], + 'enabled': True + }, + 'tags': { + 'monitored': '1', + 'production': '0', + 'scope': '' + }, + 'type': 'SERVICEGROUPS' + } + ) + + +class ParseServiceEndpointsAndServiceGroupsCsv(unittest.TestCase): + def setUp(self): + with open('tests/sample-topo.csv') as feed_file: + self.content = feed_file.read() + logger.customer = CUSTOMER_NAME + + self.topology = ParseFlatEndpoints(logger, self.content, CUSTOMER_NAME, + uidservtype=True, + fetchtype='ServiceGroups', + scope=CUSTOMER_NAME, is_csv=True) + + def test_CsvTopology(self): + group_groups = self.topology.get_groupgroups() + self.assertEqual(group_groups, + [ + { + 'group': 'CUSTOMERFOO', + 'subgroup': 'NextCloud', + 'tags': {'monitored': '1', 'scope': 'CUSTOMERFOO'}, + 'type': 'PROJECT' + }, + { + 'group': 'CUSTOMERFOO', + 'subgroup': 'AAI', + 'tags': {'monitored': '1', 'scope': 'CUSTOMERFOO'}, + 'type': 'PROJECT' + } + ] + ) + group_endpoints = self.topology.get_groupendpoints() + self.assertEqual(group_endpoints, + [ + { + 'group': 'NextCloud', + 'hostname': 'files.dev.tenant.eu_tenant_1', + 'service': 'nextcloud', + 'tags': {'hostname': 'files.dev.tenant.eu', 'info_ID': + 'tenant_1', 'info_URL': + 'https://files.dev.tenant.eu', 'monitored': '1', + 'scope': 'CUSTOMERFOO'}, + 'type': 'SERVICEGROUPS' + }, + { + 'group': 'NextCloud', + 'hostname': 'files.tenant.eu_tenant_2', + 'service': 'nextcloud', + 'tags': {'hostname': 'files.tenant.eu', 'info_ID': + 'tenant_2', 'info_URL': 'https://files.tenant.eu', + 'monitored': '1', 'scope': 'CUSTOMERFOO'}, + 'type': 'SERVICEGROUPS' + }, + { + 'group': 'AAI', + 'hostname': 'sso.tenant.eu_tenant_3', + 'service': 'aai', + 'tags': {'hostname': 'sso.tenant.eu', 'info_ID': 'tenant_3', + 'info_URL': 'https://sso.tenant.eu', 'monitored': '1', + 'scope': 'CUSTOMERFOO'}, + 'type': 'SERVICEGROUPS' + } + ] + ) + + def test_FailedCsvTopology(self): + with self.assertRaises(ConnectorParseError) as cm: + self.failed_topology = ParseFlatEndpoints(logger, 'RUBBISH_DATA', + CUSTOMER_NAME, + uidservtype=True, + fetchtype='ServiceGroups', + scope=CUSTOMER_NAME, + is_csv=True) + excep = cm.exception + self.assertTrue('CSV feed' in excep.msg) + + +class ParseServiceEndpointsAndServiceGroupsJson(unittest.TestCase): + def setUp(self): + with open('tests/sample-topo.json') as feed_file: + self.content = feed_file.read() + logger.customer = CUSTOMER_NAME + + self.topology = ParseFlatEndpoints(logger, self.content, CUSTOMER_NAME, + uidservtype=True, + fetchtype='ServiceGroups', + scope=CUSTOMER_NAME, is_csv=False) + + def test_JsonTopology(self): + group_groups = self.topology.get_groupgroups() + self.assertEqual(group_groups, + [ + { + 'group': 'CUSTOMERFOO', + 'subgroup': 'Open Telekom Cloud', + 'tags': { + 'monitored': '1', 'scope': 'CUSTOMERFOO' + }, + 'type': 'PROJECT' + }, + { + 'group': 'CUSTOMERFOO', + 'subgroup': 'PaaS Orchestrator ', + 'tags': {'monitored': '1', 'scope': 'CUSTOMERFOO'}, + 'type': 'PROJECT' + } + ] + ) + group_endpoints = self.topology.get_groupendpoints() + self.assertEqual(group_endpoints, + [ + { + 'group': 'Open Telekom Cloud', + 'hostname': 'open-telekom-cloud.com_227', + 'service': 'eu.eosc.portal.services.url', + 'tags': { + 'hostname': 'open-telekom-cloud.com', + 'info_ID': '227', + 'info_URL': 'https://open-telekom-cloud.com/en', + 'monitored': '1', + 'scope': 'CUSTOMERFOO' + }, + 'type': 'SERVICEGROUPS' + }, + { + 'group': 'PaaS Orchestrator ', + 'hostname': 'indigo-paas.cloud.ba.infn.it_243', + 'service': 'eu.eosc.portal.services.url', + 'tags': { + 'hostname': 'indigo-paas.cloud.ba.infn.it', + 'info_ID': '243', + 'info_URL': 'https://indigo-paas.cloud.ba.infn.it', + 'monitored': '1', + 'scope': 'CUSTOMERFOO' + }, + 'type': 'SERVICEGROUPS' + } + ] + ) + + def test_FailedJsonTopology(self): + with self.assertRaises(ConnectorParseError) as cm: + self.failed_topology = ParseFlatEndpoints(logger, 'RUBBISH_DATA', CUSTOMER_NAME, + uidservtype=True, + fetchtype='ServiceGroups', + scope=CUSTOMER_NAME, + is_csv=False) + excep = cm.exception + self.assertTrue('JSON feed' in excep.msg) + self.assertTrue('JSONDecodeError' in excep.msg) + + +class ParseServiceEndpointsBiomed(unittest.TestCase): + def setUp(self): + with open('tests/sample-service_endpoint_biomed.xml') as feed_file: + self.content = feed_file.read() + logger.customer = CUSTOMER_NAME + parse_service_endpoints = ParseServiceEndpoints(logger, self.content, CUSTOMER_NAME) + self.group_endpoints = parse_service_endpoints.get_group_endpoints() + + + def test_BiomedEndpoints(self): + self.assertEqual(self.group_endpoints, + [ + { + 'group': 'HG-02-IASA', + 'hostname': 'cream-ce01.marie.hellasgrid.gr', + 'service': 'APEL', + 'tags': { + 'info_ID': '451G0', + 'monitored': '1', + 'production': '1', + 'scope': 'EGI' + }, + 'type': 'SITES'}, + { + 'group': 'TR-10-ULAKBIM', + 'hostname': 'kalkan1.ulakbim.gov.tr', + 'service': 'APEL', + 'tags': { + 'info_HOSTDN': '/C=TR/O=TRGrid/OU=TUBITAK-ULAKBIM/CN=kalkan1.ulakbim.gov.tr', + 'info_ID': '375G0', + 'monitored': '1', + 'production': '1', + 'scope': 'EGI, wlcg, tier2, atlas' + }, + 'type': 'SITES' + } + ] + ) + + +class ParseSitesBiomed(unittest.TestCase): + def setUp(self): + with open('tests/sample-sites_biomed.xml') as feed_file: + self.content = feed_file.read() + logger.customer = CUSTOMER_NAME + parse_sites = ParseSites(logger, self.content, CUSTOMER_NAME) + self.group_groups = parse_sites.get_group_groups() + + def test_BiomedSites(self): + self.assertEqual(self.group_groups, + [ + { + 'group': 'NGI_FRANCE', + 'subgroup': 'AUVERGRID', + 'tags': { + 'certification': '', 'infrastructure': '', 'scope': '' + }, + 'type': 'NGI' + }, + { + 'group': 'NGI_IT', + 'subgroup': 'CNR-ILC-PISA', + 'tags': { + 'certification': '', 'infrastructure': '', 'scope': '' + }, + 'type': 'NGI' + } + ] + ) + +if __name__ == '__main__': + unittest.main()