Merge pull request #162 from vrdel/master

Resolve conflict between devel and master branches
ARGOeu · Mar 31, 2020 · d78b52b · d78b52b
2 parents 05833ee + 3f7efda
commit d78b52b
Show file tree

Hide file tree

Showing 18 changed files with 472 additions and 101 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,7 +1,7 @@
 FROM centos:6.9
 MAINTAINER Themis Zamani [email protected]
 
-RUN yum -y install epel-release 
+RUN yum -y install epel-release
 RUN yum -y makecache; yum -y update
 RUN yum install -y \
         gcc \
@@ -14,18 +14,18 @@ RUN yum install -y \
         python-argparse \
         python-devel \
         python-pip \
-        python-setuptools \
+        python-requests \
         tar \
         wget
 RUN pip install \
         argo_ams_library \
         avro \
-        coverage \
         cffi \
-        cryptography \
+        coverage==4.5.4 \
+        cryptography==2.1.4 \
         discover \
         httmock \
-        mock \
+        mock==2.0.0 \
         pyOpenSSL \
-        requests \
+        setuptools \
         unittest2
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,6 +1,5 @@
 include bin/*.py
-include cronjobs/*
-include etc/*.conf
+include etc/*.conf.template
 include etc/schemas/*.avsc
 include modules/*.py
 include argo-egi-connectors.spec

diff --git a/argo-egi-connectors.spec b/argo-egi-connectors.spec
@@ -1,11 +1,11 @@
 Name: argo-egi-connectors
-Version: 1.7.2
+Version: 1.7.4
 Release: 1%{?dist}
 Group: EGI/SA4
 License: ASL 2.0
 Summary: Components generate input for ARGO Compute Engine
 Url: http://argoeu.github.io/guides/sync/
-Vendor: SRCE <[email protected]>
+Vendor: SRCE <[email protected]>, SRCE <[email protected]>
 
 Obsoletes: ar-sync
 Prefix: %{_prefix}
@@ -42,12 +42,16 @@ rm -rf $RPM_BUILD_ROOT
 %attr(0755,root,root) %dir %{_libexecdir}/argo-egi-connectors/
 %attr(0755,root,root) %{_libexecdir}/argo-egi-connectors/*.py*
 
-%attr(0644,root,root) %{_sysconfdir}/cron.d/*
-
 %attr(0755,root,root) %dir %{_sharedstatedir}/argo-connectors/
 %attr(0755,root,root) %dir %{_localstatedir}/log/argo-connectors/
 
 %changelog
+* Mon Mar 30 2020 Daniel Vrcic <[email protected]> - 1.7.4-1%{dist}
+- ARGO-2247 Pass URL from EOSC topology
+- ARGO-2225 Support for creating empty weights and downtimes data
+- ARGO-2221 Metric profile namespace optional 
+- ARGO-2210 Introduce topology connector for EOSC-PORTAL
+- ARGO-2209 Pass PRIMARY_KEY of GOCDB service endpoint as a unique service endpoint identifier 
 * Fri Nov 8 2019 Daniel Vrcic <[email protected]> - 1.7.3-1%{?dist}
 - ARGO-2017 - Token per tenants config option
 - ARGO-2013 - Metric profiles WEB-API connector
@@ -59,13 +63,13 @@ rm -rf $RPM_BUILD_ROOT
 - ARGO-1428 ServiceGroup topology filtering
 - ARGO-1370 Optimize connectors queries to POEM
 - ARGO-1269 Refactor poem-connector
-- ARGO-1236 Datestamp of AMS msg does not match corresponding avro filename 
+- ARGO-1236 Datestamp of AMS msg does not match corresponding avro filename
 * Wed May 23 2018 Daniel Vrcic <[email protected]> - 1.7.0-1%{?dist}
 - ARGO-1093 Support for GOCDB paginated topology API
 - ARGO-1080 add support for basic-auth in Connectors
 - ARGO-966 Lower state files permissions
 * Tue Mar 27 2018 Daniel Vrcic <[email protected]> - 1.6.1-1%{?dist}
-- selectively use GOCDB paginated API for topology 
+- selectively use GOCDB paginated API for topology
 * Thu Nov 30 2017 Daniel Vrcic <[email protected]> - 1.6.0-1%{?dist}
 - ARGO-965 Support for packing connectors data in a single AMS message
 - ARGO-921 Use ComputationPower instead of HEPSPEC2006 value for weights
@@ -83,21 +87,21 @@ rm -rf $RPM_BUILD_ROOT
 - ARGO-766 Remove SRMv2 service type mapping
 * Mon Mar 20 2017 Daniel Vrcic <[email protected]> - 1.5.7-1%{?dist}
 - ARGO-767 Remove topology-vo connector
-- refactored topology filtering 
+- refactored topology filtering
 - removed schema migration helper
 * Fri Mar 17 2017 Daniel Vrcic <[email protected]> - 1.5.6-1%{?dist}
-- ARGO-762 Remove inspection logic of HEPSPEC factors for previous days 
+- ARGO-762 Remove inspection logic of HEPSPEC factors for previous days
 * Thu Mar 9 2017 Daniel Vrcic <[email protected]> - 1.5.5-1%{?dist}
 - ARGO-713 Topology connector should be able to pick only particular NGI or site
 * Mon Jan 30 2017 Daniel Vrcic <[email protected]> - 1.5.4-1%{?dist}
-- ARGO-667 filter endpoints whose groups are filtered in groups of groups 
+- ARGO-667 filter endpoints whose groups are filtered in groups of groups
 * Wed Jan 25 2017 Daniel Vrcic <[email protected]> - 1.5.3-3%{?dist}
 - prefilter output datestamp with underscores
 * Wed Jan 25 2017 Daniel Vrcic <[email protected]> - 1.5.3-2%{?dist}
-- prefilter datestamp extracted from arg tuple 
+- prefilter datestamp extracted from arg tuple
 * Thu Jan 19 2017 Daniel Vrcic <[email protected]> - 1.5.3-1%{?dist}
 - poem and output files as arguments to prefilter
-- refactored filename datestamp creation 
+- refactored filename datestamp creation
 * Wed Jan 4 2017 Daniel Vrcic <[email protected]> - 1.5.2-1%{?dist}
 - ARGO-550 Introduce states that can be checked by Nagios probe
 * Thu Nov 24 2016 Daniel Vrcic <[email protected]> - 1.5.1-2%{?dist}
@@ -108,7 +112,7 @@ rm -rf $RPM_BUILD_ROOT
 - ARGO-584 Ensure to catch all exceptions of underlying library
 * Sat Sep 24 2016 Themis Zamani <[email protected]> - 1.5.0-1%{?dist}
 - New RPM package release
-* Wed Aug 31 2016 Daniel Vrcic <[email protected]> - 1.4.6-2%{?dist} 
+* Wed Aug 31 2016 Daniel Vrcic <[email protected]> - 1.4.6-2%{?dist}
 - make use of VAPOR service for weights
 - extended cert verification with CAfile bundle
 * Tue Feb 16 2016 Daniel Vrcic <[email protected]> - 1.4.6-1%{?dist}
@@ -124,7 +128,7 @@ rm -rf $RPM_BUILD_ROOT
 - log failed VO and weights connections
   https://github.com/ARGOeu/ARGO/issues/179
 - added connection timeout for all connectors
-- config files can be passed as arguments to every component 
+- config files can be passed as arguments to every component
   https://github.com/ARGOeu/ARGO/issues/180
 - added connection retry feature forr all connectors
 - prefilter explicit input and output
@@ -208,10 +212,10 @@ rm -rf $RPM_BUILD_ROOT
 - added Dirname optional option for VO config
 - correctly renamed avro schemas
 * Mon Mar 30 2015 Daniel Vrcic <[email protected]> - 1.4.0-7%{?dist}
-- added README.md with a basic project info  
+- added README.md with a basic project info
 * Sun Mar 29 2015 Daniel Vrcic <[email protected]> - 1.4.0-6%{?dist}
 - renamed weights and more configs refactoring
-- put scripts back into libexec 
+- put scripts back into libexec
 * Fri Mar 27 2015 Daniel Vrcic <[email protected]> - 1.4.0-5%{?dist}
 - minor code cleanups and renamed connectors to reflect the source of data
 * Fri Mar 27 2015 Daniel Vrcic <[email protected]> - 1.4.0-4%{?dist}

diff --git a/bin/downtimes-gocdb-connector.py b/bin/downtimes-gocdb-connector.py
@@ -52,13 +52,27 @@ def getText(nodelist):
     return ''.join(rc)
 
 
+def all_same(elemlist):
+    prev = None
+
+    for el in elemlist:
+        if prev is None:
+            prev = el
+        else:
+            if el != prev:
+                return False
+            else:
+                return True
+
+
 class GOCDBReader(object):
-    def __init__(self, feed, auth=None):
+    def __init__(self, feed, auth=None, uid=False):
         self._o = urlparse(feed)
         self.argDateFormat = "%Y-%m-%d"
         self.WSDateFormat = "%Y-%m-%d %H:%M"
         self.state = True
         self.custauth = auth
+        self.uid = uid
 
     def getDowntimes(self, start, end):
         filteredDowntimes = list()
@@ -91,7 +105,10 @@ def getDowntimes(self, start, end):
                     startStr = getText(downtime.getElementsByTagName('FORMATED_START_DATE')[0].childNodes)
                     endStr = getText(downtime.getElementsByTagName('FORMATED_END_DATE')[0].childNodes)
                     severity = getText(downtime.getElementsByTagName('SEVERITY')[0].childNodes)
-
+                    try:
+                        serviceId = getText(downtime.getElementsByTagName('PRIMARY_KEY')[0].childNodes)
+                    except IndexError:
+                        serviceId = downtime.getAttribute('PRIMARY_KEY')
                     startTime = datetime.datetime.strptime(startStr, self.WSDateFormat)
                     endTime = datetime.datetime.strptime(endStr, self.WSDateFormat)
 
@@ -102,7 +119,10 @@ def getDowntimes(self, start, end):
 
                     if classification == 'SCHEDULED' and severity == 'OUTAGE':
                         dt = dict()
-                        dt['hostname'] = hostname
+                        if self.uid:
+                            dt['hostname'] = '{0}_{1}'.format(hostname, serviceId)
+                        else:
+                            dt['hostname'] = hostname
                         dt['service'] = serviceType
                         dt['start_time'] = startTime.strftime('%Y-%m-%d %H:%M').replace(' ', 'T', 1).replace(' ', ':') + ':00Z'
                         dt['end_time'] = endTime.strftime('%Y-%m-%d %H:%M').replace(' ', 'T', 1).replace(' ', ':') + ':00Z'
@@ -117,6 +137,7 @@ def getDowntimes(self, start, end):
             else:
                 return filteredDowntimes
 
+
 def main():
     global logger, globopts
     parser = argparse.ArgumentParser(description='Fetch downtimes from GOCDB for given date')
@@ -137,9 +158,8 @@ def main():
     confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
     feeds = confcust.get_mapfeedjobs(sys.argv[0], deffeed='https://goc.egi.eu/gocdbpi/')
 
-
     if len(args.date) == 0:
-        print parser.print_help()
+        print(parser.print_help())
         raise SystemExit(1)
 
     # calculate start and end times
@@ -153,28 +173,50 @@ def main():
         logger.error(e)
         raise SystemExit(1)
 
-
+    j = 0
     for feed, jobcust in feeds.items():
         customers = set(map(lambda jc: confcust.get_custname(jc[1]), jobcust))
         customers = customers.pop() if len(customers) == 1 else '({0})'.format(','.join(customers))
-        jobs = set(map(lambda jc: jc[0], jobcust))
-        jobs = jobs.pop() if len(jobs) == 1 else '({0})'.format(','.join(jobs))
+        sjobs = set(map(lambda jc: jc[0], jobcust))
+        jobs = list(sjobs)[0] if len(sjobs) == 1 else '({0})'.format(','.join(sjobs))
         logger.job = jobs
         logger.customer = customers
+        uidjob = confcust.pass_uidserviceendpoints(sjobs)
 
         auth_custopts = confcust.get_authopts(feed, jobcust)
         auth_opts = cglob.merge_opts(auth_custopts, 'authentication')
         auth_complete, missing = cglob.is_complete(auth_opts, 'authentication')
-        if auth_complete:
-            gocdb = GOCDBReader(feed, auth=auth_opts)
-            dts = gocdb.getDowntimes(start, end)
-        else:
+        if not auth_complete:
             logger.error('Customer:%s Jobs:%s %s options incomplete, missing %s'
-                         % (logger.customer, logger.job, 'authentication',
+                        % (logger.customer, logger.job, 'authentication',
                             ''.join(missing)))
             continue
 
+        # we don't have multiple tenant definitions in one
+        # customer file so we can safely assume one tenant/customer
+        cust = jobcust[0][1]
+        write_empty = confcust.send_empty(sys.argv[0], cust)
+
+        # do fetch only once
+        if all_same(uidjob):
+            gocdb = GOCDBReader(feed, auth_opts, uidjob[j])
+            if not write_empty:
+                dts = gocdb.getDowntimes(start, end)
+            else:
+                dts = []
+                gocdb.state = True
+
         for job, cust in jobcust:
+            # fetch for every job because of different TopoUIDServiceEndpoints
+            # setting for each job
+            if not all_same(uidjob):
+                gocdb = GOCDBReader(feed, auth_opts, uidjob[j])
+                if not write_empty:
+                    dts = gocdb.getDowntimes(start, end)
+                else:
+                    dts = []
+                    gocdb.state = True
+
             jobdir = confcust.get_fulldir(cust, job)
             jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust, job)
 
@@ -216,6 +258,8 @@ def main():
                     logger.error('Customer:%s Job:%s %s' % (logger.customer, logger.job, repr(excep)))
                     raise SystemExit(1)
 
+            j += 1
+
         if gocdb.state:
             custs = set([cust for job, cust in jobcust])
             for cust in custs:

diff --git a/bin/metricprofile-webapi-connector.py b/bin/metricprofile-webapi-connector.py
@@ -69,8 +69,12 @@ def get_profiles(self):
             for profile in target_profiles:
                 for service in profile['services']:
                     for metric in service['metrics']:
+                        if self.namespace:
+                            profile_name = '{0}.{1}'.format(self.namespace, profile['name'])
+                        else:
+                            profile_name = profile['name']
                         profile_list.append({
-                            'profile': '{0}.{1}'.format(self.namespace, profile['name']),
+                            'profile': profile_name,
                             'metric': metric,
                             'service': service['service']
                         })
@@ -79,7 +83,7 @@ def get_profiles(self):
             self.state = False
             logger.error(module_class_name(self) + ' Customer:%s : Error parsing feed %s - %s' % (logger.customer,
                                                                                                   self.host + API_PATH,
-                                                                                                  repr(e).replace('\'','').replace('\"', '')))
+                                                                                                  repr(e).replace('\'', '').replace('\"', '')))
             return []
         else:
             return self._format(profile_list)
@@ -93,7 +97,7 @@ def _fetch(self):
                 raise input.ConnectorError()
 
             json_data = input.parse_json(logger, module_class_name(self),
-                                        globopts, res, self.host + API_PATH)
+                                         globopts, res, self.host + API_PATH)
 
             if not json_data or not json_data.get('data', False):
                 raise input.ConnectorError()