Merge pull request #62 from vrdel/devel

configurable customer name and bugfix for lowercased POEM profiles
ARGOeu · Oct 16, 2015 · 153d843 · 153d843
2 parents 94ba049 + 38ff19f
commit 153d843
Show file tree

Hide file tree

Showing 9 changed files with 37 additions and 15 deletions.
diff --git a/argo-egi-connectors.spec b/argo-egi-connectors.spec
@@ -1,6 +1,6 @@
 Name: argo-egi-connectors
 Version: 1.4.4
-Release: 5%{?dist}
+Release: 6%{?dist}
 
 Group: EGI/SA4
 License: ASL 2.0
@@ -46,6 +46,11 @@ rm -rf $RPM_BUILD_ROOT
 %attr(0750,root,root) %dir %{_localstatedir}/log/argo-egi-connectors/
 
 %changelog
+* Thu Oct 15 2015 Daniel Vrcic <[email protected]> - 1.4.4-6%{?dist}
+- bugfix handling lowercase defined POEM profiles
+- remove hardcoded customer name for topology-gocdb-connector
+  https://github.com/ARGOeu/ARGO/issues/173
+- guide updated with new configuration option for customer
 * Thu Oct 8 2015 Daniel Vrcic <[email protected]> - 1.4.4-5%{?dist}
 - bugfix in case of no downtimes defined for given date
   https://github.com/ARGOeu/ARGO/issues/170

diff --git a/bin/downtimes-gocdb-connector.py b/bin/downtimes-gocdb-connector.py
@@ -150,12 +150,13 @@ def main():
                     dtslegmap[-1]['service'] = LegMapServType[dt['service']]
             for job, cust in jobcust:
                 jobdir = confcust.get_fulldir(cust, job)
+                custname = confcust.get_custname(cust)
 
                 filename = jobdir + globopts['OutputDowntimes'.lower()] % timestamp
                 avro = AvroWriter(globopts['AvroSchemasDowntimes'.lower()], filename,
                                 dts + dtslegmap, os.path.basename(sys.argv[0]))
                 avro.write()
 
-            logger.info('Jobs:%d Feed:%s Fetched Date:%s Endpoints:%d' % (len(jobcust), feed, args.date[0], len(dts + dtslegmap)))
+            logger.info('Customer:%s Jobs:%d Fetched Date:%s Endpoints:%d' % (custname, len(jobcust), args.date[0], len(dts + dtslegmap)))
 
 main()
diff --git a/bin/poem-connector.py b/bin/poem-connector.py
@@ -43,6 +43,7 @@
 logger = None
 globopts, poemopts = {}, {}
 cpoem = None
+custname = ''
 
 class PoemReader:
     def __init__(self):
@@ -156,8 +157,8 @@ def loadProfilesFromServer(self, server, vo, filterProfiles):
             if res.status == 200:
                 json_data = json.loads(res.read())
                 for profile in json_data[0]['profiles']:
-                    if not doFilterProfiles or (profile['namespace']+'.'+profile['name']).upper() in filterProfiles:
-                        validProfiles[(profile['namespace']+'.'+profile['name']).upper()] = profile
+                    if not doFilterProfiles or profile['namespace'].upper()+'.'+profile['name'] in filterProfiles:
+                        validProfiles[profile['namespace'].upper()+'.'+profile['name']] = profile
             elif res.status in (301, 302):
                 logger.warning('Redirect: ' + urlparse.urljoin(url, res.getheader('location', '')))
 
@@ -255,6 +256,8 @@ def main():
         poempref = PrefilterPoem(confcust.get_custdir(cust))
         poempref.writeProfiles(ps, timestamp)
 
+        custname = confcust.get_custname(cust)
+
         for job in confcust.get_jobs(cust):
             jobdir = confcust.get_fulldir(cust, job)
 
@@ -266,6 +269,6 @@ def main():
                               lfprofiles, os.path.basename(sys.argv[0]))
             avro.write()
 
-            logger.info('Job:'+job+' Profiles:%s Tuples:%d' % (','.join(profiles), len(lfprofiles)))
+            logger.info('Customer:'+custname+' Job:'+job+' Profiles:%s Tuples:%d' % (','.join(profiles), len(lfprofiles)))
 
 main()
diff --git a/bin/topology-gocdb-connector.py b/bin/topology-gocdb-connector.py
@@ -46,6 +46,7 @@
 fetchtype = ''
 
 globopts = {}
+custname = ''
 logger = None
 
 class GOCDBReader:
@@ -99,7 +100,7 @@ def getGroupOfGroups(self):
             for d in gl:
                 g = dict()
                 g['type'] = 'PROJECT'
-                g['group'] = 'EGI'
+                g['group'] = custname
                 g['subgroup'] = d['name']
                 g['tags'] = {'monitored' : 1 if d['monitored'] == 'Y' else 0,
                             'scope' : d['scope']}
@@ -308,6 +309,8 @@ def main():
             jobdir = confcust.get_fulldir(cust, job)
             global fetchtype
             fetchtype = confcust.get_gocdb_fetchtype(job)
+            global custname
+            custname = confcust.get_custname(cust)
 
             if fetchtype == 'ServiceGroups':
                 group_endpoints = gocdb.getGroupOfServices()
@@ -341,9 +344,9 @@ def main():
                             group_endpoints + gelegmap, os.path.basename(sys.argv[0]))
             avro.write()
 
-            logger.info('Job:'+job+' Fetched Endpoints:%d' % (numge + numgeleg) +' Groups(%s):%d' % (fetchtype, numgg))
+            logger.info('Customer:'+custname+' Job:'+job+' Fetched Endpoints:%d' % (numge + numgeleg) +' Groups(%s):%d' % (fetchtype, numgg))
             if getags or ggtags:
-                selstr = 'Job:%s Selected ' % (job)
+                selstr = 'Customer:%s Job:%s Selected ' % (custname, job)
                 selge, selgg = '', ''
                 if getags:
                     for key, value in getags.items():

diff --git a/bin/topology-vo-connector.py b/bin/topology-vo-connector.py
@@ -141,6 +141,8 @@ def main():
         for job, cust in jobcust:
             jobdir = confcust.get_fulldir(cust, job)
 
+            custname = confcust.get_custname(cust)
+
             filtlgroups = vo.get_groupgroups()
             numgg = len(filtlgroups)
             tags = confcust.get_vo_ggtags(job)
@@ -170,9 +172,9 @@ def ismatch(elem):
                                                                                        os.path.basename(sys.argv[0]))
             avro.write()
 
-            logger.info('Job:'+job+' Fetched Endpoints:%d' % (numge + len(gelegmap))+' Groups:%d' % (numgg))
+            logger.info('Customer:'+custname+' Job:'+job+' Fetched Endpoints:%d' % (numge + len(gelegmap))+' Groups:%d' % (numgg))
             if tags:
-                selstr = 'Job:%s Selected ' % (job)
+                selstr = 'Customer:%s Job:%s Selected ' % (custname, job)
                 selgg = ''
                 for key, value in tags.items():
                     if isinstance(value, list):

diff --git a/bin/weights-gstat-connector.py b/bin/weights-gstat-connector.py
@@ -165,6 +165,7 @@ def main():
 
         for job, cust in jobcust:
             jobdir = confcust.get_fulldir(cust, job)
+            custname = confcust.get_custname(cust)
 
             filename = jobdir + globopts['OutputWeights'.lower()] % timestamp
             datawr = gen_outdict(newData)
@@ -176,6 +177,6 @@ def main():
                 avro = AvroWriter(globopts['AvroSchemasWeights'.lower()], filename, datawr, os.path.basename(sys.argv[0]))
                 avro.write()
 
-        logger.info('Jobs:%d Sites:%d' % (len(jobcust), len(datawr)))
+        logger.info('Customer:%s Jobs:%d Sites:%d' % (custname, len(jobcust), len(datawr)))
 
 main()
diff --git a/doc/sync.md b/doc/sync.md
@@ -27,7 +27,7 @@ Installation narrows down to simply installing the package:
 
 	`yum -y install argo-egi-connectors`
 
-**`Components require avro package to be installed/available.`**
+**`Components require avro and pyOpenSSL packages to be installed/available.`**
 
 
 | File Types | Destination |
@@ -99,17 +99,19 @@ This configuration file lists all customers, their jobs and appropriate attribut
 Job folders for each customer are placed under the customer's `OutputDir` directory and appropriate directory names are read from the config file. Segment of configuration file that reflects the creation of directories is for example: 
 
 	[CUSTOMER_C1]
+	Name = C1Name1
 	OutputDir = /var/lib/argo-connectors/Customer1
 	Jobs = JOB_Test1, JOB_Test2
 
 	[JOB_Test1]
 	Dirname = C1Testing1
 
 	[JOB_Test2]
-	Dirname = C2Testing2
+	Dirname = C1Testing2
 
 
 	[CUSTOMER_C2]
+	Name = C2Name2
 	OutputDir = /var/lib/argo-connectors/Customer2
 	Jobs = Job_Test3, JOB_Test4
 
@@ -126,7 +128,7 @@ This will result in the following jobs directories:
 	/var/lib/argo-connectors/Customer2/C2Testing1
 	/var/lib/argo-connectors/Customer2/C2Testing2
 
-So there are two customers, C1 and C2, each one identified with its `[CUSTOMER_*]` section. `CUSTOMER_` is a section keyword and must be specified when one wants to define a new customer. Each customer has two mandatory options: `OutputDir` and `Jobs`. With `OutputDir` option, customer defines his directory where he'll write job folders and other data. Customer must also specify set of jobs listed in `Jobs` options since it can not exist without associated jobs. The name of the job folder is specified with `Dirname` option of the certain job so `JOB\_Test1`, identified with `[JOB_Test1]` section, will be named `C1Testing1` and it will be placed under customer's `/var/lib/argo-connectors/Customer1/` directory. Each component will firstly try to create described directory structure if it doesn't exist yet. Only afterward it will write its data. 
+So there are two customers, C1 and C2, each one identified with its `[CUSTOMER_*]` section. `CUSTOMER_` is a section keyword and must be specified when one wants to define a new customer. Each customer has three mandatory options: `Name`, `OutputDir` and `Jobs`. With `OutputDir` option, customer defines his directory where he'll write job folders and other data. Customer must also specify set of jobs listed in `Jobs` options since it can not exist without associated jobs. The name of the job folder is specified with `Dirname` option of the certain job so `JOB\_Test1`, identified with `[JOB_Test1]` section, will be named `C1Testing1` and it will be placed under customer's `/var/lib/argo-connectors/Customer1/` directory. Each component will firstly try to create described directory structure if it doesn't exist yet. Only afterward it will write its data.
 
 Every connector reads this configuration file because it needs to find out how many customers are there and what are theirs customer and job directory names where they will lay down its files. So `poem-connector.py`, `downtimes-gocdb-connector.py`, `weights-gstat-connector.py`, all of them are writing theirs data in each job directory for each customer. Topology for EGI (fetched from GOCDB) is different than one for the VO so exceptions to this are `topology-gocdb-connector.py` and `topology-vo-connector.py`. They are writing data for a job based on the job's topology type specified with `TopoType` attribute.
 

diff --git a/etc/customer.conf b/etc/customer.conf
@@ -2,6 +2,7 @@
 GridPPFeed = http://www-pnp.physics.ox.ac.uk/%7Emohammad/gridpp.feed.xml
 
 [CUSTOMER_EGI]
+Name = EGI
 OutputDir = /var/lib/argo-connectors/EGI/
 Jobs = JOB_EGICritical, JOB_EGICloudmon, JOB_GridPPTest
 

diff --git a/modules/config.py b/modules/config.py
@@ -164,11 +164,12 @@ def parse(self):
                     custjobs = config.get(section, 'Jobs').split(',')
                     custjobs = [job.strip() for job in custjobs]
                     custdir = config.get(section, 'OutputDir')
+                    custname = config.get(section, 'Name')
                 except ConfigParser.NoOptionError as e:
                     self.logger.error(e.message)
                     raise SystemExit(1)
 
-                self._cust.update({section: {'Jobs': custjobs, 'OutputDir': custdir}})
+                self._cust.update({section: {'Jobs': custjobs, 'OutputDir': custdir, 'Name': custname}})
 
                 if self._custattrs:
                     for attr in self._custattrs:
@@ -227,6 +228,9 @@ def get_fulldir(self, cust, job):
     def get_custdir(self, cust):
         return self._dir_from_sect(cust, self._cust)
 
+    def get_custname(self, cust):
+        return self._cust[cust]['Name']
+
     def make_dirstruct(self):
         dirs = []
         for cust in self._cust.keys():