Merge pull request #35 from ARGOeu/devel

December release

ar-sync.spec

@@ -1,7 +1,7 @@
 Name: ar-sync
 Summary: A/R Comp Engine sync scripts
 Version: 1.3.1
-Release: 12%{?dist}
+Release: 16%{?dist}
 License: ASL 2.0
 Buildroot: %{_tmppath}/%{name}-buildroot
 Group:     EGI/SA4
@@ -41,8 +41,6 @@ install --mode 644 etc/ar-sync/hepspec-sync.conf %{buildroot}/etc/ar-sync/
 install --mode 644 etc/ar-sync/hepspec-sync.EGI.Cloudmon.conf %{buildroot}/etc/ar-sync/
 install --mode 644 etc/ar-sync/hepspec-sync.EGI.Critical.conf %{buildroot}/etc/ar-sync/
 install --mode 644 etc/ar-sync/prefilter-avro.conf %{buildroot}/etc/ar-sync/
-install --mode 644 etc/ar-sync/prefilter-avro.EGI.Cloudmon.conf %{buildroot}/etc/ar-sync/
-install --mode 644 etc/ar-sync/prefilter-avro.EGI.Critical.conf %{buildroot}/etc/ar-sync/
 install --mode 644 etc/ar-sync/poem_name_mapping.cfg %{buildroot}/var/lib/ar-sync/
 install --mode 644 etc/ar-sync/downtimes.avsc %{buildroot}/etc/ar-sync/
 install --mode 644 etc/ar-sync/metric_profiles.avsc %{buildroot}/etc/ar-sync/
@@ -91,8 +89,6 @@ install --mode 644 cronjobs/hepspec %{buildroot}/etc/cron.d/hepspec
 %config(noreplace) /etc/ar-sync/hepspec-sync.EGI.Cloudmon.conf
 %config(noreplace) /etc/ar-sync/hepspec-sync.EGI.Critical.conf
 %config(noreplace) /etc/ar-sync/prefilter-avro.conf
-%config(noreplace) /etc/ar-sync/prefilter-avro.EGI.Cloudmon.conf
-%config(noreplace) /etc/ar-sync/prefilter-avro.EGI.Critical.conf
 %config(noreplace) /etc/ar-sync/downtimes.avsc
 %config(noreplace) /etc/ar-sync/metric_profiles.avsc
 %config(noreplace) /etc/ar-sync/group_endpoints.avsc
@@ -108,6 +104,15 @@ install --mode 644 cronjobs/hepspec %{buildroot}/etc/cron.d/hepspec
 %attr(0644,root,root) /etc/cron.d/hepspec
 
 %changelog
+* Tue Feb 17 2015 Daniel Vrcic <[email protected]> - 1.3.1-16%(?dist)
+- prefilter-avro has fixed configuration
+* Thu Feb 12 2015 Daniel Vrcic <[email protected]> - 1.3.1-15%(?dist)
+- legacy SRM service type handle for downtime syncs
+* Tue Feb 10 2015 Daniel Vrcic <[email protected]> - 1.3.1-14%(?dist)
+- updated .spec with removed configs for a per job prefilter-avro
+* Tue Feb 10 2015 Daniel Vrcic <[email protected]> - 1.3.1-13%(?dist)
+- different internal handle of avro poem-sync so it doesn't contain duplicated entries
+- special handle of legacy SRM service type
 * Thu Feb 5 2015 Daniel Vrcic <[email protected]> - 1.3.1-12%(?dist)
 - plaintxt prefilter has fixed configuration
 * Tue Feb 3 2015 Daniel Vrcic <[email protected]> - 1.3.1-11%(?dist)

bin/downtime-sync

@@ -37,6 +37,8 @@ from avro.datafile import DataFileWriter
 from avro.io import DatumWriter
 
 defaultConfig = '/etc/ar-sync/downtime-sync.conf'
+
+LegMapServType = {'SRM' : 'SRMv2'}
 defaultArgDateFormat = '%Y-%m-%d'
 
 readers = ['gocdb']
@@ -201,6 +203,11 @@ class FileWritter(DowntimeWriter):
                        dt['service'],
                        dt['start_time'],
                        dt['end_time'], ))
+            if dt['service'] in LegMapServType.keys():
+                outFile.write(self.outputFileFormat % (dt['hostname'],
+                        LegMapServType[dt['service']],
+                        dt['start_time'],
+                        dt['end_time'], ))
         outFile.close();
 
 class AvroWritter(DowntimeWriter):
@@ -227,6 +234,9 @@ class AvroWritter(DowntimeWriter):
 
         for dt in downtimes:
             writer.append(dt)
+            if dt['service'] in LegMapServType.keys():
+                dt['service'] = LegMapServType[dt['service']]
+                writer.append(dt)
 
         writer.close()
         avroFile.close()

bin/poem-sync

@@ -210,7 +210,9 @@ class PoemReader(ProfileReader):
         #sys.stdout.write("vp:  " + pp.pformat(validProfiles) + '\n\n')
         #sys.stdout.flush()
 
-        profileList = list()
+        profileList = []
+        profileListAvro = []
+
         for line in poemServers:
             if len(line) == 0 or line[0] == '#':
                 continue
@@ -274,7 +276,15 @@ class PoemReader(ProfileReader):
                                 for server in servers:
                                     profileList.extend(self.createProfileEntries(server, ngi, validProfiles[profile.upper()]))
 
-        return profileList
+        for profile in validProfiles.values():
+            for metric in profile['metrics']:
+                profileListAvro.append({'profile' : profile['namespace'] + '.' + profile['name'], \
+                                        'metric' : metric['name'], \
+                                        'service' : metric['service_flavour'], \
+                                        'vo' : profile['vo'], \
+                                        'fqan' : metric['fqan']})
+
+        return profileList, profileListAvro
 
     def loadFilteredProfiles(self):
 
@@ -503,23 +513,19 @@ config = loadConfiguration()
 customers, customersdir = loadCustomerConfiguration()
 
 # read profiles
-profiles = list()
 for reader in readers:
     if reader == 'poem':
         readerInstance = PoemReader()
         readerInstance.loadConfiguration(config)
-        ps = readerInstance.getProfiles()
-        profiles.extend(ps)
+        ps, psa = readerInstance.getProfiles()
 
 # write profiles
 for writer in writers:
     if writer == 'file':
         writerInstance = FileWritter()
         writerInstance.loadConfiguration(config)
-        writerInstance.writeProfiles(profiles, date)
-    elif writer == 'avro':
+        writerInstance.writeProfiles(ps, date)
+    elif writer == 'avro' and customers:
         writerInstance = AvroWritter()
         writerInstance.loadConfiguration(config)
-        writerInstance.writeProfiles(profiles, date)
-        if customers:
-            writerInstance.writeProfilesForCustomers(customers, customersdir, profiles, date)
+        writerInstance.writeProfilesForCustomers(customers, customersdir, psa, date)

bin/prefilter-avro

@@ -61,12 +61,12 @@ checkInputFileForDays = 1
 ##############################
 # load config
 ##############################
-def loadConfiguration(confile):
+def loadConfiguration():
     #load config
     configFile = None
     configFields = dict()
-    if os.path.isfile(confile):
-        configFile = open(confile, 'r')
+    if os.path.isfile(defaultConfig):
+        configFile = open(defaultConfig, 'r')
         lines = configFile.readlines()
 
         for line in lines:
@@ -370,7 +370,7 @@ if not argsOk:
     sys.exit()
 
 #load config
-loadConfiguration(configFile)
+loadConfiguration()
 
 # load poem data
 ngis = loadNGIs(year, month, day)

bin/topology-sync

@@ -39,6 +39,7 @@ from avro.io import DatumWriter
 
 defaultConfig = '/etc/ar-sync/topology-sync.conf'
 
+LegMapServType = {'SRM' : 'SRMv2'}
 SelectGG = dict()
 SelectGE = dict()
 FetchType = ''
@@ -454,6 +455,9 @@ class AvroWritter(TopologyWriter):
 
         for g in groupofendpoints:
             writer.append(g)
+            if g['service'] in LegMapServType.keys():
+                g['service'] = LegMapServType[g['service']]
+                writer.append(g)
 
         writer.close()
         avroFile.close()