From 42a9e3e37220b831513592a7083f76740acbb789 Mon Sep 17 00:00:00 2001 From: bars Date: Mon, 11 Jan 2016 16:09:15 +0100 Subject: [PATCH 01/11] Added support for connection to remote mongodb. --- mnemosyne.cfg.dist | 2 ++ persistance/mnemodb.py | 6 +++--- persistance/preagg_reports.py | 4 ++-- runner.py | 4 +++- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/mnemosyne.cfg.dist b/mnemosyne.cfg.dist index efea053..a689739 100644 --- a/mnemosyne.cfg.dist +++ b/mnemosyne.cfg.dist @@ -4,6 +4,8 @@ host = 0.0.0.0 port = 8181 [mongodb] +host = 127.0.0.1 +port = 27017 database = mnemosyne [hpfriends] diff --git a/persistance/mnemodb.py b/persistance/mnemodb.py index 8579102..ddc025a 100644 --- a/persistance/mnemodb.py +++ b/persistance/mnemodb.py @@ -32,10 +32,10 @@ class MnemoDB(object): - def __init__(self, database_name): + def __init__(self, mongo_host, mongo_port, database_name): logger.info('Connecting to mongodb, using "{0}" as database.'.format(database_name)) - conn = MongoClient(auto_start_request=False) - self.rg = ReportGenerator(database_name) + conn = MongoClient(host=mongo_host, port=mongo_port, auto_start_request=False) + self.rg = ReportGenerator(mongo_host, mongo_port, database_name) self.db = conn[database_name] self.ensure_index() diff --git a/persistance/preagg_reports.py b/persistance/preagg_reports.py index eea649f..4d348a3 100644 --- a/persistance/preagg_reports.py +++ b/persistance/preagg_reports.py @@ -27,9 +27,9 @@ class ReportGenerator: Generates pre-aggregated reports. """ - def __init__(self, database_name): + def __init__(self, mongo_host, mongo_port, database_name): logger.info('Connecting to mongodb, using "{0}" as database.'.format(database_name)) - conn = MongoClient(w=0) + conn = MongoClient(host=mongo_host, port=mongo_port, w=0) self.db = conn[database_name] def hpfeeds(self, entry): diff --git a/runner.py b/runner.py index e9389ed..6dfcac7 100644 --- a/runner.py +++ b/runner.py @@ -56,6 +56,8 @@ def parse_config(config_file): config['loggly_token'] = parser.get('loggly_log', 'token') config['mongo_db'] = parser.get('mongodb', 'database') + config['mongo_host'] = parser.get('mongodb', 'host') + config['mongo_port'] = parser.get('mongodb', 'port') config['hpf_feeds'] = parser.get('hpfriends', 'channels').split(',') config['hpf_ident'] = parser.get('hpfriends', 'ident') @@ -113,7 +115,7 @@ def do_logging(file_log=None, loggly_token=None): greenlets = {} - db = mnemodb.MnemoDB(c['mongo_db']) + db = mnemodb.MnemoDB(c['mongo_host'], c['mongo_port'], c['mongo_db']) webapi = None hpfriends_puller = None From c7e1828c68309083ac4db48704ca3ac1d31f4a36 Mon Sep 17 00:00:00 2001 From: bars Date: Wed, 20 Jan 2016 15:31:28 +0100 Subject: [PATCH 02/11] bug fix --- runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner.py b/runner.py index 6dfcac7..bbd2538 100644 --- a/runner.py +++ b/runner.py @@ -57,7 +57,7 @@ def parse_config(config_file): config['mongo_db'] = parser.get('mongodb', 'database') config['mongo_host'] = parser.get('mongodb', 'host') - config['mongo_port'] = parser.get('mongodb', 'port') + config['mongo_port'] = parser.getint('mongodb', 'port') config['hpf_feeds'] = parser.get('hpfriends', 'channels').split(',') config['hpf_ident'] = parser.get('hpfriends', 'ident') From df0fd49178db46802a861012bf58b4ceff21112e Mon Sep 17 00:00:00 2001 From: bars Date: Tue, 26 Jan 2016 15:49:06 +0100 Subject: [PATCH 03/11] added default values for mongodb host and port --- runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner.py b/runner.py index bbd2538..7347dc0 100644 --- a/runner.py +++ b/runner.py @@ -56,8 +56,8 @@ def parse_config(config_file): config['loggly_token'] = parser.get('loggly_log', 'token') config['mongo_db'] = parser.get('mongodb', 'database') - config['mongo_host'] = parser.get('mongodb', 'host') - config['mongo_port'] = parser.getint('mongodb', 'port') + config['mongo_host'] = parser.get('mongodb', 'host', 0, {'host': '127.0.0.1'}) + config['mongo_port'] = int(parser.get('mongodb', 'port', 0, {'port': '27017'})) config['hpf_feeds'] = parser.get('hpfriends', 'channels').split(',') config['hpf_ident'] = parser.get('hpfriends', 'ident') From b36a6ea851a3daf04ea29fcd681030c71c23ed55 Mon Sep 17 00:00:00 2001 From: bars Date: Wed, 24 Feb 2016 15:08:20 +0100 Subject: [PATCH 04/11] reworked assigning default values for config of mongo host and port -> fixed bug --- mnemosyne.cfg.dist | 6 +++--- runner.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mnemosyne.cfg.dist b/mnemosyne.cfg.dist index a689739..512cf37 100644 --- a/mnemosyne.cfg.dist +++ b/mnemosyne.cfg.dist @@ -4,9 +4,9 @@ host = 0.0.0.0 port = 8181 [mongodb] -host = 127.0.0.1 -port = 27017 -database = mnemosyne +mongod_host = 127.0.0.1 +mongod_port = 27017 +mongod_database = mnemosyne [hpfriends] ident = diff --git a/runner.py b/runner.py index 7347dc0..daf29f7 100644 --- a/runner.py +++ b/runner.py @@ -39,7 +39,7 @@ def parse_config(config_file): if not os.path.isfile(config_file): sys.exit("Could not find configuration file: {0}".format(config_file)) - parser = ConfigParser() + parser = ConfigParser({'mongod_host': '127.0.0.1', 'mognod_port': 27017}) parser.read(config_file) log_file = None @@ -55,9 +55,9 @@ def parse_config(config_file): if parser.getboolean('loggly_log', 'enabled'): config['loggly_token'] = parser.get('loggly_log', 'token') - config['mongo_db'] = parser.get('mongodb', 'database') - config['mongo_host'] = parser.get('mongodb', 'host', 0, {'host': '127.0.0.1'}) - config['mongo_port'] = int(parser.get('mongodb', 'port', 0, {'port': '27017'})) + config['mongo_db'] = parser.get('mongodb', 'mongod_database') + config['mongo_host'] = parser.get('mongodb', 'mongod_host') + config['mongo_port'] = parser.getint('mongodb', 'mongod_port') config['hpf_feeds'] = parser.get('hpfriends', 'channels').split(',') config['hpf_ident'] = parser.get('hpfriends', 'ident') From 1e12b91bb6e4ae4ca252ad7b4fce1890423ea18d Mon Sep 17 00:00:00 2001 From: bars Date: Wed, 24 Feb 2016 15:20:51 +0100 Subject: [PATCH 05/11] mongo database config renamed to old style -> support for old versions of conf files --- mnemosyne.cfg.dist | 2 +- runner.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mnemosyne.cfg.dist b/mnemosyne.cfg.dist index 512cf37..9efdb84 100644 --- a/mnemosyne.cfg.dist +++ b/mnemosyne.cfg.dist @@ -6,7 +6,7 @@ port = 8181 [mongodb] mongod_host = 127.0.0.1 mongod_port = 27017 -mongod_database = mnemosyne +database = mnemosyne [hpfriends] ident = diff --git a/runner.py b/runner.py index daf29f7..e7fdbf4 100644 --- a/runner.py +++ b/runner.py @@ -55,7 +55,7 @@ def parse_config(config_file): if parser.getboolean('loggly_log', 'enabled'): config['loggly_token'] = parser.get('loggly_log', 'token') - config['mongo_db'] = parser.get('mongodb', 'mongod_database') + config['mongo_db'] = parser.get('mongodb', 'database') config['mongo_host'] = parser.get('mongodb', 'mongod_host') config['mongo_port'] = parser.getint('mongodb', 'mongod_port') From b299d822f80653a097ca5dc91c76fcf561264fc0 Mon Sep 17 00:00:00 2001 From: bars Date: Tue, 12 Apr 2016 14:57:14 +0200 Subject: [PATCH 06/11] added support for authentication (mongodb) --- mnemosyne.cfg.dist | 4 ++++ persistance/mnemodb.py | 14 ++++++++++---- persistance/preagg_reports.py | 11 ++++++++--- runner.py | 13 +++++++++++-- 4 files changed, 33 insertions(+), 9 deletions(-) diff --git a/mnemosyne.cfg.dist b/mnemosyne.cfg.dist index 9efdb84..8c367bb 100644 --- a/mnemosyne.cfg.dist +++ b/mnemosyne.cfg.dist @@ -7,6 +7,10 @@ port = 8181 mongod_host = 127.0.0.1 mongod_port = 27017 database = mnemosyne +mongo_auth = False +mongo_user = user +mongo_password = password +mongo_auth_mechanism = SCRAM-SHA-1 [hpfriends] ident = diff --git a/persistance/mnemodb.py b/persistance/mnemodb.py index ddc025a..db955f8 100644 --- a/persistance/mnemodb.py +++ b/persistance/mnemodb.py @@ -32,11 +32,17 @@ class MnemoDB(object): - def __init__(self, mongo_host, mongo_port, database_name): + def __init__(self, mongo_host, mongo_port, database_name, mongo_user=None, mongo_password=None, mongo_auth_mechanism=None): logger.info('Connecting to mongodb, using "{0}" as database.'.format(database_name)) - conn = MongoClient(host=mongo_host, port=mongo_port, auto_start_request=False) - self.rg = ReportGenerator(mongo_host, mongo_port, database_name) - self.db = conn[database_name] + if mongo_user not None: + conn = MongoClient(host=mongo_host, port=mongo_port, auto_start_request=False) + self.rg = ReportGenerator(mongo_host, mongo_port, database_name, mongo_user, mongo_password,mongo_auth_mechanism) + self.db = conn[database_name] + self.db.authenticate(mongo_user, mongo_password, mechanism=mongo_auth_mechanism) + else: + conn = MongoClient(host=mongo_host, port=mongo_port, auto_start_request=False) + self.rg = ReportGenerator(mongo_host, mongo_port, database_name) + self.db = conn[database_name] self.ensure_index() def ensure_index(self): diff --git a/persistance/preagg_reports.py b/persistance/preagg_reports.py index 4d348a3..bc71bf9 100644 --- a/persistance/preagg_reports.py +++ b/persistance/preagg_reports.py @@ -27,10 +27,15 @@ class ReportGenerator: Generates pre-aggregated reports. """ - def __init__(self, mongo_host, mongo_port, database_name): + def __init__(self, mongo_host, mongo_port, database_name, mongo_user=None, mongo_password=None, mongo_auth_mechanism=None): logger.info('Connecting to mongodb, using "{0}" as database.'.format(database_name)) - conn = MongoClient(host=mongo_host, port=mongo_port, w=0) - self.db = conn[database_name] + if mongo_user not None: + conn = MongoClient(host=mongo_host, port=mongo_port, w=0) + self.db = conn[database_name] + self.db.authenticate(mongo_user, mongo_password, mechanism=mongo_auth_mechanism) + else: + conn = MongoClient(host=mongo_host, port=mongo_port, w=0) + self.db = conn[database_name] def hpfeeds(self, entry): hour = entry['timestamp'].hour diff --git a/runner.py b/runner.py index e7fdbf4..df0eed0 100644 --- a/runner.py +++ b/runner.py @@ -39,7 +39,7 @@ def parse_config(config_file): if not os.path.isfile(config_file): sys.exit("Could not find configuration file: {0}".format(config_file)) - parser = ConfigParser({'mongod_host': '127.0.0.1', 'mognod_port': 27017}) + parser = ConfigParser({'mongod_host': '127.0.0.1', 'mognod_port': 27017, 'mongo_auth': False}) parser.read(config_file) log_file = None @@ -58,6 +58,12 @@ def parse_config(config_file): config['mongo_db'] = parser.get('mongodb', 'database') config['mongo_host'] = parser.get('mongodb', 'mongod_host') config['mongo_port'] = parser.getint('mongodb', 'mongod_port') + + if parser.getboolean('mongodb', 'mongo_auth'): + config['mongo_auth'] = True + config['mongo_user'] = parser.get('mongodb', 'mongod_user') + config['mongo_password'] = parser.get('mongodb', 'mongod_password') + config['mongo_auth_mechanism'] = parser.get('mongodb', 'mongo_auth_mechanism') config['hpf_feeds'] = parser.get('hpfriends', 'channels').split(',') config['hpf_ident'] = parser.get('hpfriends', 'ident') @@ -115,7 +121,10 @@ def do_logging(file_log=None, loggly_token=None): greenlets = {} - db = mnemodb.MnemoDB(c['mongo_host'], c['mongo_port'], c['mongo_db']) + if config['mongo_auth']: + db = mnemodb.MnemoDB(c['mongo_host'], c['mongo_port'], c['mongo_db'], c['mongo_user'], c['mongo_password'], c['mongo_auth_mechanism']) + else: + db = mnemodb.MnemoDB(c['mongo_host'], c['mongo_port'], c['mongo_db']) webapi = None hpfriends_puller = None From 37f407fcad9bf1d3c43d724e5b8cd1d0b04a6294 Mon Sep 17 00:00:00 2001 From: bars Date: Fri, 29 Apr 2016 10:57:13 +0200 Subject: [PATCH 07/11] bug fix --- runner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/runner.py b/runner.py index df0eed0..ed12877 100644 --- a/runner.py +++ b/runner.py @@ -59,6 +59,8 @@ def parse_config(config_file): config['mongo_host'] = parser.get('mongodb', 'mongod_host') config['mongo_port'] = parser.getint('mongodb', 'mongod_port') + + config['mongo_auth'] = False if parser.getboolean('mongodb', 'mongo_auth'): config['mongo_auth'] = True config['mongo_user'] = parser.get('mongodb', 'mongod_user') From 04212038c5ec323218269e4a9ec05c7be22e846b Mon Sep 17 00:00:00 2001 From: bars Date: Wed, 25 May 2016 14:37:11 +0200 Subject: [PATCH 08/11] bug fix --- mnemosyne.cfg.dist | 8 ++++---- persistance/mnemodb.py | 2 +- persistance/preagg_reports.py | 2 +- runner.py | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mnemosyne.cfg.dist b/mnemosyne.cfg.dist index 8c367bb..6c89ebc 100644 --- a/mnemosyne.cfg.dist +++ b/mnemosyne.cfg.dist @@ -7,10 +7,10 @@ port = 8181 mongod_host = 127.0.0.1 mongod_port = 27017 database = mnemosyne -mongo_auth = False -mongo_user = user -mongo_password = password -mongo_auth_mechanism = SCRAM-SHA-1 +mongod_auth = False +mongod_user = user +mongod_password = password +mongod_auth_mechanism = SCRAM-SHA-1 [hpfriends] ident = diff --git a/persistance/mnemodb.py b/persistance/mnemodb.py index db955f8..50786ed 100644 --- a/persistance/mnemodb.py +++ b/persistance/mnemodb.py @@ -34,7 +34,7 @@ class MnemoDB(object): def __init__(self, mongo_host, mongo_port, database_name, mongo_user=None, mongo_password=None, mongo_auth_mechanism=None): logger.info('Connecting to mongodb, using "{0}" as database.'.format(database_name)) - if mongo_user not None: + if mongo_user is not None: conn = MongoClient(host=mongo_host, port=mongo_port, auto_start_request=False) self.rg = ReportGenerator(mongo_host, mongo_port, database_name, mongo_user, mongo_password,mongo_auth_mechanism) self.db = conn[database_name] diff --git a/persistance/preagg_reports.py b/persistance/preagg_reports.py index bc71bf9..7b8c2f3 100644 --- a/persistance/preagg_reports.py +++ b/persistance/preagg_reports.py @@ -29,7 +29,7 @@ class ReportGenerator: def __init__(self, mongo_host, mongo_port, database_name, mongo_user=None, mongo_password=None, mongo_auth_mechanism=None): logger.info('Connecting to mongodb, using "{0}" as database.'.format(database_name)) - if mongo_user not None: + if mongo_user is not None: conn = MongoClient(host=mongo_host, port=mongo_port, w=0) self.db = conn[database_name] self.db.authenticate(mongo_user, mongo_password, mechanism=mongo_auth_mechanism) diff --git a/runner.py b/runner.py index ed12877..fbf1ebb 100644 --- a/runner.py +++ b/runner.py @@ -61,11 +61,11 @@ def parse_config(config_file): config['mongo_auth'] = False - if parser.getboolean('mongodb', 'mongo_auth'): + if parser.getboolean('mongodb', 'mongod_auth'): config['mongo_auth'] = True config['mongo_user'] = parser.get('mongodb', 'mongod_user') config['mongo_password'] = parser.get('mongodb', 'mongod_password') - config['mongo_auth_mechanism'] = parser.get('mongodb', 'mongo_auth_mechanism') + config['mongo_auth_mechanism'] = parser.get('mongodb', 'mongod_auth_mechanism') config['hpf_feeds'] = parser.get('hpfriends', 'channels').split(',') config['hpf_ident'] = parser.get('hpfriends', 'ident') @@ -123,7 +123,7 @@ def do_logging(file_log=None, loggly_token=None): greenlets = {} - if config['mongo_auth']: + if c['mongo_auth']: db = mnemodb.MnemoDB(c['mongo_host'], c['mongo_port'], c['mongo_db'], c['mongo_user'], c['mongo_password'], c['mongo_auth_mechanism']) else: db = mnemodb.MnemoDB(c['mongo_host'], c['mongo_port'], c['mongo_db']) From ee40faf6f7e34c2dba9b5f5584273da4d05235d4 Mon Sep 17 00:00:00 2001 From: bars Date: Wed, 24 Aug 2016 15:31:22 +0200 Subject: [PATCH 09/11] using env variables instead of using config file for mongodb settings --- mnemosyne.cfg.dist | 6 ------ runner.py | 27 ++++++++++++++++----------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/mnemosyne.cfg.dist b/mnemosyne.cfg.dist index 45d1e8d..5109e85 100644 --- a/mnemosyne.cfg.dist +++ b/mnemosyne.cfg.dist @@ -4,13 +4,7 @@ host = 0.0.0.0 port = 8181 [mongodb] -mongod_host = 127.0.0.1 -mongod_port = 27017 database = mnemosyne -mongod_auth = False -mongod_user = user -mongod_password = password -mongod_auth_mechanism = SCRAM-SHA-1 [hpfriends] ident = diff --git a/runner.py b/runner.py index fbf1ebb..2c4a1df 100644 --- a/runner.py +++ b/runner.py @@ -39,7 +39,7 @@ def parse_config(config_file): if not os.path.isfile(config_file): sys.exit("Could not find configuration file: {0}".format(config_file)) - parser = ConfigParser({'mongod_host': '127.0.0.1', 'mognod_port': 27017, 'mongo_auth': False}) + parser = ConfigParser() parser.read(config_file) log_file = None @@ -56,16 +56,21 @@ def parse_config(config_file): config['loggly_token'] = parser.get('loggly_log', 'token') config['mongo_db'] = parser.get('mongodb', 'database') - config['mongo_host'] = parser.get('mongodb', 'mongod_host') - config['mongo_port'] = parser.getint('mongodb', 'mongod_port') - - - config['mongo_auth'] = False - if parser.getboolean('mongodb', 'mongod_auth'): - config['mongo_auth'] = True - config['mongo_user'] = parser.get('mongodb', 'mongod_user') - config['mongo_password'] = parser.get('mongodb', 'mongod_password') - config['mongo_auth_mechanism'] = parser.get('mongodb', 'mongod_auth_mechanism') + + if os.getenv("REMOTE_MONGO"): + config['mongo_host'] = os.getenv("MONGO_HOST") + config['mongo_port'] = os.getenv("MONGO_PORT") + + config['mongo_auth'] = False + if os.getenv("MONGO_AUTH"): + config['mongo_auth'] = True + config['mongo_user'] = os.getenv("MONGO_USER") + config['mongo_password'] = os.getenv("MONGO_PASSWORD") + config['mongo_auth_mechanism'] = os.getenv("MONGO_AUTH_MECHANISM") + else: + config['mongo_auth'] = False + config['mongo_host'] = "127.0.0.1" + config['mongo_port'] = 27017 config['hpf_feeds'] = parser.get('hpfriends', 'channels').split(',') config['hpf_ident'] = parser.get('hpfriends', 'ident') From 9ac99e1f2cce9f65dc393dd86cc8c04ba6ba929f Mon Sep 17 00:00:00 2001 From: bars Date: Thu, 1 Sep 2016 14:50:17 +0200 Subject: [PATCH 10/11] bug fix --- runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runner.py b/runner.py index 2c4a1df..46b04b6 100644 --- a/runner.py +++ b/runner.py @@ -57,12 +57,12 @@ def parse_config(config_file): config['mongo_db'] = parser.get('mongodb', 'database') - if os.getenv("REMOTE_MONGO"): + if os.getenv("REMOTE_MONGO") == "true": config['mongo_host'] = os.getenv("MONGO_HOST") config['mongo_port'] = os.getenv("MONGO_PORT") config['mongo_auth'] = False - if os.getenv("MONGO_AUTH"): + if os.getenv("MONGO_AUTH") == "true": config['mongo_auth'] = True config['mongo_user'] = os.getenv("MONGO_USER") config['mongo_password'] = os.getenv("MONGO_PASSWORD") From b29ba63a2b973fa9782a71965e1328989c9b338f Mon Sep 17 00:00:00 2001 From: bars Date: Thu, 1 Sep 2016 17:10:46 +0200 Subject: [PATCH 11/11] convert MONGO_PORT variable to int value --- runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runner.py b/runner.py index 46b04b6..36d7591 100644 --- a/runner.py +++ b/runner.py @@ -59,7 +59,7 @@ def parse_config(config_file): if os.getenv("REMOTE_MONGO") == "true": config['mongo_host'] = os.getenv("MONGO_HOST") - config['mongo_port'] = os.getenv("MONGO_PORT") + config['mongo_port'] = int(os.getenv("MONGO_PORT")) config['mongo_auth'] = False if os.getenv("MONGO_AUTH") == "true":