diff --git a/assimilate/configs.py b/assimilate/configs.py index bf778a6..0092a6d 100644 --- a/assimilate/configs.py +++ b/assimilate/configs.py @@ -21,8 +21,9 @@ from .preferences import CONFIG_DIR from .shlib import lsf, to_path, chmod, getmod from .utilities import report_voluptuous_errors +from collections import defaultdict from inform import ( - codicil, conjoin, is_str, join, narrate, warn, terminate, truth + Error, codicil, conjoin, is_str, join, narrate, warn, terminate, truth ) from quantiphy import Quantity, InvalidNumber from voluptuous import Schema, Invalid, MultipleInvalid, Extra @@ -694,11 +695,23 @@ def get_available_configs(keep_shared=False): return available_configs return {k:v for k, v in available_configs.items() if k != 'shared'} +# report_setting_error() {{{2 +keymaps = defaultdict(dict) +def report_setting_error(keys, error): + paths = reversed(keymaps.keys()) + for path in paths: + keymap = keymaps[path] + loc = keymap.get(keys) + if loc: + raise Error(error, culprit=(path,)+keys, codicil=loc.as_line()) + raise AssertionError # this should not happen with a user specified value + + # read_config() {{{2 def read_config(path, validate_settings): # read a file and recursively process includes try: - keymap = {} + keymap = keymaps[str(path)] settings = nt.load( path, top=dict, keymap=keymap, normalize_key=normalize_key ) diff --git a/assimilate/hooks.py b/assimilate/hooks.py index ec2a905..76e860a 100644 --- a/assimilate/hooks.py +++ b/assimilate/hooks.py @@ -18,17 +18,46 @@ # Imports {{{1 -from inform import Error, full_stop, log, os_error -from .configs import add_setting, as_string +from inform import Error, full_stop, is_str, log, os_error +from .configs import add_setting, as_string, as_dict, report_setting_error +from voluptuous import Any, Invalid, Schema import requests +# Schema {{{1 +# as_url() {{{2 +def as_url(arg): + as_string(arg) + from urllib.parse import urlparse + url = urlparse(arg) + if url.scheme not in ['http', 'https'] or not url.hostname: + raise Invalid('invalid url.') + return arg + +# as_action() {{{2 +as_action = Any( + as_string, + dict(url=as_string, params=as_dict, post=Any(as_string, as_dict)) +) + +# schema {{{2 +schema = {} + # Hooks base class {{{1 class Hooks: + NAME = "monitoring" + @classmethod def provision_hooks(cls): + schema = {} + for subclass in cls.__subclasses__(): - for k, v in subclass.ASSIMILATE_SETTINGS.items(): - add_setting(k, desc=v, validator=as_string) + schema[subclass.NAME] = subclass.VALIDATOR + + add_setting( + name = cls.NAME, + desc = "services to notify upon backup", + validator = Schema(schema) + ) def __init__(self, settings): self.active_hooks = [] @@ -37,6 +66,12 @@ def __init__(self, settings): if c.is_active(): self.active_hooks.append(c) + def get_settings(self, assimilate_settings): + monitoring = assimilate_settings.monitoring + if monitoring: + return monitoring.get(self.NAME, {}) + return {} + def report_results(self, borg): for hook in self.active_hooks: hook.borg = borg @@ -50,9 +85,6 @@ def __exit__(self, exc_type, exc_value, exc_traceback): for hook in self.active_hooks: hook.signal_end(exc_value) - def is_active(self): - return bool(self.uuid) - def signal_start(self): url = self.START_URL.format(url=self.url, uuid=self.uuid) log(f'signaling start of backups to {self.NAME}: {url}.') @@ -75,22 +107,147 @@ def signal_end(self, exception): raise Error('{self.NAME} connection error.', codicil=full_stop(e)) +# Custom class {{{1 +class Custom(Hooks): + NAME = 'custom' + VALIDATOR = dict( + id = as_string, + url = as_url, + start = as_action, + success = as_action, + failure = as_action, + finish = as_action + ) + + def __init__(self, assimilate_settings): + settings = self.get_settings(assimilate_settings) + placeholders = {} + if 'id' in settings: + placeholders['id'] = settings['id'] + try: + if 'url' in settings: + placeholders['url'] = settings['url'].format(**placeholders) + except TypeError as e: + self.invalid_key('url', e) + self.placeholders = placeholders + self.settings = settings + self.borg = None + + def is_active(self): + return bool(self.settings) + + def invalid_key(self, keys, e): + # unfortunately TypeErrors must be de-parsed to determine the key + _, _, key = str(e).partition("'") + key = key=key[:-1] + error = 'unknown key: ‘{key}’' + self.report_error(keys, error) + + def report_error(self, keys, error): + if is_str(keys): + keys = (keys,) + keys = (Hooks.NAME, self.NAME) + keys + report_setting_error(keys, error) + + def expand_value(self, keys, placeholders): + value = self.settings + for key in keys: + value = value[key] + + def expand_str(value): + try: + return value.format(**placeholders) + except TypeError as e: + self.invalid_key(e, keys) + + if is_str(value): + return expand_str(value) + else: + data = {} + for k, v in values.items(): + data[k] = expand_str(keys + (k,), placeholders) + return data + + def report(self, name, placeholders): + if not self.settings: + return + + reporter = self.settings.get(name) + if not reporter: + return + + # process reporter + method = 'get' + if is_str(reporter): + url = self.expand_value((name,), placeholders) + params = {} + else: + url = self.expand_value((name, 'url'), placeholders) + params = self.expand_value((name, 'params'), placeholders) + if 'post' in reporter: + method = 'post' + data = self.expand_value( + (name, 'post'), placeholders + ) + + if not url: + self.report_error(name, 'missing url.') + try: + as_url(url) + except Invalid: + self.report_error((), 'invalid url.') + + log(f'signaling {name} of backups to {self.NAME}: {url}.') + try: + if method == 'get': + requests.get(url, params=params) + else: + requests.post(url, params=params, data=data) + except requests.exceptions.RequestException as e: + raise Error('{self.NAME} connection error.', codicil=full_stop(e)) + + def signal_start(self): + self.report('start', self.placeholders) + + def signal_end(self, exception): + if exception: + names = ['failure', 'finish'] + else: + names = ['success', 'finish'] + + placeholders = self.placeholders.copy() + if exception: + if isinstance(exception, OSError): + placeholders['error'] = os_error(exception) + placeholders['exit_status'] = "2" + else: + placeholders['error'] = str(exception) + placeholders['exit_status'] = str(getattr(exception, 'status', 2)) + placeholders['stderr'] = getattr(exception, 'stderr', '') + else: + placeholders['exit_status'] = '0' + + for name in names: + self.report(name, placeholders) + + # HealthChecks class {{{1 class HealthChecks(Hooks): NAME = 'healthchecks.io' - ASSIMILATE_SETTINGS = dict( - healthchecks_url = 'the healthchecks.io URL for monitoring back ups', - healthchecks_uuid = 'the healthchecks.io UUID for monitoring back ups', - ) + VALIDATOR = dict(url=as_url, uuid=as_string) URL = 'https://hc-ping.com' - def __init__(self, settings): - self.uuid = settings.healthchecks_uuid - self.url = settings.healthchecks_url + def __init__(self, assimilate_settings): + settings = self.get_settings(assimilate_settings) + self.uuid = settings.get('uuid') + self.url = settings.get('url') if not self.url: self.url = self.URL self.borg = None + def is_active(self): + return bool(self.uuid) + def signal_start(self): url = f'{self.url}/{self.uuid}/start' log(f'signaling start of backups to {self.NAME}: {url}.') @@ -135,17 +292,18 @@ def signal_end(self, exception): # CronHub class {{{1 class CronHub(Hooks): NAME = 'cronhub.io' - ASSIMILATE_SETTINGS = dict( - cronhub_uuid = 'the cronhub.io UUID for back-ups monitor', - cronhub_url = 'the cronhub.io URL for back-ups monitor', - ) + VALIDATOR = dict(url=as_url, uuid=as_string) START_URL = '{url}/start/{uuid}' SUCCESS_URL = '{url}/finish/{uuid}' FAIL_URL = '{url}/fail/{uuid}' URL = 'https://cronhub.io' - def __init__(self, settings): - self.uuid = settings.cronhub_uuid - self.url = settings.cronhub_url + def __init__(self, assimilate_settings): + settings = self.get_settings(assimilate_settings) + self.uuid = settings.get('uuid') + self.url = settings.get('url') if not self.url: self.url = self.URL + + def is_active(self): + return bool(self.uuid) diff --git a/assimilate/main.py b/assimilate/main.py index c5c6890..cb45d02 100644 --- a/assimilate/main.py +++ b/assimilate/main.py @@ -109,9 +109,9 @@ def main(): if cmdline["--narrate"]: inform.narrate = True - shared_settings = read_settings('shared') - + # read shared settings Hooks.provision_hooks() + shared_settings = read_settings('shared') # find the command cmd, cmd_name, alias_args = Command.find(command, shared_settings) diff --git a/assimilate/overdue.py b/assimilate/overdue.py index 7c8226f..4ccf455 100644 --- a/assimilate/overdue.py +++ b/assimilate/overdue.py @@ -160,12 +160,11 @@ def get_local_data(description, config, path, max_age): if config: if not path: path = to_path(DATA_DIR) - path = path / f"{config}.latest.nt" - latest = read_latest(path) - locked = (path / f"{config}.lock").exists() + latest = read_latest(path / f"{config}.latest.nt") mtime = latest.get('create last run') if not mtime: raise Error('create time is not available.', culprit=path) + locked = (path / f"{config}.lock").exists() else: if not path: raise Error("‘sentinel_dir’ setting is required.", culprit=description)