From af4f9b9791a530de84bba34d76e0cf31f62d37df Mon Sep 17 00:00:00 2001 From: Ken Kundert Date: Wed, 8 Jan 2025 12:07:35 -0800 Subject: [PATCH] Complete hooks code and documentation. Add monitoring, command_aliases, and overdue to do_not_expand by default.o Support codicil when reporting setting errors. Refine hooks code: - fix bugs - add place holders Document new monitoring services settings. Fixed bug in overdue. --- assimilate/assimilate.py | 2 +- assimilate/configs.py | 4 +- assimilate/hooks.py | 49 +++++++++---- assimilate/overdue.py | 5 +- doc/configuring.rst | 10 +++ doc/monitoring.rst | 147 ++++++++++++++++++++++++++++++++++++--- tests/assimilate.nt | 7 +- 7 files changed, 190 insertions(+), 34 deletions(-) diff --git a/assimilate/assimilate.py b/assimilate/assimilate.py index 622efb3..262cfdc 100644 --- a/assimilate/assimilate.py +++ b/assimilate/assimilate.py @@ -268,7 +268,7 @@ def __init__(self, config=None, assimilate_opts=None, shared_settings=None, **kw warn(f'unknown colorscheme: {self.colorscheme}.') # determine the do_not_expand list - do_not_expand = set() + do_not_expand = set(['monitoring', 'command_aliases', 'overdue']) for key, value in ASSIMILATE_SETTINGS.items(): if value.get('do_not_expand'): do_not_expand.add(key) diff --git a/assimilate/configs.py b/assimilate/configs.py index 0092a6d..4744d95 100644 --- a/assimilate/configs.py +++ b/assimilate/configs.py @@ -697,13 +697,13 @@ def get_available_configs(keep_shared=False): # report_setting_error() {{{2 keymaps = defaultdict(dict) -def report_setting_error(keys, error): +def report_setting_error(keys, error, codicil=None): paths = reversed(keymaps.keys()) for path in paths: keymap = keymaps[path] loc = keymap.get(keys) if loc: - raise Error(error, culprit=(path,)+keys, codicil=loc.as_line()) + raise Error(error, culprit=(path,)+keys, codicil=(codicil, loc.as_line())) raise AssertionError # this should not happen with a user specified value diff --git a/assimilate/hooks.py b/assimilate/hooks.py index 76e860a..711bb0c 100644 --- a/assimilate/hooks.py +++ b/assimilate/hooks.py @@ -18,7 +18,7 @@ # Imports {{{1 -from inform import Error, full_stop, is_str, log, os_error +from inform import Error, conjoin, full_stop, is_str, log, os_error, truth from .configs import add_setting, as_string, as_dict, report_setting_error from voluptuous import Any, Invalid, Schema import requests @@ -121,12 +121,12 @@ class Custom(Hooks): def __init__(self, assimilate_settings): settings = self.get_settings(assimilate_settings) - placeholders = {} + placeholders = dict(config=assimilate_settings.config_name) if 'id' in settings: - placeholders['id'] = settings['id'] + placeholders['id'] = settings['id'].strip() try: if 'url' in settings: - placeholders['url'] = settings['url'].format(**placeholders) + placeholders['url'] = settings['url'].format(**placeholders).strip() except TypeError as e: self.invalid_key('url', e) self.placeholders = placeholders @@ -143,29 +143,36 @@ def invalid_key(self, keys, e): error = 'unknown key: ‘{key}’' self.report_error(keys, error) - def report_error(self, keys, error): + def report_error(self, keys, error, codicil=None): if is_str(keys): keys = (keys,) keys = (Hooks.NAME, self.NAME) + keys - report_setting_error(keys, error) + report_setting_error(keys, error, codicil) def expand_value(self, keys, placeholders): value = self.settings for key in keys: + if key not in value: + return value = value[key] - def expand_str(value): + def expand_str(keys, value): try: return value.format(**placeholders) except TypeError as e: - self.invalid_key(e, keys) + self.invalid_key(keys, e) + except KeyError as e: + self.report_error( + keys, f"unknown key: {e.args[0]}", + f"Choose from {conjoin(placeholders.keys(), conj=' or ')}." + ) if is_str(value): - return expand_str(value) + return expand_str(keys, value) else: data = {} - for k, v in values.items(): - data[k] = expand_str(keys + (k,), placeholders) + for k, v in value.items(): + data[k] = expand_str(keys + (k,), v) return data def report(self, name, placeholders): @@ -197,7 +204,7 @@ def report(self, name, placeholders): except Invalid: self.report_error((), 'invalid url.') - log(f'signaling {name} of backups to {self.NAME}: {url}.') + log(f'signaling {name} of backups to {self.NAME}: {url} via {method}.') try: if method == 'get': requests.get(url, params=params) @@ -216,16 +223,28 @@ def signal_end(self, exception): names = ['success', 'finish'] placeholders = self.placeholders.copy() + placeholders['error'] = '' + placeholders['stderr'] = '' + placeholders['stdout'] = '' if exception: if isinstance(exception, OSError): placeholders['error'] = os_error(exception) - placeholders['exit_status'] = "2" + placeholders['status'] = "2" + elif isinstance(exception, KeyboardInterrupt): + placeholders['error'] = "Killed by user." + placeholders['status'] = "2" else: placeholders['error'] = str(exception) - placeholders['exit_status'] = str(getattr(exception, 'status', 2)) + placeholders['status'] = str(getattr(exception, 'status', 2)) placeholders['stderr'] = getattr(exception, 'stderr', '') + placeholders['stdout'] = getattr(exception, 'stdout', '') + elif self.borg: + placeholders['status'] = str(self.borg.status) + placeholders['stderr'] = self.borg.stderr + placeholders['stdout'] = self.borg.stdout else: - placeholders['exit_status'] = '0' + placeholders['status'] = '0' + placeholders['success'] = truth(placeholders['status'] in '01') for name in names: self.report(name, placeholders) diff --git a/assimilate/overdue.py b/assimilate/overdue.py index 4ccf455..cdc986e 100644 --- a/assimilate/overdue.py +++ b/assimilate/overdue.py @@ -160,11 +160,12 @@ def get_local_data(description, config, path, max_age): if config: if not path: path = to_path(DATA_DIR) - latest = read_latest(path / f"{config}.latest.nt") + locked = (path / f"{config}.lock").exists() + path = path / f"{config}.latest.nt" + latest = read_latest(path) mtime = latest.get('create last run') if not mtime: raise Error('create time is not available.', culprit=path) - locked = (path / f"{config}.lock").exists() else: if not path: raise Error("‘sentinel_dir’ setting is required.", culprit=description) diff --git a/doc/configuring.rst b/doc/configuring.rst index 54a21f2..9b91c5c 100644 --- a/doc/configuring.rst +++ b/doc/configuring.rst @@ -1072,6 +1072,16 @@ a large number of log entries are kept. It is recommended that you specify a reasonable value for *max_entries*. +.. _monitoring setting: + +monitoring +~~~~~~~~~~ + +*monitoring* is a dictionary setting that configures status reporting to +monitoring services. Detailed information about this setting can be found in +:ref:`monitoring `. + + .. _must_exist: must_exist diff --git a/doc/monitoring.rst b/doc/monitoring.rst index 623ac43..6333ada 100644 --- a/doc/monitoring.rst +++ b/doc/monitoring.rst @@ -427,10 +427,137 @@ Various monitoring services are available on the web. You can configure services allow you to monitor many of your routine tasks and assure they have completed recently and successfully. -There are many such services available and they are not difficult to add. If -the service you prefer is not currently available, feel free to request it on -`Github `_ or add it yourself -and issue a pull request. +There are many such services available and they are not difficult to add. There +is built-in support for a few common services. For others you can use the +*custom* service. It can handle most web-based services. If the service you +prefer is not currently available and cannot be supported as a custom service, +feel free to request it on `Github +`_ or add it yourself and issue +a pull request. + +.. _custom monitoring service: + +Custom +~~~~~~ + +You can configure *Assimilate* to send custom web-based messages to your +monitoring service when backing up. You can configure four different types of +messages: *start*, *success*, *failure* and *finish*. These messages are sent +as follows: + +start: + When the backup begins. + +success: + When the backup completes, but only if there were no errors. + +failure: + When the backup completes, but only if there were errors. + +finish: + When the backup completes. + +Generally you do not configure all of them as they are redundant. Specifically, +you would configure *success* and *failure* together, or you would configure +*finish* in such a way as to indicate whether the backup succeeded. For +example, here is how one might configure HealthCheck.io using the custom +service: + +.. code-block:: nestedtext + + monitoring: + custom: + id: 51cb35d8-2975-110b-67a7-11b65d432027 + url: https://hc-ping.com/{id} + start: {url}/start + success: {url}/0 + failure: + url: {url}/fail + post: + > CONFIG: {config} + > EXIT STATUS: {status} + > ERROR: {error} + > + > STDOUT: + > {stdout} + > + > STDERR: + > {stderr} + +In this example *start*, *success* and *failure* were configured. With each, you +can simply specify a URL, or you can specify key-value pairs that control the +message that is sent. Furthermore, you can specify *id* and *url* up-front and +simply refer to them in when configuring your message. For example, *start* is +specified as ``{url}/start``. Here ``{url}`` is replaced by the value you +specified earlier for *url*. Similarly, *success* is specified only by its URL, +``{url}/0``. But, *fail* is given as a collection of key-value pairs. Three +keys are allowed: *url*, *params*, and *post*. *url* is required, but the other +two are optional. *params* is a collection of key-value pairs. These will be +passed in the URL as parameters. Specify *params* only if your service requires +them. *post* indicates that the *post* method is to be used, otherwise the +*get* method is used. The value of post may be a string, or a collection of +key-value pairs. You would specify both *params* and *post* to conform with the +requirements of your service. + +When constructing your messages you can insert placeholders that are replaced +before sending the message. The available placeholders are: + +config: + The name of the config being backed up. + +status: + The exit status of the *Borg* process performing the backup. + +error: + A short message that describes error that occurred during the backup if + appropriate. + +stdout: + The text sent to the standard output by the *Borg* process performing the + backup. + +stderr: + The text sent to the standard error output by the *Borg* process performing + the backup. + +id: + The value specified as *id*. + +url: + The value specified as *url*. + +success: + A Boolean `truth object `_ + that evaluates to *yes* if *Borg* returned with an exit status of 0 or 1, + which implies that the command completed as expected, though there might + have been issues with individual files or directories. If *Borg* returns + with an exit status of 2 or greater, *success* evaluates to *no*. However, + you can specify different values by specifying a format string. For + example, the above example specifies both *success* and *failure*, but this + could be collapsed to using only *finish* by using *success* to modify the + URL used to communicate the completion message. In this example, the URL is + specified as ``{url}/{success:0/fail}``. Here ``{success:0/fail}`` + evaluates to ``0`` if *Borg* succeeds and ``fail`` otherwise. + +.. code-block:: nestedtext + + monitoring: + custom: + id: 51cb35d8-2975-110b-67a7-11b65d432027 + url: https://hc-ping.com/{id} + start: {url}/start + finish: + url: {url}/{success:0/fail} + post: + > CONFIG: {config} + > EXIT STATUS: {status} + > ERROR: {error} + > + > STDOUT: + > {stdout} + > + > STDERR: + > {stderr} .. _cronhub: @@ -442,9 +569,11 @@ health check for your *Assimilate* configuration, you will be given a UUID (a 32 digit hexadecimal number partitioned into 5 parts by dashes). Add that to the following setting in your configuration file: -.. code-block:: python +.. code-block:: nestedtext - cronhub_uuid = '51cb35d8-2975-110b-67a7-11b65d432027' + monitoring: + cronhub.io: + uuid: 51cb35d8-2975-110b-67a7-11b65d432027 If given, this setting should be specified on an individual configuration. It causes a report to be sent to *CronHub* each time an archive is created. @@ -464,9 +593,11 @@ the health check for your *Assimilate* configuration, you will be given a UUID (a 32 digit hexadecimal number partitioned into 5 parts by dashes). Add that to the following setting in your configuration file: -.. code-block:: python +.. code-block:: nestedtext - healthchecks_uuid = '51cb35d8-2975-110b-67a7-11b65d432027' + monitoring: + healthchecks.io: + uuid: 51cb35d8-2975-110b-67a7-11b65d432027 If given, this setting should be specified on an individual configuration. It causes a report to be sent to *HealthChecks* each time an archive is created. diff --git a/tests/assimilate.nt b/tests/assimilate.nt index 4549a8a..9f63d70 100644 --- a/tests/assimilate.nt +++ b/tests/assimilate.nt @@ -629,8 +629,6 @@ assimilate: > create_retries: number of times to retry a create if failures > occur > create_retry_sleep: time to sleep between retries [s] - > cronhub_url: the cronhub.io URL for back-ups monitor - > cronhub_uuid: the cronhub.io UUID for back-ups monitor > default_config: default Assimilate configuration > default_list_format: the format that the list command should use if > none is specified @@ -643,10 +641,6 @@ assimilate: > exclude_from: file that contains exclude patterns > excludes: list of glob strings of files or directories > to skip - > healthchecks_url: the healthchecks.io URL for monitoring back - > ups - > healthchecks_uuid: the healthchecks.io UUID for monitoring back - > ups > home_dir: users home directory (read only) > include: include the contents of another file > list_formats: named format strings available to list command @@ -654,6 +648,7 @@ assimilate: > logging: logging options > manage_diffs_cmd: command to use to manage differences in files > and directories + > monitoring: services to notify upon backup > must_exist: if set, each of these files or directories > must exist or create will quit with an error > needs_ssh_agent: when set Assimilate complains if ssh_agent is