Complete hooks code and documentation.

Add monitoring, command_aliases, and overdue to do_not_expand by default.o
Support codicil when reporting setting errors.
Refine hooks code:
- fix bugs
- add place holders
Document new monitoring services settings.
Fixed bug in overdue.

assimilate/assimilate.py

@@ -268,7 +268,7 @@ def __init__(self, config=None, assimilate_opts=None, shared_settings=None, **kw
                 warn(f'unknown colorscheme: {self.colorscheme}.')
 
         # determine the do_not_expand list
-        do_not_expand = set()
+        do_not_expand = set(['monitoring', 'command_aliases', 'overdue'])
         for key, value in ASSIMILATE_SETTINGS.items():
             if value.get('do_not_expand'):
                 do_not_expand.add(key)

assimilate/configs.py

@@ -697,13 +697,13 @@ def get_available_configs(keep_shared=False):
 
 # report_setting_error() {{{2
 keymaps = defaultdict(dict)
-def report_setting_error(keys, error):
+def report_setting_error(keys, error, codicil=None):
     paths = reversed(keymaps.keys())
     for path in paths:
         keymap = keymaps[path]
         loc = keymap.get(keys)
         if loc:
-            raise Error(error, culprit=(path,)+keys, codicil=loc.as_line())
+            raise Error(error, culprit=(path,)+keys, codicil=(codicil, loc.as_line()))
     raise AssertionError  # this should not happen with a user specified value
 
 

assimilate/hooks.py

@@ -18,7 +18,7 @@
 
 
 # Imports {{{1
-from inform import Error, full_stop, is_str, log, os_error
+from inform import Error, conjoin, full_stop, is_str, log, os_error, truth
 from .configs import add_setting, as_string, as_dict, report_setting_error
 from voluptuous import Any, Invalid, Schema
 import requests
@@ -121,12 +121,12 @@ class Custom(Hooks):
 
     def __init__(self, assimilate_settings):
         settings = self.get_settings(assimilate_settings)
-        placeholders = {}
+        placeholders = dict(config=assimilate_settings.config_name)
         if 'id' in settings:
-            placeholders['id'] = settings['id']
+            placeholders['id'] = settings['id'].strip()
         try:
             if 'url' in settings:
-                placeholders['url'] = settings['url'].format(**placeholders)
+                placeholders['url'] = settings['url'].format(**placeholders).strip()
         except TypeError as e:
             self.invalid_key('url', e)
         self.placeholders = placeholders
@@ -143,29 +143,36 @@ def invalid_key(self, keys, e):
         error = 'unknown key: ‘{key}’'
         self.report_error(keys, error)
 
-    def report_error(self, keys, error):
+    def report_error(self, keys, error, codicil=None):
         if is_str(keys):
             keys = (keys,)
         keys = (Hooks.NAME, self.NAME) + keys
-        report_setting_error(keys, error)
+        report_setting_error(keys, error, codicil)
 
     def expand_value(self, keys, placeholders):
         value = self.settings
         for key in keys:
+            if key not in value:
+                return
             value = value[key]
 
-        def expand_str(value):
+        def expand_str(keys, value):
             try:
                 return value.format(**placeholders)
             except TypeError as e:
-                self.invalid_key(e, keys)
+                self.invalid_key(keys, e)
+            except KeyError as e:
+                self.report_error(
+                    keys, f"unknown key: {e.args[0]}",
+                    f"Choose from {conjoin(placeholders.keys(), conj=' or ')}."
+                )
 
         if is_str(value):
-            return expand_str(value)
+            return expand_str(keys, value)
         else:
             data = {}
-            for k, v in values.items():
-                data[k] = expand_str(keys + (k,), placeholders)
+            for k, v in value.items():
+                data[k] = expand_str(keys + (k,), v)
             return data
 
     def report(self, name, placeholders):
@@ -197,7 +204,7 @@ def report(self, name, placeholders):
         except Invalid:
             self.report_error((), 'invalid url.')
 
-        log(f'signaling {name} of backups to {self.NAME}: {url}.')
+        log(f'signaling {name} of backups to {self.NAME}: {url} via {method}.')
         try:
             if method == 'get':
                 requests.get(url, params=params)
@@ -216,16 +223,28 @@ def signal_end(self, exception):
             names = ['success', 'finish']
 
         placeholders = self.placeholders.copy()
+        placeholders['error'] = ''
+        placeholders['stderr'] = ''
+        placeholders['stdout'] = ''
         if exception:
             if isinstance(exception, OSError):
                 placeholders['error'] = os_error(exception)
-                placeholders['exit_status'] = "2"
+                placeholders['status'] = "2"
+            elif isinstance(exception, KeyboardInterrupt):
+                placeholders['error'] = "Killed by user."
+                placeholders['status'] = "2"
             else:
                 placeholders['error'] = str(exception)
-                placeholders['exit_status'] = str(getattr(exception, 'status', 2))
+                placeholders['status'] = str(getattr(exception, 'status', 2))
                 placeholders['stderr'] = getattr(exception, 'stderr', '')
+                placeholders['stdout'] = getattr(exception, 'stdout', '')
+        elif self.borg:
+            placeholders['status'] = str(self.borg.status)
+            placeholders['stderr'] = self.borg.stderr
+            placeholders['stdout'] = self.borg.stdout
         else:
-            placeholders['exit_status'] = '0'
+            placeholders['status'] = '0'
+        placeholders['success'] = truth(placeholders['status'] in '01')
 
         for name in names:
             self.report(name, placeholders)

assimilate/overdue.py

@@ -160,11 +160,12 @@ def get_local_data(description, config, path, max_age):
     if config:
         if not path:
             path = to_path(DATA_DIR)
-        latest = read_latest(path / f"{config}.latest.nt")
+        locked = (path /  f"{config}.lock").exists()
+        path = path / f"{config}.latest.nt"
+        latest = read_latest(path)
         mtime = latest.get('create last run')
         if not mtime:
             raise Error('create time is not available.', culprit=path)
-        locked = (path /  f"{config}.lock").exists()
     else:
         if not path:
            raise Error("‘sentinel_dir’ setting is required.", culprit=description)

doc/configuring.rst

@@ -1072,6 +1072,16 @@ a large number of log entries are kept.  It is recommended that you specify
 a reasonable value for *max_entries*.
 
 
+.. _monitoring setting:
+
+monitoring
+~~~~~~~~~~
+
+*monitoring* is a dictionary setting that configures status reporting to 
+monitoring services.  Detailed information about this setting can be found in 
+:ref:`monitoring <monitoring_services>`.
+
+
 .. _must_exist:
 
 must_exist

doc/monitoring.rst

@@ -427,10 +427,137 @@ Various monitoring services are available on the web.  You can configure
 services allow you to monitor many of your routine tasks and assure they have 
 completed recently and successfully.
 
-There are many such services available and they are not difficult to add.  If 
-the service you prefer is not currently available, feel free to request it on 
-`Github <https://github.com/KenKundert/assimilate/issues>`_ or add it yourself 
-and issue a pull request.
+There are many such services available and they are not difficult to add.  There 
+is built-in support for a few common services.  For others you can use the 
+*custom* service.  It can handle most web-based services.  If the service you 
+prefer is not currently available and cannot be supported as a custom service, 
+feel free to request it on `Github 
+<https://github.com/KenKundert/assimilate/issues>`_ or add it yourself and issue 
+a pull request.
+
+.. _custom monitoring service:
+
+Custom
+~~~~~~
+
+You can configure *Assimilate* to send custom web-based messages to your 
+monitoring service when backing up.  You can configure four different types of 
+messages: *start*, *success*, *failure* and *finish*.  These messages are sent 
+as follows:
+
+start:
+    When the backup begins.
+
+success:
+    When the backup completes, but only if there were no errors.
+
+failure:
+    When the backup completes, but only if there were errors.
+
+finish:
+    When the backup completes.
+
+Generally you do not configure all of them as they are redundant.  Specifically, 
+you would configure *success* and *failure* together, or you would configure 
+*finish* in such a way as to indicate whether the backup succeeded.  For 
+example, here is how one might configure HealthCheck.io using the custom 
+service:
+
+.. code-block:: nestedtext
+
+    monitoring:
+        custom:
+            id: 51cb35d8-2975-110b-67a7-11b65d432027
+            url: https://hc-ping.com/{id}
+            start: {url}/start
+            success: {url}/0
+            failure:
+                url: {url}/fail
+                post:
+                    > CONFIG: {config}
+                    > EXIT STATUS: {status}
+                    > ERROR: {error}
+                    >
+                    > STDOUT:
+                    > {stdout}
+                    >
+                    > STDERR:
+                    > {stderr}
+
+In this example *start*, *success* and *failure* were configured. With each, you 
+can simply specify a URL, or you can specify key-value pairs that control the 
+message that is sent.  Furthermore, you can specify *id* and *url* up-front and 
+simply refer to them in when configuring your message.  For example, *start* is 
+specified as ``{url}/start``.  Here ``{url}`` is replaced by the value you 
+specified earlier for *url*.  Similarly, *success* is specified only by its URL, 
+``{url}/0``.  But, *fail* is given as a collection of key-value pairs.  Three 
+keys are allowed: *url*, *params*, and *post*.  *url* is required, but the other 
+two are optional.  *params* is a collection of key-value pairs.  These will be 
+passed in the URL as parameters.  Specify *params* only if your service requires 
+them.  *post* indicates that the *post* method is to be used, otherwise the 
+*get* method is used.  The value of post may be a string, or a collection of 
+key-value pairs.  You would specify both *params* and *post* to conform with the 
+requirements of your service.
+
+When constructing your messages you can insert placeholders that are replaced 
+before sending the message.  The available placeholders are:
+
+config:
+    The name of the config being backed up.
+
+status:
+    The exit status of the *Borg* process performing the backup.
+
+error:
+    A short message that describes error that occurred during the backup if 
+    appropriate.
+
+stdout:
+    The text sent to the standard output by the *Borg* process performing the 
+    backup.
+
+stderr:
+    The text sent to the standard error output by the *Borg* process performing 
+    the backup.
+
+id:
+    The value specified as *id*.
+
+url:
+    The value specified as *url*.
+
+success:
+    A Boolean `truth object <https://inform.readthedocs.io/en/stable/user.html#truth>`_
+    that evaluates to *yes* if *Borg* returned with an exit status of 0 or 1, 
+    which implies that the command completed as expected, though there might 
+    have been issues with individual files or directories.  If *Borg* returns 
+    with an exit status of 2 or greater, *success* evaluates to *no*.  However, 
+    you can specify different values by specifying a format string.  For 
+    example, the above example specifies both *success* and *failure*, but this 
+    could be collapsed to using only *finish* by using *success* to modify the 
+    URL used to communicate the completion message.  In this example, the URL is 
+    specified as ``{url}/{success:0/fail}``.  Here ``{success:0/fail}`` 
+    evaluates to ``0`` if *Borg* succeeds and ``fail`` otherwise.
+
+.. code-block:: nestedtext
+
+    monitoring:
+        custom:
+            id: 51cb35d8-2975-110b-67a7-11b65d432027
+            url: https://hc-ping.com/{id}
+            start: {url}/start
+            finish:
+                url: {url}/{success:0/fail}
+                post:
+                    > CONFIG: {config}
+                    > EXIT STATUS: {status}
+                    > ERROR: {error}
+                    >
+                    > STDOUT:
+                    > {stdout}
+                    >
+                    > STDERR:
+                    > {stderr}
 
 .. _cronhub:
 
@@ -442,9 +569,11 @@ health check for your *Assimilate* configuration, you will be given a UUID (a 32
 digit hexadecimal number partitioned into 5 parts by dashes).  Add that to the 
 following setting in your configuration file:
 
-.. code-block:: python
+.. code-block:: nestedtext
 
-    cronhub_uuid = '51cb35d8-2975-110b-67a7-11b65d432027'
+    monitoring:
+        cronhub.io:
+            uuid: 51cb35d8-2975-110b-67a7-11b65d432027
 
 If given, this setting should be specified on an individual configuration.  It 
 causes a report to be sent to *CronHub* each time an archive is created.  
@@ -464,9 +593,11 @@ the health check for your *Assimilate* configuration, you will be given a UUID
 (a 32 digit hexadecimal number partitioned into 5 parts by dashes).  Add that to 
 the following setting in your configuration file:
 
-.. code-block:: python
+.. code-block:: nestedtext
 
-    healthchecks_uuid = '51cb35d8-2975-110b-67a7-11b65d432027'
+    monitoring:
+        healthchecks.io:
+            uuid: 51cb35d8-2975-110b-67a7-11b65d432027
 
 If given, this setting should be specified on an individual configuration.  It 
 causes a report to be sent to *HealthChecks* each time an archive is created.  

tests/assimilate.nt

@@ -629,8 +629,6 @@ assimilate:
                             >             create_retries: number of times to retry a create if failures
                             >                             occur
                             >         create_retry_sleep: time to sleep between retries [s]
-                            >                cronhub_url: the cronhub.io URL for back-ups monitor
-                            >               cronhub_uuid: the cronhub.io UUID for back-ups monitor
                             >             default_config: default Assimilate configuration
                             >        default_list_format: the format that the list command should use if
                             >                             none is specified
@@ -643,17 +641,14 @@ assimilate:
                             >               exclude_from: file that contains exclude patterns
                             >                   excludes: list of glob strings of files or directories
                             >                             to skip
-                            >           healthchecks_url: the healthchecks.io URL for monitoring back
-                            >                             ups
-                            >          healthchecks_uuid: the healthchecks.io UUID for monitoring back
-                            >                             ups
                             >                   home_dir: users home directory (read only)
                             >                    include: include the contents of another file
                             >               list_formats: named format strings available to list command
                             >                    log_dir: log directory (read_only)
                             >                    logging: logging options
                             >           manage_diffs_cmd: command to use to manage differences in files
                             >                             and directories
+                            >                 monitoring: services to notify upon backup
                             >                 must_exist: if set, each of these files or directories
                             >                             must exist or create will quit with an error
                             >            needs_ssh_agent: when set Assimilate complains if ssh_agent is