From 8000c95551877d4848be667c8c70f38448202b78 Mon Sep 17 00:00:00 2001 From: Jeff Kala Date: Fri, 8 Mar 2024 21:49:23 -0600 Subject: [PATCH 01/11] force the enable call to allow many cisco ios platforms to work --- nornir_nautobot/plugins/tasks/dispatcher/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nornir_nautobot/plugins/tasks/dispatcher/default.py b/nornir_nautobot/plugins/tasks/dispatcher/default.py index 44b4747..4ebc1a1 100644 --- a/nornir_nautobot/plugins/tasks/dispatcher/default.py +++ b/nornir_nautobot/plugins/tasks/dispatcher/default.py @@ -459,7 +459,7 @@ def get_config( command = cls.config_command try: - result = task.run(task=netmiko_send_command, command_string=command) + result = task.run(task=netmiko_send_command, command_string=command, enable=True) except NornirSubTaskError as exc: if isinstance(exc.result.exception, NetmikoAuthenticationException): error_msg = f"`E1017:` Failed with an authentication issue: `{exc.result.exception}`" From 21525498a96c1fd135e8cf20f841299dafed26fb Mon Sep 17 00:00:00 2001 From: Justin Pettit <47164813+jmpettit@users.noreply.github.com> Date: Wed, 20 Mar 2024 22:23:04 -0500 Subject: [PATCH 02/11] Refactor Jinja error handling and add stack trace --- .../plugins/tasks/dispatcher/default.py | 41 ++++++++----------- nornir_nautobot/utils/helpers.py | 7 ++++ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/nornir_nautobot/plugins/tasks/dispatcher/default.py b/nornir_nautobot/plugins/tasks/dispatcher/default.py index 44b4747..b09d437 100644 --- a/nornir_nautobot/plugins/tasks/dispatcher/default.py +++ b/nornir_nautobot/plugins/tasks/dispatcher/default.py @@ -25,7 +25,7 @@ from nornir_netmiko.tasks import netmiko_send_command from nornir_nautobot.exceptions import NornirNautobotException -from nornir_nautobot.utils.helpers import make_folder +from nornir_nautobot.utils.helpers import make_folder, format_jinja_stack_trace _logger = logging.getLogger(__name__) @@ -174,29 +174,22 @@ def generate_config( jinja_env=jinja_env, )[0].result except NornirSubTaskError as exc: - if isinstance(exc.result.exception, jinja2.exceptions.UndefinedError): # pylint: disable=no-else-raise - error_msg = ( - f"`E1010:` There was a jinja2.exceptions.UndefinedError error: ``{str(exc.result.exception)}``" - ) - logger.error(error_msg, extra={"object": obj}) - raise NornirNautobotException(error_msg) - - elif isinstance(exc.result.exception, jinja2.TemplateSyntaxError): - error_msg = (f"`E1011:` There was a jinja2.TemplateSyntaxError error: ``{str(exc.result.exception)}``",) - logger.error(error_msg, extra={"object": obj}) - raise NornirNautobotException(error_msg) - - elif isinstance(exc.result.exception, jinja2.TemplateNotFound): - error_msg = f"`E1012:` There was an issue finding the template and a jinja2.TemplateNotFound error was raised: ``{str(exc.result.exception)}``" - logger.error(error_msg, extra={"object": obj}) - raise NornirNautobotException(error_msg) - - elif isinstance(exc.result.exception, jinja2.TemplateError): - error_msg = f"`E1013:` There was an issue general Jinja error: ``{str(exc.result.exception)}``" - logger.error(error_msg, extra={"object": obj}) - raise NornirNautobotException(error_msg) - - error_msg = f"`E1014:` Failed with an unknown issue. `{exc.result.exception}`" + stack_trace = format_jinja_stack_trace(exc.result.exception) + + error_mapping = { + jinja2.exceptions.UndefinedError: ("E1010", "Undefined variable in Jinja2 template"), + jinja2.TemplateSyntaxError: ("E1011", "Syntax error in Jinja2 template"), + jinja2.TemplateNotFound: ("E1012", "Jinja2 template not found"), + jinja2.TemplateError: ("E1013", "General Jinja2 template error"), + } + + for error, (code, message) in error_mapping.items(): + if isinstance(exc.result.exception, error): + error_msg = f"`{code}:` {message} - ``{str(exc.result.exception)}``\n```\n{stack_trace}\n```" + logger.error(error_msg, extra={"object": obj}) + raise NornirNautobotException(error_msg) + + error_msg = f"`E1014:` Unknown error - {exc.result.exception}\n```\n{stack_trace}\n```" logger.error(error_msg, extra={"object": obj}) raise NornirNautobotException(error_msg) diff --git a/nornir_nautobot/utils/helpers.py b/nornir_nautobot/utils/helpers.py index 4912758..46f4a00 100644 --- a/nornir_nautobot/utils/helpers.py +++ b/nornir_nautobot/utils/helpers.py @@ -4,6 +4,7 @@ import os import logging import importlib +import traceback LOGGER = logging.getLogger(__name__) @@ -31,3 +32,9 @@ def import_string(dotted_path): return getattr(importlib.import_module(module_name), class_name) except (ModuleNotFoundError, AttributeError): return None + +def format_jinja_stack_trace(exc: Exception) -> str: + """Generate and format a stack trace string for a given Jinja exception.""" + stack_trace_lines = traceback.format_exception(type(exc), exc, exc.__traceback__) + stack_trace_lines = [line for line in stack_trace_lines if '.j2' in line or '{{' in line] + return "\n".join(stack_trace_lines) From 0fea1678ca1a421e1d8dc62bbfa4c64d524767e3 Mon Sep 17 00:00:00 2001 From: Justin Pettit <47164813+jmpettit@users.noreply.github.com> Date: Wed, 20 Mar 2024 22:47:38 -0500 Subject: [PATCH 03/11] Fix format for E1014 --- nornir_nautobot/plugins/tasks/dispatcher/default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nornir_nautobot/plugins/tasks/dispatcher/default.py b/nornir_nautobot/plugins/tasks/dispatcher/default.py index b09d437..e49d34e 100644 --- a/nornir_nautobot/plugins/tasks/dispatcher/default.py +++ b/nornir_nautobot/plugins/tasks/dispatcher/default.py @@ -189,7 +189,7 @@ def generate_config( logger.error(error_msg, extra={"object": obj}) raise NornirNautobotException(error_msg) - error_msg = f"`E1014:` Unknown error - {exc.result.exception}\n```\n{stack_trace}\n```" + error_msg = f"`E1014:` Unknown error - `{exc.result.exception}`" logger.error(error_msg, extra={"object": obj}) raise NornirNautobotException(error_msg) From 15076b7ca4daa527110c2e4d9910f8520885ac70 Mon Sep 17 00:00:00 2001 From: Justin Pettit <47164813+jmpettit@users.noreply.github.com> Date: Thu, 21 Mar 2024 08:41:45 -0500 Subject: [PATCH 04/11] Include full stack trace in error logging --- nornir_nautobot/plugins/tasks/dispatcher/default.py | 6 +++--- nornir_nautobot/utils/helpers.py | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/nornir_nautobot/plugins/tasks/dispatcher/default.py b/nornir_nautobot/plugins/tasks/dispatcher/default.py index e49d34e..c02fea9 100644 --- a/nornir_nautobot/plugins/tasks/dispatcher/default.py +++ b/nornir_nautobot/plugins/tasks/dispatcher/default.py @@ -25,7 +25,7 @@ from nornir_netmiko.tasks import netmiko_send_command from nornir_nautobot.exceptions import NornirNautobotException -from nornir_nautobot.utils.helpers import make_folder, format_jinja_stack_trace +from nornir_nautobot.utils.helpers import make_folder, get_stack_trace _logger = logging.getLogger(__name__) @@ -174,7 +174,7 @@ def generate_config( jinja_env=jinja_env, )[0].result except NornirSubTaskError as exc: - stack_trace = format_jinja_stack_trace(exc.result.exception) + stack_trace = get_stack_trace(exc.result.exception) error_mapping = { jinja2.exceptions.UndefinedError: ("E1010", "Undefined variable in Jinja2 template"), @@ -189,7 +189,7 @@ def generate_config( logger.error(error_msg, extra={"object": obj}) raise NornirNautobotException(error_msg) - error_msg = f"`E1014:` Unknown error - `{exc.result.exception}`" + error_msg = f"`E1014:` Unknown error - `{exc.result.exception}`\n```\n{stack_trace}\n```" logger.error(error_msg, extra={"object": obj}) raise NornirNautobotException(error_msg) diff --git a/nornir_nautobot/utils/helpers.py b/nornir_nautobot/utils/helpers.py index 46f4a00..ea2df8e 100644 --- a/nornir_nautobot/utils/helpers.py +++ b/nornir_nautobot/utils/helpers.py @@ -33,8 +33,7 @@ def import_string(dotted_path): except (ModuleNotFoundError, AttributeError): return None -def format_jinja_stack_trace(exc: Exception) -> str: +def get_stack_trace(exc: Exception) -> str: """Generate and format a stack trace string for a given Jinja exception.""" stack_trace_lines = traceback.format_exception(type(exc), exc, exc.__traceback__) - stack_trace_lines = [line for line in stack_trace_lines if '.j2' in line or '{{' in line] return "\n".join(stack_trace_lines) From 201e684a465ed37992476f613692a5d8b9de6b02 Mon Sep 17 00:00:00 2001 From: Justin Pettit <47164813+jmpettit@users.noreply.github.com> Date: Thu, 21 Mar 2024 09:55:20 -0500 Subject: [PATCH 05/11] Fix Linting issues --- nornir_nautobot/utils/helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nornir_nautobot/utils/helpers.py b/nornir_nautobot/utils/helpers.py index ea2df8e..168a964 100644 --- a/nornir_nautobot/utils/helpers.py +++ b/nornir_nautobot/utils/helpers.py @@ -33,6 +33,7 @@ def import_string(dotted_path): except (ModuleNotFoundError, AttributeError): return None + def get_stack_trace(exc: Exception) -> str: """Generate and format a stack trace string for a given Jinja exception.""" stack_trace_lines = traceback.format_exception(type(exc), exc, exc.__traceback__) From 6fcd84c78d7e88e77c72128d95ed8d0cd357ebcd Mon Sep 17 00:00:00 2001 From: Justin Pettit <47164813+jmpettit@users.noreply.github.com> Date: Thu, 21 Mar 2024 10:07:06 -0500 Subject: [PATCH 06/11] Update docstring on get_stack_trace --- nornir_nautobot/utils/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nornir_nautobot/utils/helpers.py b/nornir_nautobot/utils/helpers.py index 168a964..5954456 100644 --- a/nornir_nautobot/utils/helpers.py +++ b/nornir_nautobot/utils/helpers.py @@ -35,6 +35,6 @@ def import_string(dotted_path): def get_stack_trace(exc: Exception) -> str: - """Generate and format a stack trace string for a given Jinja exception.""" + """Converts the provided exception's stack trace into a string.""" stack_trace_lines = traceback.format_exception(type(exc), exc, exc.__traceback__) return "\n".join(stack_trace_lines) From 7f62f5089aa085fbd0c5143650397ee233ef7005 Mon Sep 17 00:00:00 2001 From: Justin Pettit <47164813+jmpettit@users.noreply.github.com> Date: Thu, 21 Mar 2024 10:28:06 -0500 Subject: [PATCH 07/11] Add pylint disable comment for too-many-locals --- nornir_nautobot/plugins/tasks/dispatcher/default.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nornir_nautobot/plugins/tasks/dispatcher/default.py b/nornir_nautobot/plugins/tasks/dispatcher/default.py index c02fea9..436aa4b 100644 --- a/nornir_nautobot/plugins/tasks/dispatcher/default.py +++ b/nornir_nautobot/plugins/tasks/dispatcher/default.py @@ -149,6 +149,7 @@ def generate_config( jinja_filters: Optional[dict] = None, jinja_env: Optional[jinja2.Environment] = None, ) -> Result: + # pylint: disable=too-many-locals """A small wrapper around template_file Nornir task. Args: From 6f215d082ab3b0aebf8915b5f4af738842ec375c Mon Sep 17 00:00:00 2001 From: Justin Pettit <47164813+jmpettit@users.noreply.github.com> Date: Thu, 21 Mar 2024 12:20:06 -0500 Subject: [PATCH 08/11] Fix stack trace formatting in get_stack_trace function --- nornir_nautobot/utils/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nornir_nautobot/utils/helpers.py b/nornir_nautobot/utils/helpers.py index 5954456..463bb53 100644 --- a/nornir_nautobot/utils/helpers.py +++ b/nornir_nautobot/utils/helpers.py @@ -37,4 +37,4 @@ def import_string(dotted_path): def get_stack_trace(exc: Exception) -> str: """Converts the provided exception's stack trace into a string.""" stack_trace_lines = traceback.format_exception(type(exc), exc, exc.__traceback__) - return "\n".join(stack_trace_lines) + return "".join(stack_trace_lines) From 1f15aa999175de7363347bac593e4ffa454fb60d Mon Sep 17 00:00:00 2001 From: Jeff Kala Date: Thu, 28 Mar 2024 11:06:45 -0500 Subject: [PATCH 09/11] add enable default os env var to override --- docs/task/task.md | 9 ++++- examples/basic_with_task.py | 35 +++++++++++++++++++ .../plugins/tasks/dispatcher/default.py | 9 +++-- nornir_nautobot/utils/helpers.py | 22 ++++++++++++ 4 files changed, 71 insertions(+), 4 deletions(-) create mode 100755 examples/basic_with_task.py diff --git a/docs/task/task.md b/docs/task/task.md index d8f0d99..71cb9af 100644 --- a/docs/task/task.md +++ b/docs/task/task.md @@ -102,4 +102,11 @@ class DispatcherMixin: if isinstance(config_context, int): return config_context return cls.tcp_port -``` \ No newline at end of file +``` + +## Environment Variables + +| Environment Variable | Explanation | +| ----- | ----------- | +| NORNIR_NAUTOBOT_REVERT_IN_SECONDS | Amount in seconds to revert if a config based method fails. | +| NORNIR_NAUTOBOT_NETMIKO_ENABLE_DEFAULT | Override the default(True) to not automatically call the `enable` function before running commands. | diff --git a/examples/basic_with_task.py b/examples/basic_with_task.py new file mode 100755 index 0000000..a46ed72 --- /dev/null +++ b/examples/basic_with_task.py @@ -0,0 +1,35 @@ +from nornir import InitNornir +from nornir_nautobot.plugins.tasks.dispatcher import dispatcher +from nornir_utils.plugins.functions import print_result + +import logging + +LOGGER = logging.getLogger(__name__) + +my_nornir = InitNornir( + inventory={ + "plugin": "NautobotInventory", + "options": { + "nautobot_url": "http://localhost:8080/", + "nautobot_token": "0123456789abcdef0123456789abcdef01234567", + "filter_parameters": {"location": "Site 1"}, + "ssl_verify": False, + }, + }, +) +my_nornir.inventory.defaults.username = "jeff" +my_nornir.inventory.defaults.password = "cisco" + +for nr_host, nr_obj in my_nornir.inventory.hosts.items(): + network_driver = my_nornir.inventory.hosts[nr_host].platform + result = my_nornir.run( + task=dispatcher, + logger=LOGGER, + method="get_config", + obj=nr_host, + framework="netmiko", + backup_file="./ios.cfg", + remove_lines=None, + substitute_lines=None, + ) + print_result(result) diff --git a/nornir_nautobot/plugins/tasks/dispatcher/default.py b/nornir_nautobot/plugins/tasks/dispatcher/default.py index 4ebc1a1..40c6255 100644 --- a/nornir_nautobot/plugins/tasks/dispatcher/default.py +++ b/nornir_nautobot/plugins/tasks/dispatcher/default.py @@ -23,9 +23,8 @@ from nornir_jinja2.plugins.tasks import template_file from nornir_napalm.plugins.tasks import napalm_configure, napalm_get from nornir_netmiko.tasks import netmiko_send_command - from nornir_nautobot.exceptions import NornirNautobotException -from nornir_nautobot.utils.helpers import make_folder +from nornir_nautobot.utils.helpers import make_folder, is_truthy _logger = logging.getLogger(__name__) @@ -459,7 +458,11 @@ def get_config( command = cls.config_command try: - result = task.run(task=netmiko_send_command, command_string=command, enable=True) + result = task.run( + task=netmiko_send_command, + command_string=command, + enable=is_truthy(os.getenv("NORNIR_NAUTOBOT_NETMIKO_ENABLE_DEFAULT", default="True")), + ) except NornirSubTaskError as exc: if isinstance(exc.result.exception, NetmikoAuthenticationException): error_msg = f"`E1017:` Failed with an authentication issue: `{exc.result.exception}`" diff --git a/nornir_nautobot/utils/helpers.py b/nornir_nautobot/utils/helpers.py index 4912758..d711913 100644 --- a/nornir_nautobot/utils/helpers.py +++ b/nornir_nautobot/utils/helpers.py @@ -31,3 +31,25 @@ def import_string(dotted_path): return getattr(importlib.import_module(module_name), class_name) except (ModuleNotFoundError, AttributeError): return None + + +def is_truthy(arg): + """Convert "truthy" strings into Booleans. + + Args: + arg (str): Truthy string (True values are y, yes, t, true, on and 1; false values are n, no, + f, false, off and 0. Raises ValueError if val is anything else. + + Examples: + >>> is_truthy('yes') + True + """ + if isinstance(arg, bool): + return arg + + val = str(arg).lower() + if val in ("y", "yes", "t", "true", "on", "1"): + return True + if val in ("n", "no", "f", "false", "off", "0"): + return False + return True From 4e89dcf4f26a14f640d63e2670a1d9e65dc976fd Mon Sep 17 00:00:00 2001 From: Jeff Kala Date: Thu, 28 Mar 2024 11:14:28 -0500 Subject: [PATCH 10/11] fix linting and other issues in new example --- examples/basic_with_task.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/examples/basic_with_task.py b/examples/basic_with_task.py index a46ed72..60c5c04 100755 --- a/examples/basic_with_task.py +++ b/examples/basic_with_task.py @@ -1,8 +1,11 @@ +"""Example with a actual dispatcher task.""" + +import logging +import os from nornir import InitNornir -from nornir_nautobot.plugins.tasks.dispatcher import dispatcher from nornir_utils.plugins.functions import print_result +from nornir_nautobot.plugins.tasks.dispatcher import dispatcher -import logging LOGGER = logging.getLogger(__name__) @@ -17,8 +20,8 @@ }, }, ) -my_nornir.inventory.defaults.username = "jeff" -my_nornir.inventory.defaults.password = "cisco" +my_nornir.inventory.defaults.username = os.getenv("NORNIR_USERNAME") +my_nornir.inventory.defaults.password = os.getenv("NORNIR_PASSWORD") for nr_host, nr_obj in my_nornir.inventory.hosts.items(): network_driver = my_nornir.inventory.hosts[nr_host].platform From ad2d4c210d79f5086df99336cc7a9096b0012f4c Mon Sep 17 00:00:00 2001 From: Josh VanDeraa Date: Fri, 29 Mar 2024 08:25:52 -0500 Subject: [PATCH 11/11] Release 3.2.0 --- docs/dev/CHANGELOG.md | 10 ++++++++++ pyproject.toml | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/dev/CHANGELOG.md b/docs/dev/CHANGELOG.md index cdd81b0..cbb930d 100644 --- a/docs/dev/CHANGELOG.md +++ b/docs/dev/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## 3.2.0 + +* [#144] force the enable call to allow many cisco ios platforms to work +* [#149] Enhanced Jinja Error Handling and Stack Trace Logging by @jmpettit + +### New Contributors +* @jmpettit made their first contribution in https://github.com/nautobot/nornir-nautobot/pull/149 + +**Full Changelog**: https://github.com/nautobot/nornir-nautobot/compare/v3.1.2...v3.2.0 + ## 3.1.2 - [#145](https://github.com/nautobot/nornir-nautobot/pull/145) Update httpx diff --git a/pyproject.toml b/pyproject.toml index e425f27..b22ef37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "nornir-nautobot" -version = "3.1.1" +version = "3.2.0" description = "Nornir Nautobot" authors = ["Network to Code, LLC "] readme = "README.md"