From 453e33533db66f1b495ad3f666c69ae941de3977 Mon Sep 17 00:00:00 2001 From: Steve Keay Date: Wed, 13 Nov 2024 12:57:05 +0000 Subject: [PATCH 1/7] Add power_status to the ChassisInfo that we gather from the BMC --- python/understack-workflows/tests/test_bmc_chassis_info.py | 1 + python/understack-workflows/tests/test_nautobot_device.py | 1 + .../understack_workflows/bmc_chassis_info.py | 2 ++ 3 files changed, 4 insertions(+) diff --git a/python/understack-workflows/tests/test_bmc_chassis_info.py b/python/understack-workflows/tests/test_bmc_chassis_info.py index 7160b545..a08a7470 100644 --- a/python/understack-workflows/tests/test_bmc_chassis_info.py +++ b/python/understack-workflows/tests/test_bmc_chassis_info.py @@ -48,6 +48,7 @@ def test_chassis_info_R7615(): serial_number="33GSW04", bios_version="1.6.10", bmc_ip_address="1.2.3.4", + power_on=True, interfaces=[ bmc_chassis_info.InterfaceInfo( name="iDRAC", diff --git a/python/understack-workflows/tests/test_nautobot_device.py b/python/understack-workflows/tests/test_nautobot_device.py index beb5fece..ab12ac2c 100644 --- a/python/understack-workflows/tests/test_nautobot_device.py +++ b/python/understack-workflows/tests/test_nautobot_device.py @@ -118,6 +118,7 @@ def test_find_or_create(dell_nautobot_device): serial_number="33GSW04", bios_version="1.6.10", bmc_ip_address="1.2.3.4", + power_on=True, interfaces=[ InterfaceInfo( name="iDRAC", diff --git a/python/understack-workflows/understack_workflows/bmc_chassis_info.py b/python/understack-workflows/understack_workflows/bmc_chassis_info.py index bea87201..d4b83a85 100644 --- a/python/understack-workflows/understack_workflows/bmc_chassis_info.py +++ b/python/understack-workflows/understack_workflows/bmc_chassis_info.py @@ -30,6 +30,7 @@ class ChassisInfo: serial_number: str bmc_ip_address: str bios_version: str + power_on: bool interfaces: list[InterfaceInfo] @property @@ -67,6 +68,7 @@ def chassis_info(bmc: Bmc) -> ChassisInfo: model_number=chassis_data["Model"], serial_number=chassis_data["SKU"], bios_version=chassis_data["BiosVersion"], + power_on=(chassis_data["PowerState"] == "On"), bmc_ip_address=bmc.ip_address, interfaces=interfaces, ) From 41dae07d4607efc9a67fcc03cc56463bab091c19 Mon Sep 17 00:00:00 2001 From: Steve Keay Date: Wed, 13 Nov 2024 14:16:48 +0000 Subject: [PATCH 2/7] Add LLDP neighbors property to ChassisInfo --- .../tests/test_bmc_chassis_info.py | 10 ++++++++++ .../understack_workflows/bmc_chassis_info.py | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/python/understack-workflows/tests/test_bmc_chassis_info.py b/python/understack-workflows/tests/test_bmc_chassis_info.py index a08a7470..53456aaa 100644 --- a/python/understack-workflows/tests/test_bmc_chassis_info.py +++ b/python/understack-workflows/tests/test_bmc_chassis_info.py @@ -40,6 +40,16 @@ def read_fixture(path, filename): return json.loads(f.read()) +def test_chassis_neighbors(): + bmc = FakeBmc(read_fixtures("json_samples/bmc_chassis_info/R7615")) + chassis_info = bmc_chassis_info.chassis_info(bmc) + assert chassis_info.neighbors == { + "C4:7E:E0:E4:10:7F", + "C4:4D:84:48:61:80", + "C4:7E:E0:E4:32:DF", + } + + def test_chassis_info_R7615(): bmc = FakeBmc(read_fixtures("json_samples/bmc_chassis_info/R7615")) assert bmc_chassis_info.chassis_info(bmc) == bmc_chassis_info.ChassisInfo( diff --git a/python/understack-workflows/understack_workflows/bmc_chassis_info.py b/python/understack-workflows/understack_workflows/bmc_chassis_info.py index d4b83a85..1787da0a 100644 --- a/python/understack-workflows/understack_workflows/bmc_chassis_info.py +++ b/python/understack-workflows/understack_workflows/bmc_chassis_info.py @@ -41,6 +41,15 @@ def bmc_interface(self) -> InterfaceInfo: def bmc_hostname(self) -> str: return str(self.bmc_interface.hostname) + @property + def neighbors(self) -> set: + """A set of switch MAC addresses to which this chassis is connected.""" + return { + interface.remote_switch_mac_address + for interface in self.interfaces + if interface.remote_switch_mac_address + } + REDFISH_SYSTEM_ENDPOINT = "/redfish/v1/Systems/System.Embedded.1/" REDFISH_ETHERNET_ENDPOINT = f"{REDFISH_SYSTEM_ENDPOINT}EthernetInterfaces/" From 5ca9c4321ce7c05a131e7d9c2edbaa131cbd380f Mon Sep 17 00:00:00 2001 From: Steve Keay Date: Wed, 13 Nov 2024 14:16:59 +0000 Subject: [PATCH 3/7] Set a more useful date format in logging output --- python/understack-workflows/understack_workflows/helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/understack-workflows/understack_workflows/helpers.py b/python/understack-workflows/understack_workflows/helpers.py index f2fa6963..8b8310b9 100644 --- a/python/understack-workflows/understack_workflows/helpers.py +++ b/python/understack-workflows/understack_workflows/helpers.py @@ -17,6 +17,7 @@ def setup_logger(name: str | None = None, level: int = logging.DEBUG): """ logging.basicConfig( format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S %z", level=level, ) return logging.getLogger(name) From 80f1aba305437d167db5db56e5ccc45277757880 Mon Sep 17 00:00:00 2001 From: Steve Keay Date: Wed, 13 Nov 2024 14:18:11 +0000 Subject: [PATCH 4/7] Remove TODO comment - I think we are going to use Ironic for this --- .../understack_workflows/main/enroll_server.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/understack-workflows/understack_workflows/main/enroll_server.py b/python/understack-workflows/understack_workflows/main/enroll_server.py index 8fff4475..dedd2993 100644 --- a/python/understack-workflows/understack_workflows/main/enroll_server.py +++ b/python/understack-workflows/understack_workflows/main/enroll_server.py @@ -53,8 +53,6 @@ def main(): - TODO: create and install SSL certificate - - TODO: update BMC firmware - - TODO: set NTP Server IPs for DRAC (NTP server IP addresses are different per region) From 24a64e68aa32103b3aa48e48ec089524e9d4af29 Mon Sep 17 00:00:00 2001 From: Steve Keay Date: Wed, 13 Nov 2024 14:31:29 +0000 Subject: [PATCH 5/7] If server if off, power it on, and wait for LLDP neighbors to appear --- .../understack_workflows/bmc_power.py | 13 ++++++ .../understack_workflows/discover.py | 44 +++++++++++++++++++ .../main/enroll_server.py | 25 ++++++++--- 3 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 python/understack-workflows/understack_workflows/bmc_power.py create mode 100644 python/understack-workflows/understack_workflows/discover.py diff --git a/python/understack-workflows/understack_workflows/bmc_power.py b/python/understack-workflows/understack_workflows/bmc_power.py new file mode 100644 index 00000000..b9ef38a9 --- /dev/null +++ b/python/understack-workflows/understack_workflows/bmc_power.py @@ -0,0 +1,13 @@ +from understack_workflows.bmc import Bmc +from understack_workflows.helpers import setup_logger + +logger = setup_logger(__name__) + + +def bmc_power_on(bmc: Bmc): + """Make a redfish call to switch on the power to the system.""" + bmc.redfish_request( + "/redfish/v1/Systems/System.Embedded.1/Actions/ComputerSystem.Reset", + payload={"ResetType": "On"}, + method="POST", + ) diff --git a/python/understack-workflows/understack_workflows/discover.py b/python/understack-workflows/understack_workflows/discover.py new file mode 100644 index 00000000..a20c490e --- /dev/null +++ b/python/understack-workflows/understack_workflows/discover.py @@ -0,0 +1,44 @@ +import time + +from understack_workflows.bmc import Bmc +from understack_workflows.bmc_chassis_info import ChassisInfo +from understack_workflows.bmc_chassis_info import chassis_info +from understack_workflows.bmc_power import bmc_power_on +from understack_workflows.helpers import setup_logger + +logger = setup_logger(__name__) + +MIN_REQURED_NEIGHBOR_COUNT = 3 + + +def discover_chassis_info(bmc: Bmc) -> ChassisInfo: + """Query redfish, retrying until we get data that is acceptable. + + If the server is off, power it on. + + Make sure that we have at lease MIN_REQURED_NEIGHBOR_COUNT LLDP neighbors in + the returned ChassisInfo. If that can't be achieved in a reasonable time + then raise an Exception. + """ + device_info = chassis_info(bmc) + + if not device_info.power_on: + logger.info(f"Server is powered off, sending power-on command to {bmc}") + bmc_power_on(bmc) + + attempts_remaining = 6 + while len(device_info.neighbors) < MIN_REQURED_NEIGHBOR_COUNT: + logger.info( + f"{bmc} does not have enough LLDP neighbors " + f"(saw {device_info.neighbors}), need at " + f"least {MIN_REQURED_NEIGHBOR_COUNT}. " + ) + if not attempts_remaining: + raise Exception("No neighbors appeared") + logger.info(f"Retry in 30 seconds ({attempts_remaining=})") + attempts_remaining = attempts_remaining - 1 + + time.sleep(30) + device_info = chassis_info(bmc) + + return device_info diff --git a/python/understack-workflows/understack_workflows/main/enroll_server.py b/python/understack-workflows/understack_workflows/main/enroll_server.py index dedd2993..88d87d75 100644 --- a/python/understack-workflows/understack_workflows/main/enroll_server.py +++ b/python/understack-workflows/understack_workflows/main/enroll_server.py @@ -8,19 +8,23 @@ from understack_workflows import nautobot_device from understack_workflows import sync_interfaces from understack_workflows import topology +from understack_workflows.bmc import Bmc from understack_workflows.bmc import bmc_for_ip_address from understack_workflows.bmc_bios import update_dell_bios_settings -from understack_workflows.bmc_chassis_info import chassis_info from understack_workflows.bmc_credentials import set_bmc_password from understack_workflows.bmc_hostname import bmc_set_hostname from understack_workflows.bmc_network_config import bmc_set_permanent_ip_addr from understack_workflows.bmc_settings import update_dell_drac_settings +from understack_workflows.discover import discover_chassis_info from understack_workflows.helpers import credential from understack_workflows.helpers import parser_nautobot_args from understack_workflows.helpers import setup_logger +from understack_workflows.nautobot_device import NautobotDevice logger = setup_logger(__name__) +MIN_REQURED_NEIGHBOR_COUNT = 4 + def main(): """On-board new or Refresh existing baremetal node. @@ -49,7 +53,7 @@ def main(): - if DHCP, set permanent IP address, netmask, default gw - - TODO: if server is off, power it on and wait (otherwise LLDP doesn't work) + - if server is off, power it on and wait (otherwise LLDP doesn't work) - TODO: create and install SSL certificate @@ -99,13 +103,22 @@ def main(): nautobot = pynautobot.api(url, token=token) bmc = bmc_for_ip_address(bmc_ip_address) + + nb_device = enroll_server(bmc, nautobot, args.old_bmc_password) + + # argo workflows captures stdout as the results which we can use + # to return the device UUID + print(str(nb_device.id)) + + +def enroll_server(bmc: Bmc, nautobot, old_password: str | None) -> NautobotDevice: set_bmc_password( ip_address=bmc.ip_address, new_password=bmc.password, - old_password=args.old_bmc_password, + old_password=old_password, ) - device_info = chassis_info(bmc) + device_info = discover_chassis_info(bmc) logger.info(f"Discovered {pformat(device_info)}") update_dell_drac_settings(bmc) @@ -132,9 +145,7 @@ def main(): logger.info(f"{__file__} complete for {bmc.ip_address}") - # argo workflows captures stdout as the results which we can use - # return the device UUID - print(str(nb_device.id)) + return nb_device def argument_parser(): From bbb9589b7f46b961cf6a86255ea473f07d521eae Mon Sep 17 00:00:00 2001 From: Steve Keay Date: Wed, 13 Nov 2024 14:38:55 +0000 Subject: [PATCH 6/7] Fix a spelling mistake --- .../understack-workflows/understack_workflows/discover.py | 8 ++++---- .../understack_workflows/main/enroll_server.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/understack-workflows/understack_workflows/discover.py b/python/understack-workflows/understack_workflows/discover.py index a20c490e..4c5c623b 100644 --- a/python/understack-workflows/understack_workflows/discover.py +++ b/python/understack-workflows/understack_workflows/discover.py @@ -8,7 +8,7 @@ logger = setup_logger(__name__) -MIN_REQURED_NEIGHBOR_COUNT = 3 +MIN_REQUIRED_NEIGHBOR_COUNT = 3 def discover_chassis_info(bmc: Bmc) -> ChassisInfo: @@ -16,7 +16,7 @@ def discover_chassis_info(bmc: Bmc) -> ChassisInfo: If the server is off, power it on. - Make sure that we have at lease MIN_REQURED_NEIGHBOR_COUNT LLDP neighbors in + Make sure that we have at lease MIN_REQUIRED_NEIGHBOR_COUNT LLDP neighbors in the returned ChassisInfo. If that can't be achieved in a reasonable time then raise an Exception. """ @@ -27,11 +27,11 @@ def discover_chassis_info(bmc: Bmc) -> ChassisInfo: bmc_power_on(bmc) attempts_remaining = 6 - while len(device_info.neighbors) < MIN_REQURED_NEIGHBOR_COUNT: + while len(device_info.neighbors) < MIN_REQUIRED_NEIGHBOR_COUNT: logger.info( f"{bmc} does not have enough LLDP neighbors " f"(saw {device_info.neighbors}), need at " - f"least {MIN_REQURED_NEIGHBOR_COUNT}. " + f"least {MIN_REQUIRED_NEIGHBOR_COUNT}. " ) if not attempts_remaining: raise Exception("No neighbors appeared") diff --git a/python/understack-workflows/understack_workflows/main/enroll_server.py b/python/understack-workflows/understack_workflows/main/enroll_server.py index 88d87d75..e287b2ac 100644 --- a/python/understack-workflows/understack_workflows/main/enroll_server.py +++ b/python/understack-workflows/understack_workflows/main/enroll_server.py @@ -23,7 +23,7 @@ logger = setup_logger(__name__) -MIN_REQURED_NEIGHBOR_COUNT = 4 +MIN_REQUIRED_NEIGHBOR_COUNT = 4 def main(): From dec4b1018bae3478ae570ccfa2696251979acdf8 Mon Sep 17 00:00:00 2001 From: Steve Keay Date: Wed, 13 Nov 2024 20:13:32 +0000 Subject: [PATCH 7/7] Fix up discover code as per review suggestions --- .../understack_workflows/discover.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/python/understack-workflows/understack_workflows/discover.py b/python/understack-workflows/understack_workflows/discover.py index 4c5c623b..e7ffed34 100644 --- a/python/understack-workflows/understack_workflows/discover.py +++ b/python/understack-workflows/understack_workflows/discover.py @@ -9,6 +9,7 @@ logger = setup_logger(__name__) MIN_REQUIRED_NEIGHBOR_COUNT = 3 +LLDP_DISCOVERY_ATTEMPTS = 6 def discover_chassis_info(bmc: Bmc) -> ChassisInfo: @@ -16,8 +17,8 @@ def discover_chassis_info(bmc: Bmc) -> ChassisInfo: If the server is off, power it on. - Make sure that we have at lease MIN_REQUIRED_NEIGHBOR_COUNT LLDP neighbors in - the returned ChassisInfo. If that can't be achieved in a reasonable time + Make sure that we have at least MIN_REQUIRED_NEIGHBOR_COUNT LLDP neighbors + in the returned ChassisInfo. If that can't be achieved in a reasonable time then raise an Exception. """ device_info = chassis_info(bmc) @@ -26,7 +27,7 @@ def discover_chassis_info(bmc: Bmc) -> ChassisInfo: logger.info(f"Server is powered off, sending power-on command to {bmc}") bmc_power_on(bmc) - attempts_remaining = 6 + attempts_remaining = LLDP_DISCOVERY_ATTEMPTS while len(device_info.neighbors) < MIN_REQUIRED_NEIGHBOR_COUNT: logger.info( f"{bmc} does not have enough LLDP neighbors " @@ -34,7 +35,10 @@ def discover_chassis_info(bmc: Bmc) -> ChassisInfo: f"least {MIN_REQUIRED_NEIGHBOR_COUNT}. " ) if not attempts_remaining: - raise Exception("No neighbors appeared") + raise Exception( + f"Only {len(device_info.neighbors)} LLDP neighbors appeared, " + f" but {MIN_REQUIRED_NEIGHBOR_COUNT} are required." + ) logger.info(f"Retry in 30 seconds ({attempts_remaining=})") attempts_remaining = attempts_remaining - 1