diff --git a/python/understack-workflows/tests/test_bmc_chassis_info.py b/python/understack-workflows/tests/test_bmc_chassis_info.py index 7160b545..53456aaa 100644 --- a/python/understack-workflows/tests/test_bmc_chassis_info.py +++ b/python/understack-workflows/tests/test_bmc_chassis_info.py @@ -40,6 +40,16 @@ def read_fixture(path, filename): return json.loads(f.read()) +def test_chassis_neighbors(): + bmc = FakeBmc(read_fixtures("json_samples/bmc_chassis_info/R7615")) + chassis_info = bmc_chassis_info.chassis_info(bmc) + assert chassis_info.neighbors == { + "C4:7E:E0:E4:10:7F", + "C4:4D:84:48:61:80", + "C4:7E:E0:E4:32:DF", + } + + def test_chassis_info_R7615(): bmc = FakeBmc(read_fixtures("json_samples/bmc_chassis_info/R7615")) assert bmc_chassis_info.chassis_info(bmc) == bmc_chassis_info.ChassisInfo( @@ -48,6 +58,7 @@ def test_chassis_info_R7615(): serial_number="33GSW04", bios_version="1.6.10", bmc_ip_address="1.2.3.4", + power_on=True, interfaces=[ bmc_chassis_info.InterfaceInfo( name="iDRAC", diff --git a/python/understack-workflows/tests/test_nautobot_device.py b/python/understack-workflows/tests/test_nautobot_device.py index beb5fece..ab12ac2c 100644 --- a/python/understack-workflows/tests/test_nautobot_device.py +++ b/python/understack-workflows/tests/test_nautobot_device.py @@ -118,6 +118,7 @@ def test_find_or_create(dell_nautobot_device): serial_number="33GSW04", bios_version="1.6.10", bmc_ip_address="1.2.3.4", + power_on=True, interfaces=[ InterfaceInfo( name="iDRAC", diff --git a/python/understack-workflows/understack_workflows/bmc_chassis_info.py b/python/understack-workflows/understack_workflows/bmc_chassis_info.py index bea87201..1787da0a 100644 --- a/python/understack-workflows/understack_workflows/bmc_chassis_info.py +++ b/python/understack-workflows/understack_workflows/bmc_chassis_info.py @@ -30,6 +30,7 @@ class ChassisInfo: serial_number: str bmc_ip_address: str bios_version: str + power_on: bool interfaces: list[InterfaceInfo] @property @@ -40,6 +41,15 @@ def bmc_interface(self) -> InterfaceInfo: def bmc_hostname(self) -> str: return str(self.bmc_interface.hostname) + @property + def neighbors(self) -> set: + """A set of switch MAC addresses to which this chassis is connected.""" + return { + interface.remote_switch_mac_address + for interface in self.interfaces + if interface.remote_switch_mac_address + } + REDFISH_SYSTEM_ENDPOINT = "/redfish/v1/Systems/System.Embedded.1/" REDFISH_ETHERNET_ENDPOINT = f"{REDFISH_SYSTEM_ENDPOINT}EthernetInterfaces/" @@ -67,6 +77,7 @@ def chassis_info(bmc: Bmc) -> ChassisInfo: model_number=chassis_data["Model"], serial_number=chassis_data["SKU"], bios_version=chassis_data["BiosVersion"], + power_on=(chassis_data["PowerState"] == "On"), bmc_ip_address=bmc.ip_address, interfaces=interfaces, ) diff --git a/python/understack-workflows/understack_workflows/bmc_power.py b/python/understack-workflows/understack_workflows/bmc_power.py new file mode 100644 index 00000000..b9ef38a9 --- /dev/null +++ b/python/understack-workflows/understack_workflows/bmc_power.py @@ -0,0 +1,13 @@ +from understack_workflows.bmc import Bmc +from understack_workflows.helpers import setup_logger + +logger = setup_logger(__name__) + + +def bmc_power_on(bmc: Bmc): + """Make a redfish call to switch on the power to the system.""" + bmc.redfish_request( + "/redfish/v1/Systems/System.Embedded.1/Actions/ComputerSystem.Reset", + payload={"ResetType": "On"}, + method="POST", + ) diff --git a/python/understack-workflows/understack_workflows/discover.py b/python/understack-workflows/understack_workflows/discover.py new file mode 100644 index 00000000..e7ffed34 --- /dev/null +++ b/python/understack-workflows/understack_workflows/discover.py @@ -0,0 +1,48 @@ +import time + +from understack_workflows.bmc import Bmc +from understack_workflows.bmc_chassis_info import ChassisInfo +from understack_workflows.bmc_chassis_info import chassis_info +from understack_workflows.bmc_power import bmc_power_on +from understack_workflows.helpers import setup_logger + +logger = setup_logger(__name__) + +MIN_REQUIRED_NEIGHBOR_COUNT = 3 +LLDP_DISCOVERY_ATTEMPTS = 6 + + +def discover_chassis_info(bmc: Bmc) -> ChassisInfo: + """Query redfish, retrying until we get data that is acceptable. + + If the server is off, power it on. + + Make sure that we have at least MIN_REQUIRED_NEIGHBOR_COUNT LLDP neighbors + in the returned ChassisInfo. If that can't be achieved in a reasonable time + then raise an Exception. + """ + device_info = chassis_info(bmc) + + if not device_info.power_on: + logger.info(f"Server is powered off, sending power-on command to {bmc}") + bmc_power_on(bmc) + + attempts_remaining = LLDP_DISCOVERY_ATTEMPTS + while len(device_info.neighbors) < MIN_REQUIRED_NEIGHBOR_COUNT: + logger.info( + f"{bmc} does not have enough LLDP neighbors " + f"(saw {device_info.neighbors}), need at " + f"least {MIN_REQUIRED_NEIGHBOR_COUNT}. " + ) + if not attempts_remaining: + raise Exception( + f"Only {len(device_info.neighbors)} LLDP neighbors appeared, " + f" but {MIN_REQUIRED_NEIGHBOR_COUNT} are required." + ) + logger.info(f"Retry in 30 seconds ({attempts_remaining=})") + attempts_remaining = attempts_remaining - 1 + + time.sleep(30) + device_info = chassis_info(bmc) + + return device_info diff --git a/python/understack-workflows/understack_workflows/helpers.py b/python/understack-workflows/understack_workflows/helpers.py index f2fa6963..8b8310b9 100644 --- a/python/understack-workflows/understack_workflows/helpers.py +++ b/python/understack-workflows/understack_workflows/helpers.py @@ -17,6 +17,7 @@ def setup_logger(name: str | None = None, level: int = logging.DEBUG): """ logging.basicConfig( format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + datefmt="%Y-%m-%d %H:%M:%S %z", level=level, ) return logging.getLogger(name) diff --git a/python/understack-workflows/understack_workflows/main/enroll_server.py b/python/understack-workflows/understack_workflows/main/enroll_server.py index 8fff4475..e287b2ac 100644 --- a/python/understack-workflows/understack_workflows/main/enroll_server.py +++ b/python/understack-workflows/understack_workflows/main/enroll_server.py @@ -8,19 +8,23 @@ from understack_workflows import nautobot_device from understack_workflows import sync_interfaces from understack_workflows import topology +from understack_workflows.bmc import Bmc from understack_workflows.bmc import bmc_for_ip_address from understack_workflows.bmc_bios import update_dell_bios_settings -from understack_workflows.bmc_chassis_info import chassis_info from understack_workflows.bmc_credentials import set_bmc_password from understack_workflows.bmc_hostname import bmc_set_hostname from understack_workflows.bmc_network_config import bmc_set_permanent_ip_addr from understack_workflows.bmc_settings import update_dell_drac_settings +from understack_workflows.discover import discover_chassis_info from understack_workflows.helpers import credential from understack_workflows.helpers import parser_nautobot_args from understack_workflows.helpers import setup_logger +from understack_workflows.nautobot_device import NautobotDevice logger = setup_logger(__name__) +MIN_REQUIRED_NEIGHBOR_COUNT = 4 + def main(): """On-board new or Refresh existing baremetal node. @@ -49,12 +53,10 @@ def main(): - if DHCP, set permanent IP address, netmask, default gw - - TODO: if server is off, power it on and wait (otherwise LLDP doesn't work) + - if server is off, power it on and wait (otherwise LLDP doesn't work) - TODO: create and install SSL certificate - - TODO: update BMC firmware - - TODO: set NTP Server IPs for DRAC (NTP server IP addresses are different per region) @@ -101,13 +103,22 @@ def main(): nautobot = pynautobot.api(url, token=token) bmc = bmc_for_ip_address(bmc_ip_address) + + nb_device = enroll_server(bmc, nautobot, args.old_bmc_password) + + # argo workflows captures stdout as the results which we can use + # to return the device UUID + print(str(nb_device.id)) + + +def enroll_server(bmc: Bmc, nautobot, old_password: str | None) -> NautobotDevice: set_bmc_password( ip_address=bmc.ip_address, new_password=bmc.password, - old_password=args.old_bmc_password, + old_password=old_password, ) - device_info = chassis_info(bmc) + device_info = discover_chassis_info(bmc) logger.info(f"Discovered {pformat(device_info)}") update_dell_drac_settings(bmc) @@ -134,9 +145,7 @@ def main(): logger.info(f"{__file__} complete for {bmc.ip_address}") - # argo workflows captures stdout as the results which we can use - # return the device UUID - print(str(nb_device.id)) + return nb_device def argument_parser():