feat: Have Enroll power on server and retry fetching LLDP info

python/understack-workflows/tests/test_bmc_chassis_info.py

@@ -40,6 +40,16 @@ def read_fixture(path, filename):
         return json.loads(f.read())
 
 
+def test_chassis_neighbors():
+    bmc = FakeBmc(read_fixtures("json_samples/bmc_chassis_info/R7615"))
+    chassis_info = bmc_chassis_info.chassis_info(bmc)
+    assert chassis_info.neighbors == {
+        "C4:7E:E0:E4:10:7F",
+        "C4:4D:84:48:61:80",
+        "C4:7E:E0:E4:32:DF",
+    }
+
+
 def test_chassis_info_R7615():
     bmc = FakeBmc(read_fixtures("json_samples/bmc_chassis_info/R7615"))
     assert bmc_chassis_info.chassis_info(bmc) == bmc_chassis_info.ChassisInfo(
@@ -48,6 +58,7 @@ def test_chassis_info_R7615():
         serial_number="33GSW04",
         bios_version="1.6.10",
         bmc_ip_address="1.2.3.4",
+        power_on=True,
         interfaces=[
             bmc_chassis_info.InterfaceInfo(
                 name="iDRAC",

python/understack-workflows/tests/test_nautobot_device.py

@@ -118,6 +118,7 @@ def test_find_or_create(dell_nautobot_device):
         serial_number="33GSW04",
         bios_version="1.6.10",
         bmc_ip_address="1.2.3.4",
+        power_on=True,
         interfaces=[
             InterfaceInfo(
                 name="iDRAC",

python/understack-workflows/understack_workflows/bmc_chassis_info.py

@@ -30,6 +30,7 @@ class ChassisInfo:
     serial_number: str
     bmc_ip_address: str
     bios_version: str
+    power_on: bool
     interfaces: list[InterfaceInfo]
 
     @property
@@ -40,6 +41,15 @@ def bmc_interface(self) -> InterfaceInfo:
     def bmc_hostname(self) -> str:
         return str(self.bmc_interface.hostname)
 
+    @property
+    def neighbors(self) -> set:
+        """A set of switch MAC addresses to which this chassis is connected."""
+        return {
+            interface.remote_switch_mac_address
+            for interface in self.interfaces
+            if interface.remote_switch_mac_address
+        }
+
 
 REDFISH_SYSTEM_ENDPOINT = "/redfish/v1/Systems/System.Embedded.1/"
 REDFISH_ETHERNET_ENDPOINT = f"{REDFISH_SYSTEM_ENDPOINT}EthernetInterfaces/"
@@ -67,6 +77,7 @@ def chassis_info(bmc: Bmc) -> ChassisInfo:
         model_number=chassis_data["Model"],
         serial_number=chassis_data["SKU"],
         bios_version=chassis_data["BiosVersion"],
+        power_on=(chassis_data["PowerState"] == "On"),
         bmc_ip_address=bmc.ip_address,
         interfaces=interfaces,
     )

python/understack-workflows/understack_workflows/bmc_power.py

@@ -0,0 +1,13 @@
+from understack_workflows.bmc import Bmc
+from understack_workflows.helpers import setup_logger
+
+logger = setup_logger(__name__)
+
+
+def bmc_power_on(bmc: Bmc):
+    """Make a redfish call to switch on the power to the system."""
+    bmc.redfish_request(
+        "/redfish/v1/Systems/System.Embedded.1/Actions/ComputerSystem.Reset",
+        payload={"ResetType": "On"},
+        method="POST",
+    )

python/understack-workflows/understack_workflows/discover.py

@@ -0,0 +1,48 @@
+import time
+
+from understack_workflows.bmc import Bmc
+from understack_workflows.bmc_chassis_info import ChassisInfo
+from understack_workflows.bmc_chassis_info import chassis_info
+from understack_workflows.bmc_power import bmc_power_on
+from understack_workflows.helpers import setup_logger
+
+logger = setup_logger(__name__)
+
+MIN_REQUIRED_NEIGHBOR_COUNT = 3
+LLDP_DISCOVERY_ATTEMPTS = 6
+
+
+def discover_chassis_info(bmc: Bmc) -> ChassisInfo:
+    """Query redfish, retrying until we get data that is acceptable.
+
+    If the server is off, power it on.
+
+    Make sure that we have at least MIN_REQUIRED_NEIGHBOR_COUNT LLDP neighbors
+    in the returned ChassisInfo.  If that can't be achieved in a reasonable time
+    then raise an Exception.
+    """
+    device_info = chassis_info(bmc)
+
+    if not device_info.power_on:
+        logger.info(f"Server is powered off, sending power-on command to {bmc}")
+        bmc_power_on(bmc)
+
+    attempts_remaining = LLDP_DISCOVERY_ATTEMPTS
+    while len(device_info.neighbors) < MIN_REQUIRED_NEIGHBOR_COUNT:
+        logger.info(
+            f"{bmc} does not have enough LLDP neighbors "
+            f"(saw {device_info.neighbors}), need at "
+            f"least {MIN_REQUIRED_NEIGHBOR_COUNT}. "
+        )
+        if not attempts_remaining:
+            raise Exception(
+                f"Only {len(device_info.neighbors)} LLDP neighbors appeared, "
+                f" but {MIN_REQUIRED_NEIGHBOR_COUNT} are required."
+            )
+        logger.info(f"Retry in 30 seconds ({attempts_remaining=})")
+        attempts_remaining = attempts_remaining - 1
+
+        time.sleep(30)
+        device_info = chassis_info(bmc)
+
+    return device_info

python/understack-workflows/understack_workflows/helpers.py

@@ -17,6 +17,7 @@ def setup_logger(name: str | None = None, level: int = logging.DEBUG):
     """
     logging.basicConfig(
         format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S %z",
         level=level,
     )
     return logging.getLogger(name)

python/understack-workflows/understack_workflows/main/enroll_server.py

@@ -8,19 +8,23 @@
 from understack_workflows import nautobot_device
 from understack_workflows import sync_interfaces
 from understack_workflows import topology
+from understack_workflows.bmc import Bmc
 from understack_workflows.bmc import bmc_for_ip_address
 from understack_workflows.bmc_bios import update_dell_bios_settings
-from understack_workflows.bmc_chassis_info import chassis_info
 from understack_workflows.bmc_credentials import set_bmc_password
 from understack_workflows.bmc_hostname import bmc_set_hostname
 from understack_workflows.bmc_network_config import bmc_set_permanent_ip_addr
 from understack_workflows.bmc_settings import update_dell_drac_settings
+from understack_workflows.discover import discover_chassis_info
 from understack_workflows.helpers import credential
 from understack_workflows.helpers import parser_nautobot_args
 from understack_workflows.helpers import setup_logger
+from understack_workflows.nautobot_device import NautobotDevice
 
 logger = setup_logger(__name__)
 
+MIN_REQUIRED_NEIGHBOR_COUNT = 4
+
 
 def main():
     """On-board new or Refresh existing baremetal node.
@@ -49,12 +53,10 @@ def main():
 
     - if DHCP, set permanent IP address, netmask, default gw
 
-    - TODO: if server is off, power it on and wait (otherwise LLDP doesn't work)
+    - if server is off, power it on and wait (otherwise LLDP doesn't work)
 
     - TODO: create and install SSL certificate
 
-    - TODO: update BMC firmware
-
     - TODO: set NTP Server IPs for DRAC
       (NTP server IP addresses are different per region)
 
@@ -101,13 +103,22 @@ def main():
     nautobot = pynautobot.api(url, token=token)
 
     bmc = bmc_for_ip_address(bmc_ip_address)
+
+    nb_device = enroll_server(bmc, nautobot, args.old_bmc_password)
+
+    # argo workflows captures stdout as the results which we can use
+    # to return the device UUID
+    print(str(nb_device.id))
+
+
+def enroll_server(bmc: Bmc, nautobot, old_password: str | None) -> NautobotDevice:
     set_bmc_password(
         ip_address=bmc.ip_address,
         new_password=bmc.password,
-        old_password=args.old_bmc_password,
+        old_password=old_password,
     )
 
-    device_info = chassis_info(bmc)
+    device_info = discover_chassis_info(bmc)
     logger.info(f"Discovered {pformat(device_info)}")
 
     update_dell_drac_settings(bmc)
@@ -134,9 +145,7 @@ def main():
 
     logger.info(f"{__file__} complete for {bmc.ip_address}")
 
-    # argo workflows captures stdout as the results which we can use
-    # return the device UUID
-    print(str(nb_device.id))
+    return nb_device
 
 
 def argument_parser():