From aededbd4f0ca3dc91bb01fa0a14e62ed782552ab Mon Sep 17 00:00:00 2001
From: Tim Meusel <tim@bastelfreak.de>
Date: Thu, 25 Apr 2024 12:39:44 +0200
Subject: [PATCH] Add function to check if nodes are reachable via bolt

At the moment the plans assume that all nodes are available. I had a few
customer setups where one of the compilers wasn't reachable during a
convert/upgrade. To not put the PE infra into an undefined state, it
makes sense to check the availability before running the plans.
---
 REFERENCE.md                    | 25 +++++++++++++++++++++++++
 functions/check_availability.pp | 30 ++++++++++++++++++++++++++++++
 plans/convert.pp                |  1 +
 plans/install.pp                |  9 +++++++++
 plans/upgrade.pp                |  1 +
 5 files changed, 66 insertions(+)
 create mode 100644 functions/check_availability.pp

diff --git a/REFERENCE.md b/REFERENCE.md
index e09e7f17..173ab89d 100644
--- a/REFERENCE.md
+++ b/REFERENCE.md
@@ -20,6 +20,7 @@
 * [`peadm::assert_supported_pe_version`](#peadm--assert_supported_pe_version): Assert that the PE version given is supported by PEAdm
 * [`peadm::bolt_version`](#peadm--bolt_version)
 * [`peadm::certname`](#peadm--certname): Return the certname of the given target-like input
+* [`peadm::check_availability`](#peadm--check_availability): check if a group of targets are reachable for bolt
 * [`peadm::check_version_and_known_hosts`](#peadm--check_version_and_known_hosts): Checks PE verison and warns about setting r10k_known_hosts
 * [`peadm::convert_hash`](#peadm--convert_hash): converts two arrays into hash
 * [`peadm::convert_status`](#peadm--convert_status): Transforms a value in a human readable status with or without colors
@@ -266,6 +267,30 @@ Variant[Target,
 
 
 
+### <a name="peadm--check_availability"></a>`peadm::check_availability`
+
+Type: Puppet Language
+
+check if a group of targets are reachable for bolt
+
+#### `peadm::check_availability(TargetSpec $targets, Boolean $output_details = true)`
+
+The peadm::check_availability function.
+
+Returns: `Integer` counter for unavailable nodes
+
+##### `targets`
+
+Data type: `TargetSpec`
+
+list of targets that are going to be checked
+
+##### `output_details`
+
+Data type: `Boolean`
+
+flag to enable/disable error output for failed nodes
+
 ### <a name="peadm--check_version_and_known_hosts"></a>`peadm::check_version_and_known_hosts`
 
 Type: Puppet Language
diff --git a/functions/check_availability.pp b/functions/check_availability.pp
new file mode 100644
index 00000000..4dcde5fb
--- /dev/null
+++ b/functions/check_availability.pp
@@ -0,0 +1,30 @@
+#
+# @summary check if a group of targets are reachable for bolt
+#
+# @param targets list of targets that are going to be checked
+# @param output_details flag to enable/disable error output for failed nodes
+#
+# @return counter for unavailable nodes
+#
+# @author Tim Meusel <tim@bastelfreak.de>
+#
+function peadm::check_availability(
+  TargetSpec $targets,
+  Boolean $output_details = true
+) >> Integer {
+  $check_result = wait_until_available($targets, wait_time => 2, _catch_errors => true)
+  if $check_result.ok {
+  } else {
+    $messages = $check_result.error_set.map |$result| {
+      "## node ${result.target} has connection error '${result.error.kind}' with message '${result.error.msg}'"
+    }
+    $end_message = "${check_result.error_set.count} targets are not reachable, stopping plan"
+    $fail_message = $output_details ? {
+      true  => "${messages.join("\n")}\n${end_message}",
+      false => $end_message,
+    }
+    fail_plan($fail_message)
+  }
+
+  return $check_result.error_set.count
+}
diff --git a/plans/convert.pp b/plans/convert.pp
index 1995a0b0..310212dd 100644
--- a/plans/convert.pp
+++ b/plans/convert.pp
@@ -56,6 +56,7 @@
   )
 
   out::message('# Gathering information')
+  peadm::check_availability($all_targets)
 
   # Get trusted fact information for all compilers. Use peadm::certname() as
   # the hash key because the apply block below will break trying to parse the
diff --git a/plans/install.pp b/plans/install.pp
index 1f131533..d742e765 100644
--- a/plans/install.pp
+++ b/plans/install.pp
@@ -76,6 +76,15 @@
 ) {
   peadm::assert_supported_bolt_version()
 
+  out::message('# Gathering information')
+  $all_targets = peadm::flatten_compact([
+      $primary_host,
+      $replica_host,
+      $replica_postgresql_host,
+      $compiler_hosts,
+      $primary_postgresql_host,
+  ])
+  peadm::check_availability($all_targets)
   peadm::assert_supported_pe_version($version, $permit_unsafe_versions)
 
   $install_result = run_plan('peadm::subplans::install',
diff --git a/plans/upgrade.pp b/plans/upgrade.pp
index 06ac068d..2e03b724 100644
--- a/plans/upgrade.pp
+++ b/plans/upgrade.pp
@@ -98,6 +98,7 @@
   ])
 
   out::message('# Gathering information')
+  peadm::check_availability($all_targets)
 
   # lint:ignore:strict_indent
   $primary_target.peadm::fail_on_transport('pcp', @(HEREDOC/n))