FEATURE: exclude terminating node from alb target (#5)

To prevent 502/504 errors when worker node is terminating, remove the
node from the ALB target group before termination.

modules/kubernetes/functions/handler.py

@@ -9,7 +9,7 @@
 from kubernetes import config as k8s_config
 from kubernetes.client.rest import ApiException
 
-from k8s_utils import (abandon_lifecycle_action, continue_lifecycle_action, cordon_node, node_exists, node_ready, append_node_labels, master_ready, remove_all_pods)
+from k8s_utils import (abandon_lifecycle_action, continue_lifecycle_action, cordon_node, node_exists, node_ready, append_node_labels, master_ready, remove_all_pods, exclude_node_from_loadbalancer)
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -106,6 +106,7 @@ def terminate_node(k8s_api, hook_info):
             return
 
         cordon_node(k8s_api, hook_info['node_name'])
+        exclude_node_from_loadbalancer(k8s_api, hook_info['node_name'])
         remove_all_pods(k8s_api, hook_info['node_name'])
 
         continue_lifecycle_action(asg, hook_info['asg_name'], hook_info['name'], hook_info['instance_id'])

modules/kubernetes/functions/k8s_utils.py

@@ -232,4 +232,20 @@ def continue_lifecycle_action(asg_client, auto_scaling_group_name, lifecycle_hoo
     asg_client.complete_lifecycle_action(LifecycleHookName=lifecycle_hook_name,
                                          AutoScalingGroupName=auto_scaling_group_name,
                                          LifecycleActionResult='CONTINUE',
-                                         InstanceId=instance_id)
+                                         InstanceId=instance_id)
+
+def exclude_node_from_loadbalancer(api, node_name):
+    """Excludes the node from external load balancers, such as AWS load balancer target groups.
+    """
+    patch_body = {
+        "metadata": {
+            "labels": {
+                "node.kubernetes.io/exclude-from-external-load-balancers": "asg-lifecycle-hook"
+            }
+        }
+    }
+
+    try:
+        api.patch_node(node_name, patch_body)
+    except:
+        logger.exception('There was an error appending exclude from loadbalancer label to the node {} '.format(node_name))