Skip to content

Commit

Permalink
Split resource and stack update handling
Browse files Browse the repository at this point in the history
  • Loading branch information
farski committed Sep 23, 2024
1 parent eb5f9ff commit f27e7fa
Show file tree
Hide file tree
Showing 3 changed files with 162 additions and 54 deletions.
10 changes: 7 additions & 3 deletions src/status-change-slack-notifications/index.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ import {
EventBridgeClient,
PutEventsCommand,
} from "@aws-sdk/client-eventbridge";
import stackMessage from "./stack-change.mjs";
import stackStatusChangeMessage from "./stack-status-change.mjs";
import resourceStatusChangeMessage from "./resource-status-change.mjs";

import stackSetMessage from "./stack-set-change.mjs";

const eventbridge = new EventBridgeClient({ apiVersion: "2015-10-07" });
Expand All @@ -22,8 +24,10 @@ export const handler = async (event) => {

if (event["detail-type"].includes("CloudFormation StackSet")) {
message = stackSetMessage(event);
} else {
message = stackMessage(event);
} else if (event["detail-type"] === "CloudFormation Stack Status Change") {
message = stackStatusChangeMessage(event);
} else if (event["detail-type"] === "CloudFormation Resource Status Change") {
message = resourceStatusChangeMessage(event);
}

if (message) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import regions from "./regions.mjs";
import accounts from "./accounts.mjs";

const SLACK_DEBUG_CHANNEL = "G2QHC11SM"; // #ops-debug
const SLACK_INFO_CHANNEL = "G2QHBL6UX"; // #ops-info
const SLACK_SKIPPED_CHANNEL = "C0358B7NE9W"; // #ops-delete-skipped
const SLACK_ICON = ":ops-cloudformation:";
const SLACK_USERNAME = "AWS CloudFormation";

Expand Down Expand Up @@ -55,20 +55,6 @@ function colorForResourceStatus(status) {
return "#000000";
}

const concerning = [
"ROLLBACK_COMPLETE",
"UPDATE_ROLLBACK_COMPLETE",
"DELETE_IN_PROGRESS",
"ROLLBACK_IN_PROGRESS",
"CREATE_FAILED",
"DELETE_FAILED",
"UPDATE_FAILED",
"ROLLBACK_FAILED",
"UPDATE_ROLLBACK_FAILED",
"UPDATE_ROLLBACK_IN_PROGRESS",
"UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS",
];

export default function message(event) {
// Each event includes information about the stack where the change is
// happening. These will be present on both stack status and resource status
Expand Down Expand Up @@ -98,16 +84,13 @@ export default function message(event) {

const regionNickname = regions(region);
const accountNickname = accounts(event.account);

const header = [
`*<${deepStackUrl}|${accountNickname} - ${regionNickname} » ${stackName}>*`,
resourceType
? `Resource Status Change: *${status}* for \`${resourceType}\``
: `Stack Status Change: *${status}*`,
`Resource Status Change: *${status}* for \`${resourceType}\``,
].join("\n");

const fallback = resourceType
? `${accountNickname} - ${regionNickname} » ${resourceType} ${logicalResourceId} in ${stackName} is now ${status}`
: `${accountNickname} - ${regionNickname} » Stack ${stackName} is now ${status}`;
const fallback = `${accountNickname} - ${regionNickname} » ${resourceType} ${logicalResourceId} in ${stackName} is now ${status}`;

const msg = {
username: SLACK_USERNAME,
Expand All @@ -133,7 +116,7 @@ export default function message(event) {
// DELETE_SKIPPED events are funnelled to a specific Slack channel so they
// can be cleaned up if necessary
if (status === "DELETE_SKIPPED") {
msg.channel = "#ops-delete-skipped";
msg.channel = SLACK_SKIPPED_CHANNEL;
msg.attachments[0].blocks.push({
type: "section",
text: {
Expand All @@ -147,33 +130,6 @@ export default function message(event) {
return msg;
}

// For Spire root stacks, send all start, finish, and concerning status
// notifications to INFO
// For stack status events, send all start, finish, and concerning
// notifications to the INFO channel
if (
!resourceType &&
(stackName.endsWith("root-staging") ||
stackName.endsWith("root-production")) &&
(concerning.includes(status) ||
["UPDATE_IN_PROGRESS", "UPDATE_COMPLETE"].includes(status))
) {
msg.channel = SLACK_INFO_CHANNEL;
// eslint-disable-next-line consistent-return
return;
}

// For other stacks, send finish and concerning notifications to DEBUG
if (
!resourceType &&
(concerning.includes(status) || ["UPDATE_COMPLETE"].includes(status)) &&
!stackName.includes("infrastructure-cd-root-") &&
!stackName.startsWith("StackSet-")
) {
msg.channel = SLACK_DEBUG_CHANNEL;
return msg;
}

// For everything that isn't a root stack, send any notifications that
// include a reason to DEBUG. Reasons are most often provided when there is
// an issue ("resources failed to create", "handler returned message", etc).
Expand All @@ -193,6 +149,5 @@ export default function message(event) {
return msg;
}

// eslint-disable-next-line consistent-return, no-useless-return
return;
return false;
}
149 changes: 149 additions & 0 deletions src/status-change-slack-notifications/stack-status-change.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
import regions from "./regions.mjs";
import accounts from "./accounts.mjs";

const SLACK_DEBUG_CHANNEL = "G2QHC11SM"; // #ops-debug
// const SLACK_INFO_CHANNEL = "G2QHBL6UX"; // #ops-info
const SLACK_ICON = ":ops-cloudformation:";
const SLACK_USERNAME = "AWS CloudFormation";

// These colors match events in the CloudFormation console
function colorForResourceStatus(status) {
const green = [
"CREATE_COMPLETE",
"ROLLBACK_COMPLETE",
"UPDATE_COMPLETE",
"UPDATE_ROLLBACK_COMPLETE",
];

const yellow = [
"CREATE_IN_PROGRESS",
"DELETE_IN_PROGRESS",
"REVIEW_IN_PROGRESS",
"ROLLBACK_IN_PROGRESS",
"UPDATE_IN_PROGRESS",
"UPDATE_COMPLETE_CLEANUP_IN_PROGRESS",
];

const red = [
"CREATE_FAILED",
"DELETE_FAILED",
"UPDATE_FAILED",
"ROLLBACK_FAILED",
"UPDATE_ROLLBACK_FAILED",
"UPDATE_ROLLBACK_IN_PROGRESS",
"UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS",
];

const grey = ["DELETE_COMPLETE"];

if (green.includes(status)) {
return "#2eb886";
}

if (yellow.includes(status)) {
return "#f4f323";
}

if (red.includes(status)) {
return "#a30200";
}

if (grey.includes(status)) {
return "#AAAAAA";
}

return "#000000";
}

const concerning = [
"CREATE_FAILED",
"DELETE_FAILED",
"DELETE_IN_PROGRESS",
"ROLLBACK_COMPLETE",
"ROLLBACK_FAILED",
"ROLLBACK_IN_PROGRESS",
"UPDATE_FAILED",
"UPDATE_ROLLBACK_COMPLETE",
"UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS",
"UPDATE_ROLLBACK_FAILED",
"UPDATE_ROLLBACK_IN_PROGRESS",
"IMPORT_ROLLBACK_FAILED",
];

export default function message(event) {
// Each event includes information about the stack where the change is
// happening. These will be present on both stack status and resource status
// events.
const stackId = event.detail["stack-id"];

// Extract the stack name from the stack ID
const stackName = stackId.split(":stack/")[1].split("/")[0];

// Both stack status and resource status events will have a status and reason
const { status } = event.detail["status-details"];
const statusReason = event.detail["status-details"]["status-reason"];

const { region } = event;
const stackUrl = `https://${region}.console.aws.amazon.com/cloudformation/home?region=${region}#/stacks/stackinfo?stackId=${stackId}`;

const deepLinkRoleName = "AdministratorAccess";
const urlEncodedStackUrl = encodeURIComponent(stackUrl);
const deepStackUrl = `https://d-906713e952.awsapps.com/start/#/console?account_id=${event.account}&role_name=${deepLinkRoleName}&destination=${urlEncodedStackUrl}`;

const regionNickname = regions(region);
const accountNickname = accounts(event.account);
const header = [
`*<${deepStackUrl}|${accountNickname} - ${regionNickname} » ${stackName}>*`,
`Stack Status Change: *${status}*`,
].join("\n");

const fallback = `${accountNickname} - ${regionNickname} » Stack ${stackName} is now ${status}`;

const msg = {
username: SLACK_USERNAME,
icon_emoji: SLACK_ICON,
channel: "#sandbox2",
attachments: [
{
color: colorForResourceStatus(status),
fallback,
blocks: [
{
type: "section",
text: {
type: "mrkdwn",
text: header,
},
},
],
},
],
};

// Send end-stage and concerning notifications to DEBUG
if (concerning.includes(status) || status.endsWith("_COMPLETE")) {
msg.channel = SLACK_DEBUG_CHANNEL;
return msg;
}

// Send any notifications that
// include a reason to DEBUG. Reasons are most often provided when there is
// an issue ("resources failed to create", "handler returned message", etc).
// But some nominal updates do include reasons.
// Certain irrelevant reasons are filtered out.
if (
statusReason &&
![
"User Initiated",
"Transformation succeeded",
"Resource creation Initiated",
"Requested update required the provider to create a new physical resource",
"Requested update requires the creation of a new physical resource; hence creating one.",
].includes(statusReason)
) {
msg.channel = SLACK_DEBUG_CHANNEL;
return msg;
}

return false;
}

0 comments on commit f27e7fa

Please sign in to comment.