CMDCT-4238 - Prevent destroy from destroying serverless stacks unless…

… particular resources are set to retain (#15026)

Co-authored-by: Pete Dunlap <[email protected]>

services/app-api/serverless.yml

@@ -77,7 +77,7 @@ custom:
   StateFormsTableStreamArn: ${env:LOCAL_PLACEHOLDER_ARN, param:StateFormsTableStreamArn, cf:database-master.StateFormsTableStreamArn}
   AuthUserRolesTableStreamArn: ${env:LOCAL_PLACEHOLDER_ARN, param:AuthUserRolesTableStreamArn, cf:database-master.AuthUserRolesTableStreamArn}
   bootstrapBrokerStringTls: ${env:LOCAL_DEFAULT_STRING, ssm:/configuration/${self:custom.stage}/seds/bootstrapBrokerStringTls, ssm:/configuration/default/seds/bootstrapBrokerStringTls, ""}
-  vpcId: ${env:LOCAL_DEFAULT_STRING, ssm:/configuration/${self:custom.stage}/vpc/id, ssm:/configuration/default/vpc/id, ""}
+  vpcId: ${ssm:/configuration/${self:custom.stage}/vpc/id, ssm:/configuration/default/vpc/id, ""}
   webAclName: ${self:service}-${self:custom.stage}-webacl-waf
   associateWaf:
     name: ${self:custom.webAclName}

services/ui-auth/serverless.yml

@@ -14,9 +14,9 @@ provider:
   name: aws
   runtime: nodejs20.x
   region: us-east-1
-  stackTags:  
+  stackTags:
     PROJECT: ${self:custom.project}
-    SERVICE: ${self:service}  
+    SERVICE: ${self:service}
   iam:
     role:
       # Even though we are creating our own IAM role that is used in each lambda function below
@@ -151,11 +151,11 @@ resources:
 
     # Associate the WAF Web ACL with the Cognito User Pool
     CognitoUserPoolWAFAssociation:
-      Type: 'AWS::WAFv2::WebACLAssociation'
+      Type: "AWS::WAFv2::WebACLAssociation"
       Properties:
         ResourceArn: !GetAtt CognitoUserPool.Arn
         WebACLArn: !GetAtt WafPluginAcl.Arn
-        
+
     CognitoUserPoolClient:
       Type: AWS::Cognito::UserPoolClient
       Properties:

src/getCloudFormationTemplatesForStage.ts

@@ -0,0 +1,77 @@
+import {
+  CloudFormationClient,
+  GetTemplateCommand,
+  paginateDescribeStacks,
+} from "@aws-sdk/client-cloudformation";
+
+type Tag = {
+  Key: string;
+  Value: string;
+};
+
+async function getAllStacksForRegion(region: string) {
+  const client = new CloudFormationClient({ region: region });
+  const stacks = [];
+  for await (const page of paginateDescribeStacks({ client }, {})) {
+    stacks.push(...(page.Stacks || []));
+  }
+  return stacks;
+}
+
+export async function getAllStacksForStage(
+  region: string,
+  stage: string,
+  addFilters?: Tag[]
+) {
+  let stacks = await getAllStacksForRegion(region);
+  const matchTags = [
+    {
+      Key: "STAGE",
+      Value: stage,
+    },
+  ];
+  matchTags.push(...(addFilters || []));
+  for (let matchTag of matchTags) {
+    stacks = stacks.filter((i) =>
+      i.Tags?.find((j) => j.Key == matchTag.Key && j.Value == matchTag.Value)
+    );
+  }
+  return stacks;
+}
+
+export async function getCloudFormationTemplatesForStage(
+  region: string,
+  stage: string,
+  filters?: Tag[]
+): Promise<Record<string, any>> {
+  const stacks = await getAllStacksForStage(region, stage, filters);
+
+  // If no stacks found, return an empty object
+  if (stacks.length === 0) {
+    return {};
+  }
+
+  const cfnClient = new CloudFormationClient({ region });
+  const templatesByStack: Record<string, any> = {};
+
+  // For each matching stack, retrieve and parse its template
+  for (const stack of stacks) {
+    const { TemplateBody } = await cfnClient.send(
+      new GetTemplateCommand({
+        StackName: stack.StackName,
+      })
+    );
+
+    let parsedTemplate: any;
+    try {
+      parsedTemplate = JSON.parse(TemplateBody ?? "");
+    } catch {
+      console.log("error, received yaml, need to update to handle");
+      console.log(TemplateBody);
+    }
+
+    templatesByStack[stack.StackName!] = parsedTemplate;
+  }
+
+  return templatesByStack;
+}

src/run.ts

@@ -2,6 +2,10 @@ import yargs from "yargs";
 import * as dotenv from "dotenv";
 import LabeledProcessRunner from "./runner.js";
 import { ServerlessStageDestroyer } from "@stratiformdigital/serverless-stage-destroyer";
+import {
+  getAllStacksForStage,
+  getCloudFormationTemplatesForStage,
+} from "./getCloudFormationTemplatesForStage.js";
 import { execSync } from "child_process";
 
 // load .env
@@ -166,6 +170,44 @@ async function deploy(options: { stage: string }) {
   await runner.run_command_and_output("Serverless deploy", deployCmd, ".");
 }
 
+async function getNotRetainedResources(
+  stage: string,
+  filters: { Key: string; Value: string }[] | undefined
+) {
+  const templates = await getCloudFormationTemplatesForStage(
+    `${process.env.REGION_A}`,
+    stage,
+    filters
+  );
+
+  const resourcesToCheck = {
+    [`database-${stage}`]: [
+      "FormAnswersTable",
+      "FormQuestionsTable",
+      "FormTemplatesTable",
+      "FormsTable",
+      "StateFormsTable",
+      "StatesTable",
+      "AuthUserTable",
+    ],
+    [`ui-${stage}`]: ["CloudFrontDistribution"],
+    [`ui-auth-${stage}`]: ["CognitoUserPool"],
+  };
+
+  const notRetained: { templateKey: string; resourceKey: string }[] = [];
+  for (const [templateKey, resourceKeys] of Object.entries(resourcesToCheck)) {
+    resourceKeys.forEach((resourceKey) => {
+      const policy =
+        templates?.[templateKey]?.Resources?.[resourceKey]?.DeletionPolicy;
+      if (policy !== "Retain") {
+        notRetained.push({ templateKey, resourceKey });
+      }
+    });
+  }
+
+  return notRetained;
+}
+
 async function destroy_stage(options: {
   stage: string;
   service: string | undefined;
@@ -186,6 +228,44 @@ async function destroy_stage(options: {
     });
   }
 
+  const stacks = await getAllStacksForStage(
+    `${process.env.REGION_A}`,
+    options.stage,
+    filters
+  );
+
+  const protectedStacks = stacks
+    .filter((i) => i.EnableTerminationProtection)
+    .map((i) => i.StackName);
+
+  if (protectedStacks.length > 0) {
+    console.log(
+      `We cannot proceed with the destroy because the following stacks have termination protection enabled:\n${protectedStacks.join(
+        "\n"
+      )}`
+    );
+    return;
+  } else {
+    console.log(
+      "No stacks have termination protection enabled. Proceeding with the destroy."
+    );
+  }
+
+  let notRetained: { templateKey: string; resourceKey: string }[] = [];
+  if (["master", "val", "production"].includes(options.stage)) {
+    notRetained = await getNotRetainedResources(options.stage, filters);
+  }
+
+  if (notRetained.length > 0) {
+    console.log(
+      "Will not destroy the stage because it's an important stage and some important resources are not yet set to be retained:"
+    );
+    notRetained.forEach(({ templateKey, resourceKey }) =>
+      console.log(` - ${templateKey}/${resourceKey}`)
+    );
+    return;
+  }
+
   await destroyer.destroy(`${process.env.REGION_A}`, options.stage, {
     wait: options.wait,
     filters: filters,