From 72d1caa7979e668f86f100c19ec238af05a60207 Mon Sep 17 00:00:00 2001
From: Jon Holman <9025884+JonHolman@users.noreply.github.com>
Date: Thu, 9 Jan 2025 11:22:32 -0800
Subject: [PATCH 01/12] add serverless-iam-helper to services where it was
 missing

---
 services/database/serverless.yml | 1 +
 services/ui-auth/serverless.yml  | 9 +++++----
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/services/database/serverless.yml b/services/database/serverless.yml
index f29e06793..9a4bc48af 100644
--- a/services/database/serverless.yml
+++ b/services/database/serverless.yml
@@ -6,6 +6,7 @@ plugins:
   - serverless-s3-bucket-helper
   - serverless-dynamodb
   - serverless-stack-termination-protection
+  - serverless-iam-helper
   - serverless-plugin-scripts
   - serverless-offline
 
diff --git a/services/ui-auth/serverless.yml b/services/ui-auth/serverless.yml
index ba3a4d1cf..862bb811d 100644
--- a/services/ui-auth/serverless.yml
+++ b/services/ui-auth/serverless.yml
@@ -5,6 +5,7 @@ frameworkVersion: "3"
 plugins:
   - serverless-s3-bucket-helper
   - serverless-stack-termination-protection
+  - serverless-iam-helper
   - serverless-plugin-scripts
   - serverless-bundle
   - serverless-iam-helper
@@ -14,9 +15,9 @@ provider:
   name: aws
   runtime: nodejs20.x
   region: us-east-1
-  stackTags:  
+  stackTags:
     PROJECT: ${self:custom.project}
-    SERVICE: ${self:service}  
+    SERVICE: ${self:service}
   iam:
     role:
       # Even though we are creating our own IAM role that is used in each lambda function below
@@ -151,11 +152,11 @@ resources:
 
     # Associate the WAF Web ACL with the Cognito User Pool
     CognitoUserPoolWAFAssociation:
-      Type: 'AWS::WAFv2::WebACLAssociation'
+      Type: "AWS::WAFv2::WebACLAssociation"
       Properties:
         ResourceArn: !GetAtt CognitoUserPool.Arn
         WebACLArn: !GetAtt WafPluginAcl.Arn
-        
+
     CognitoUserPoolClient:
       Type: AWS::Cognito::UserPoolClient
       Properties:

From 6261d68d8fa20d52ac6eac19f5037624e24b961c Mon Sep 17 00:00:00 2001
From: Jon Holman <9025884+JonHolman@users.noreply.github.com>
Date: Thu, 9 Jan 2025 12:56:12 -0800
Subject: [PATCH 02/12] fix vpcId

---
 services/app-api/serverless.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/services/app-api/serverless.yml b/services/app-api/serverless.yml
index c74527b0c..bbd66fe06 100644
--- a/services/app-api/serverless.yml
+++ b/services/app-api/serverless.yml
@@ -77,7 +77,7 @@ custom:
   StateFormsTableStreamArn: ${env:LOCAL_PLACEHOLDER_ARN, param:StateFormsTableStreamArn, cf:database-master.StateFormsTableStreamArn}
   AuthUserRolesTableStreamArn: ${env:LOCAL_PLACEHOLDER_ARN, param:AuthUserRolesTableStreamArn, cf:database-master.AuthUserRolesTableStreamArn}
   bootstrapBrokerStringTls: ${env:LOCAL_DEFAULT_STRING, ssm:/configuration/${self:custom.stage}/seds/bootstrapBrokerStringTls, ssm:/configuration/default/seds/bootstrapBrokerStringTls, ""}
-  vpcId: ${env:LOCAL_DEFAULT_STRING, ssm:/configuration/${self:custom.stage}/vpc/id, ssm:/configuration/default/vpc/id, ""}
+  vpcId: ${ssm:/configuration/${self:custom.stage}/vpc/id, ssm:/configuration/default/vpc/id, "jon"}
   webAclName: ${self:service}-${self:custom.stage}-webacl-waf
   associateWaf:
     name: ${self:custom.webAclName}
@@ -95,9 +95,9 @@ provider:
   name: aws
   runtime: nodejs20.x
   region: us-east-1
-  stackTags:  
+  stackTags:
     PROJECT: ${self:custom.project}
-    SERVICE: ${self:service}  
+    SERVICE: ${self:service}
   tracing:
     apiGateway: true
   logs:

From cef60ba1cdc356df56084a71fc03e3591db717c8 Mon Sep 17 00:00:00 2001
From: Jon Holman <9025884+JonHolman@users.noreply.github.com>
Date: Thu, 9 Jan 2025 14:32:17 -0800
Subject: [PATCH 03/12] Create getCloudFormationTemplatesForStage.ts

---
 src/getCloudFormationTemplatesForStage.ts | 77 +++++++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 src/getCloudFormationTemplatesForStage.ts

diff --git a/src/getCloudFormationTemplatesForStage.ts b/src/getCloudFormationTemplatesForStage.ts
new file mode 100644
index 000000000..a5c396c19
--- /dev/null
+++ b/src/getCloudFormationTemplatesForStage.ts
@@ -0,0 +1,77 @@
+import {
+  CloudFormationClient,
+  GetTemplateCommand,
+  paginateDescribeStacks,
+} from "@aws-sdk/client-cloudformation";
+
+type Tag = {
+  Key: string;
+  Value: string;
+};
+
+async function getAllStacksForRegion(region: string) {
+  const client = new CloudFormationClient({ region: region });
+  const stacks = [];
+  for await (const page of paginateDescribeStacks({ client }, {})) {
+    stacks.push(...(page.Stacks || []));
+  }
+  return stacks;
+}
+
+async function getAllStacksForStage(
+  region: string,
+  stage: string,
+  addFilters?: Tag[]
+) {
+  let stacks = await getAllStacksForRegion(region);
+  const matchTags = [
+    {
+      Key: "STAGE",
+      Value: stage,
+    },
+  ];
+  matchTags.push(...(addFilters || []));
+  for (let matchTag of matchTags) {
+    stacks = stacks.filter((i) =>
+      i.Tags?.find((j) => j.Key == matchTag.Key && j.Value == matchTag.Value)
+    );
+  }
+  return stacks;
+}
+
+export async function getCloudFormationTemplatesForStage(
+  region: string,
+  stage: string,
+  filters?: Tag[]
+): Promise<Record<string, any>> {
+  const stacks = await getAllStacksForStage(region, stage, filters);
+
+  // If no stacks found, return an empty object
+  if (stacks.length === 0) {
+    return {};
+  }
+
+  const cfnClient = new CloudFormationClient({ region });
+  const templatesByStack: Record<string, any> = {};
+
+  // For each matching stack, retrieve and parse its template
+  for (const stack of stacks) {
+    const { TemplateBody } = await cfnClient.send(
+      new GetTemplateCommand({
+        StackName: stack.StackName,
+      })
+    );
+
+    let parsedTemplate: any;
+    try {
+      parsedTemplate = JSON.parse(TemplateBody ?? "");
+    } catch {
+      console.log("error, received yaml, need to update to handle");
+      console.log(TemplateBody);
+    }
+
+    templatesByStack[stack.StackName!] = parsedTemplate;
+  }
+
+  return templatesByStack;
+}

From 476105d21432f80a12f743a86577564b7e5e1e90 Mon Sep 17 00:00:00 2001
From: Jon Holman <9025884+JonHolman@users.noreply.github.com>
Date: Thu, 9 Jan 2025 14:37:03 -0800
Subject: [PATCH 04/12] Add logic to check that important resources are set to
 be retained prior to destroy

---
 src/run.ts | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/src/run.ts b/src/run.ts
index b53907d04..e525a6e2a 100644
--- a/src/run.ts
+++ b/src/run.ts
@@ -2,6 +2,7 @@ import yargs from "yargs";
 import * as dotenv from "dotenv";
 import LabeledProcessRunner from "./runner.js";
 import { ServerlessStageDestroyer } from "@stratiformdigital/serverless-stage-destroyer";
+import { getCloudFormationTemplatesForStage } from "./getCloudFormationTemplatesForStage.js";
 import { execSync } from "child_process";
 
 // load .env
@@ -186,6 +187,57 @@ async function destroy_stage(options: {
     });
   }
 
+  const templates = await getCloudFormationTemplatesForStage(
+    `${process.env.REGION_A}`,
+    options.stage,
+    filters
+  );
+
+  const resourcesToCheck = [
+    {
+      templateKey: `database-${options.stage}`,
+      resourceKey: "FormAnswersTable",
+    },
+    {
+      templateKey: `database-${options.stage}`,
+      resourceKey: "FormQuestionsTable",
+    },
+    {
+      templateKey: `database-${options.stage}`,
+      resourceKey: "FormTemplatesTable",
+    },
+    { templateKey: `database-${options.stage}`, resourceKey: "FormsTable" },
+    {
+      templateKey: `database-${options.stage}`,
+      resourceKey: "StateFormsTable",
+    },
+    { templateKey: `database-${options.stage}`, resourceKey: "StatesTable" },
+    { templateKey: `database-${options.stage}`, resourceKey: "AuthUserTable" },
+    {
+      templateKey: `ui-${options.stage}`,
+      resourceKey: "CloudFrontDistribution",
+    },
+    { templateKey: `ui-auth-${options.stage}`, resourceKey: "CognitoUserPool" },
+  ];
+
+  const notRetained = resourcesToCheck.filter(
+    ({ templateKey, resourceKey }) => {
+      const policy =
+        templates?.[templateKey]?.Resources?.[resourceKey]?.DeletionPolicy;
+      return policy !== "Retain";
+    }
+  );
+
+  if (notRetained.length > 0) {
+    console.log(
+      "Will not destroy the stage because some important resources are not yet set to be retained:"
+    );
+    notRetained.forEach(({ templateKey, resourceKey }) =>
+      console.log(` - ${templateKey}/${resourceKey}`)
+    );
+    return;
+  }
+
   await destroyer.destroy(`${process.env.REGION_A}`, options.stage, {
     wait: options.wait,
     filters: filters,

From 3c6548cd268544c2e3544f71ae0215d10547eda4 Mon Sep 17 00:00:00 2001
From: Jon Holman <9025884+JonHolman@users.noreply.github.com>
Date: Thu, 9 Jan 2025 14:44:59 -0800
Subject: [PATCH 05/12] typos

---
 services/app-api/serverless.yml | 3 +--
 services/ui-auth/serverless.yml | 1 -
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/services/app-api/serverless.yml b/services/app-api/serverless.yml
index bbd66fe06..69be55877 100644
--- a/services/app-api/serverless.yml
+++ b/services/app-api/serverless.yml
@@ -12,7 +12,6 @@ plugins:
   - serverless-offline
   - serverless-associate-waf
   - serverless-stack-termination-protection
-  - serverless-iam-helper
   - serverless-offline-ssm
   - "@enterprise-cmcs/serverless-waf-plugin"
 
@@ -77,7 +76,7 @@ custom:
   StateFormsTableStreamArn: ${env:LOCAL_PLACEHOLDER_ARN, param:StateFormsTableStreamArn, cf:database-master.StateFormsTableStreamArn}
   AuthUserRolesTableStreamArn: ${env:LOCAL_PLACEHOLDER_ARN, param:AuthUserRolesTableStreamArn, cf:database-master.AuthUserRolesTableStreamArn}
   bootstrapBrokerStringTls: ${env:LOCAL_DEFAULT_STRING, ssm:/configuration/${self:custom.stage}/seds/bootstrapBrokerStringTls, ssm:/configuration/default/seds/bootstrapBrokerStringTls, ""}
-  vpcId: ${ssm:/configuration/${self:custom.stage}/vpc/id, ssm:/configuration/default/vpc/id, "jon"}
+  vpcId: ${ssm:/configuration/${self:custom.stage}/vpc/id, ssm:/configuration/default/vpc/id, ""}
   webAclName: ${self:service}-${self:custom.stage}-webacl-waf
   associateWaf:
     name: ${self:custom.webAclName}
diff --git a/services/ui-auth/serverless.yml b/services/ui-auth/serverless.yml
index 862bb811d..27918d7a4 100644
--- a/services/ui-auth/serverless.yml
+++ b/services/ui-auth/serverless.yml
@@ -5,7 +5,6 @@ frameworkVersion: "3"
 plugins:
   - serverless-s3-bucket-helper
   - serverless-stack-termination-protection
-  - serverless-iam-helper
   - serverless-plugin-scripts
   - serverless-bundle
   - serverless-iam-helper

From 88ef85bc0eddd09eb4cc5c309b36f00843bf5ea9 Mon Sep 17 00:00:00 2001
From: Jon Holman <9025884+JonHolman@users.noreply.github.com>
Date: Thu, 9 Jan 2025 14:48:45 -0800
Subject: [PATCH 06/12] undo accidental removal of serverless-iam-helper

---
 services/app-api/serverless.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/services/app-api/serverless.yml b/services/app-api/serverless.yml
index 69be55877..579ccc11c 100644
--- a/services/app-api/serverless.yml
+++ b/services/app-api/serverless.yml
@@ -12,6 +12,7 @@ plugins:
   - serverless-offline
   - serverless-associate-waf
   - serverless-stack-termination-protection
+  - serverless-iam-helper
   - serverless-offline-ssm
   - "@enterprise-cmcs/serverless-waf-plugin"
 

From 6b1cfb00e647eb5a34ae90ef7f7e41166f1f7f74 Mon Sep 17 00:00:00 2001
From: Jon Holman <9025884+JonHolman@users.noreply.github.com>
Date: Thu, 9 Jan 2025 14:54:23 -0800
Subject: [PATCH 07/12] remove uneccessary change

---
 services/database/serverless.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/services/database/serverless.yml b/services/database/serverless.yml
index 9a4bc48af..f29e06793 100644
--- a/services/database/serverless.yml
+++ b/services/database/serverless.yml
@@ -6,7 +6,6 @@ plugins:
   - serverless-s3-bucket-helper
   - serverless-dynamodb
   - serverless-stack-termination-protection
-  - serverless-iam-helper
   - serverless-plugin-scripts
   - serverless-offline
 

From a6d9ac2be23f09b46e51bb2fb3aca563a43bead4 Mon Sep 17 00:00:00 2001
From: Jon Holman <9025884+JonHolman@users.noreply.github.com>
Date: Tue, 14 Jan 2025 08:49:52 -0800
Subject: [PATCH 08/12] Trigger Build


From e892d8feccd850024e6b44900b6bf07b4f82c68d Mon Sep 17 00:00:00 2001
From: Jon Holman <9025884+JonHolman@users.noreply.github.com>
Date: Tue, 14 Jan 2025 09:27:35 -0800
Subject: [PATCH 09/12] Add check for termination protection prior to destroy

---
 src/getCloudFormationTemplatesForStage.ts |  2 +-
 src/run.ts                                | 27 ++++++++++++++++++++++-
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/src/getCloudFormationTemplatesForStage.ts b/src/getCloudFormationTemplatesForStage.ts
index a5c396c19..96df770b4 100644
--- a/src/getCloudFormationTemplatesForStage.ts
+++ b/src/getCloudFormationTemplatesForStage.ts
@@ -18,7 +18,7 @@ async function getAllStacksForRegion(region: string) {
   return stacks;
 }
 
-async function getAllStacksForStage(
+export async function getAllStacksForStage(
   region: string,
   stage: string,
   addFilters?: Tag[]
diff --git a/src/run.ts b/src/run.ts
index e525a6e2a..42da167ce 100644
--- a/src/run.ts
+++ b/src/run.ts
@@ -2,7 +2,10 @@ import yargs from "yargs";
 import * as dotenv from "dotenv";
 import LabeledProcessRunner from "./runner.js";
 import { ServerlessStageDestroyer } from "@stratiformdigital/serverless-stage-destroyer";
-import { getCloudFormationTemplatesForStage } from "./getCloudFormationTemplatesForStage.js";
+import {
+  getAllStacksForStage,
+  getCloudFormationTemplatesForStage,
+} from "./getCloudFormationTemplatesForStage.js";
 import { execSync } from "child_process";
 
 // load .env
@@ -187,6 +190,28 @@ async function destroy_stage(options: {
     });
   }
 
+  const stacks = await getAllStacksForStage(
+    `${process.env.REGION_A}`,
+    options.stage,
+    filters
+  );
+
+  const protectedStacks = stacks
+    .filter((i) => i.EnableTerminationProtection)
+    .map((i) => i.StackName);
+
+  if (protectedStacks.length > 0) {
+    console.log(
+      `We cannot proceed with the destroy because the following stacks have termination protection enabled:\n${protectedStacks.join(
+        "\n"
+      )}`
+    );
+  } else {
+    console.log(
+      "No stacks have termination protection enabled. Proceeding with the destroy."
+    );
+  }
+
   const templates = await getCloudFormationTemplatesForStage(
     `${process.env.REGION_A}`,
     options.stage,

From 74e03a287c7fb731b47e40c09733c7029da20a7d Mon Sep 17 00:00:00 2001
From: Jon Holman <9025884+JonHolman@users.noreply.github.com>
Date: Tue, 14 Jan 2025 09:31:32 -0800
Subject: [PATCH 10/12] Updated to only check for retained resources for
 important stages

---
 src/run.ts | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/run.ts b/src/run.ts
index 42da167ce..f9ebd6ca6 100644
--- a/src/run.ts
+++ b/src/run.ts
@@ -253,9 +253,12 @@ async function destroy_stage(options: {
     }
   );
 
-  if (notRetained.length > 0) {
+  if (
+    ["master", "val", "production"].includes(options.stage) &&
+    notRetained.length > 0
+  ) {
     console.log(
-      "Will not destroy the stage because some important resources are not yet set to be retained:"
+      "Will not destroy the stage because its an important stage and some important resources are not yet set to be retained:"
     );
     notRetained.forEach(({ templateKey, resourceKey }) =>
       console.log(` - ${templateKey}/${resourceKey}`)

From 22d3cf029bce31a08b696e379ce8f48cb5440467 Mon Sep 17 00:00:00 2001
From: Jon Holman <9025884+JonHolman@users.noreply.github.com>
Date: Tue, 14 Jan 2025 11:28:43 -0800
Subject: [PATCH 11/12] Update src/run.ts

Co-authored-by: Pete Dunlap <pete.dunlap@stratiform.digital>
---
 src/run.ts | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/run.ts b/src/run.ts
index f9ebd6ca6..c998bd23d 100644
--- a/src/run.ts
+++ b/src/run.ts
@@ -254,8 +254,10 @@ async function destroy_stage(options: {
   );
 
   if (
-    ["master", "val", "production"].includes(options.stage) &&
-    notRetained.length > 0
+    ["master", "val", "production"].includes(options.stage) {
+    # all the code that gets notRetained
+      if (notRetained.length > 0) {
+       # then the code to bail
   ) {
     console.log(
       "Will not destroy the stage because its an important stage and some important resources are not yet set to be retained:"

From 4df86bf113d47deb629beb68b02981187c0893a9 Mon Sep 17 00:00:00 2001
From: Jon Holman <9025884+JonHolman@users.noreply.github.com>
Date: Tue, 14 Jan 2025 11:56:26 -0800
Subject: [PATCH 12/12] Updated based on PR feedback

---
 src/run.ts | 92 ++++++++++++++++++++++++++----------------------------
 1 file changed, 45 insertions(+), 47 deletions(-)

diff --git a/src/run.ts b/src/run.ts
index c998bd23d..b918090e2 100644
--- a/src/run.ts
+++ b/src/run.ts
@@ -170,6 +170,44 @@ async function deploy(options: { stage: string }) {
   await runner.run_command_and_output("Serverless deploy", deployCmd, ".");
 }
 
+async function getNotRetainedResources(
+  stage: string,
+  filters: { Key: string; Value: string }[] | undefined
+) {
+  const templates = await getCloudFormationTemplatesForStage(
+    `${process.env.REGION_A}`,
+    stage,
+    filters
+  );
+
+  const resourcesToCheck = {
+    [`database-${stage}`]: [
+      "FormAnswersTable",
+      "FormQuestionsTable",
+      "FormTemplatesTable",
+      "FormsTable",
+      "StateFormsTable",
+      "StatesTable",
+      "AuthUserTable",
+    ],
+    [`ui-${stage}`]: ["CloudFrontDistribution"],
+    [`ui-auth-${stage}`]: ["CognitoUserPool"],
+  };
+
+  const notRetained: { templateKey: string; resourceKey: string }[] = [];
+  for (const [templateKey, resourceKeys] of Object.entries(resourcesToCheck)) {
+    resourceKeys.forEach((resourceKey) => {
+      const policy =
+        templates?.[templateKey]?.Resources?.[resourceKey]?.DeletionPolicy;
+      if (policy !== "Retain") {
+        notRetained.push({ templateKey, resourceKey });
+      }
+    });
+  }
+
+  return notRetained;
+}
+
 async function destroy_stage(options: {
   stage: string;
   service: string | undefined;
@@ -206,61 +244,21 @@ async function destroy_stage(options: {
         "\n"
       )}`
     );
+    return;
   } else {
     console.log(
       "No stacks have termination protection enabled. Proceeding with the destroy."
     );
   }
 
-  const templates = await getCloudFormationTemplatesForStage(
-    `${process.env.REGION_A}`,
-    options.stage,
-    filters
-  );
-
-  const resourcesToCheck = [
-    {
-      templateKey: `database-${options.stage}`,
-      resourceKey: "FormAnswersTable",
-    },
-    {
-      templateKey: `database-${options.stage}`,
-      resourceKey: "FormQuestionsTable",
-    },
-    {
-      templateKey: `database-${options.stage}`,
-      resourceKey: "FormTemplatesTable",
-    },
-    { templateKey: `database-${options.stage}`, resourceKey: "FormsTable" },
-    {
-      templateKey: `database-${options.stage}`,
-      resourceKey: "StateFormsTable",
-    },
-    { templateKey: `database-${options.stage}`, resourceKey: "StatesTable" },
-    { templateKey: `database-${options.stage}`, resourceKey: "AuthUserTable" },
-    {
-      templateKey: `ui-${options.stage}`,
-      resourceKey: "CloudFrontDistribution",
-    },
-    { templateKey: `ui-auth-${options.stage}`, resourceKey: "CognitoUserPool" },
-  ];
-
-  const notRetained = resourcesToCheck.filter(
-    ({ templateKey, resourceKey }) => {
-      const policy =
-        templates?.[templateKey]?.Resources?.[resourceKey]?.DeletionPolicy;
-      return policy !== "Retain";
-    }
-  );
+  let notRetained: { templateKey: string; resourceKey: string }[] = [];
+  if (["master", "val", "production"].includes(options.stage)) {
+    notRetained = await getNotRetainedResources(options.stage, filters);
+  }
 
-  if (
-    ["master", "val", "production"].includes(options.stage) {
-    # all the code that gets notRetained
-      if (notRetained.length > 0) {
-       # then the code to bail
-  ) {
+  if (notRetained.length > 0) {
     console.log(
-      "Will not destroy the stage because its an important stage and some important resources are not yet set to be retained:"
+      "Will not destroy the stage because it's an important stage and some important resources are not yet set to be retained:"
     );
     notRetained.forEach(({ templateKey, resourceKey }) =>
       console.log(` - ${templateKey}/${resourceKey}`)