feat(refactor indices): Refactor main index and organize our transfor…

…m pattern (#337) * remove rai data from main index, remove obe railist component * is this more readable? not sure * remove transforms that don't do anything * move action trasnforms * bind the transforms to their index * rename, move, etc * and now the changelog index.... * break apart sink handlers for clarity * refactor onemac * add missing error logging * rename transforms * naming * fix tests * Fix error * Fix logic for legacy * Unscrew attachments * Rename schema and type * Fix some codeclimate issues * feat(refactor indices): refactor the refactor * feat(refactor indices): refactor the refactor * feat(refactor indices): refactor the refactor * fix broken callouts to getLatestRai * Remove obe code * Correct rai logic --------- Co-authored-by: Paul Kim <[email protected]>
Enterprise-CMCS · Jan 24, 2024 · ab536a3 · ab536a3
1 parent b984055
commit ab536a3
Show file tree

Hide file tree

Showing 39 changed files with 599 additions and 1,033 deletions.
diff --git a/src/packages/shared-types/action-types/index.ts b/src/packages/shared-types/action-types/index.ts
@@ -3,3 +3,6 @@ export * from "./issue-rai";
 export * from "./respond-to-rai";
 export * from "./withdraw-rai";
 export * from "./withdraw-package";
+export * from "./new-submission";
+export * from "./legacy-submission";
+export * from "./seatool";
diff --git a/src/packages/shared-types/action-types/issue-rai.ts b/src/packages/shared-types/action-types/issue-rai.ts
@@ -1,34 +1,14 @@
 import { z } from "zod";
-import { onemacAttachmentSchema, handleAttachment } from "../attachments";
+import { attachmentSchema } from "../attachments";
 
 export const raiIssueSchema = z.object({
   id: z.string(),
   authority: z.string(),
   origin: z.string(),
   requestedDate: z.number(),
-  attachments: z.array(onemacAttachmentSchema).nullish(),
+  attachments: z.array(attachmentSchema).nullish(),
   additionalInformation: z.string().nullable().default(null),
   submitterName: z.string(),
   submitterEmail: z.string(),
 });
 export type RaiIssue = z.infer<typeof raiIssueSchema>;
-
-export const transformRaiIssue = (id: string) => {
-  return raiIssueSchema.transform((data) => ({
-    id,
-    rais: {
-      [data.requestedDate]: {
-        request: {
-          attachments:
-            data.attachments?.map((attachment) => {
-              return handleAttachment(attachment);
-            }) ?? null,
-          additionalInformation: data.additionalInformation,
-          submitterName: data.submitterName,
-          submitterEmail: data.submitterEmail,
-        },
-      },
-    },
-  }));
-};
-export type RaiIssueTransform = z.infer<ReturnType<typeof transformRaiIssue>>;
diff --git a/src/packages/shared-types/action-types/legacy-submission.ts b/src/packages/shared-types/action-types/legacy-submission.ts
@@ -0,0 +1,12 @@
+import { z } from "zod";
+import { legacyAttachmentSchema } from "../attachments";
+
+// Event schema for legacy records
+export const onemacLegacySchema = z.object({
+  additionalInformation: z.string().nullable().default(null),
+  submitterName: z.string(),
+  submitterEmail: z.string(),
+  attachments: z.array(legacyAttachmentSchema).nullish(),
+  raiWithdrawEnabled: z.boolean().default(false),
+});
+export type OnemacLegacy = z.infer<typeof onemacLegacySchema>;
diff --git a/src/packages/shared-types/action-types/new-submission.ts b/src/packages/shared-types/action-types/new-submission.ts
@@ -0,0 +1,15 @@
+import { z } from "zod";
+import { attachmentSchema } from "../attachments";
+
+// This is the event schema for ne submissions from our system
+export const onemacSchema = z.object({
+  authority: z.string(),
+  origin: z.string(),
+  additionalInformation: z.string().nullable().default(null),
+  submitterName: z.string(),
+  submitterEmail: z.string(),
+  attachments: z.array(attachmentSchema).nullish(),
+  raiWithdrawEnabled: z.boolean().default(false),
+});
+
+export type OneMac = z.infer<typeof onemacSchema>;
diff --git a/src/packages/shared-types/action-types/respond-to-rai.ts b/src/packages/shared-types/action-types/respond-to-rai.ts
@@ -1,37 +1,15 @@
 import { z } from "zod";
-import { onemacAttachmentSchema, handleAttachment } from "../attachments";
+import { attachmentSchema } from "../attachments";
 
 export const raiResponseSchema = z.object({
   id: z.string(),
   authority: z.string(),
   origin: z.string(),
   requestedDate: z.number(),
   responseDate: z.number(),
-  attachments: z.array(onemacAttachmentSchema).nullish(),
+  attachments: z.array(attachmentSchema).nullish(),
   additionalInformation: z.string().nullable().default(null),
   submitterName: z.string(),
   submitterEmail: z.string(),
 });
 export type RaiResponse = z.infer<typeof raiResponseSchema>;
-
-export const transformRaiResponse = (id: string) => {
-  return raiResponseSchema.transform((data) => ({
-    id,
-    rais: {
-      [data.requestedDate]: {
-        response: {
-          attachments:
-            data.attachments?.map((attachment) => {
-              return handleAttachment(attachment);
-            }) ?? null,
-          additionalInformation: data.additionalInformation,
-          submitterName: data.submitterName,
-          submitterEmail: data.submitterEmail,
-        },
-      },
-    },
-  }));
-};
-export type RaiResponseTransform = z.infer<
-  ReturnType<typeof transformRaiResponse>
->;
diff --git a/src/packages/shared-types/action-types/seatool.ts b/src/packages/shared-types/action-types/seatool.ts
@@ -0,0 +1,62 @@
+import { z } from "zod";
+
+// Reusable schema for officer data.
+export const seatoolOfficerSchema = z.object({
+  OFFICER_ID: z.number(),
+  FIRST_NAME: z.string(),
+  LAST_NAME: z.string(),
+});
+export type SeatoolOfficer = z.infer<typeof seatoolOfficerSchema>;
+
+export const seatoolSchema = z.object({
+  ACTION_OFFICERS: z.array(seatoolOfficerSchema).nullish(),
+  LEAD_ANALYST: z.array(seatoolOfficerSchema).nullable(),
+  PLAN_TYPES: z
+    .array(
+      z.object({
+        PLAN_TYPE_NAME: z.string(),
+      })
+    )
+    .nonempty()
+    .nullable(),
+  STATE_PLAN: z.object({
+    SUBMISSION_DATE: z.number().nullable(),
+    PLAN_TYPE: z.number().nullable(),
+    LEAD_ANALYST_ID: z.number().nullable(),
+    CHANGED_DATE: z.number().nullable(),
+    APPROVED_EFFECTIVE_DATE: z.number().nullable(),
+    PROPOSED_DATE: z.number().nullable(),
+    SPW_STATUS_ID: z.number().nullable(),
+    STATE_CODE: z.string().nullish(),
+    STATUS_DATE: z.number().nullish(),
+    SUMMARY_MEMO: z.string().nullish(),
+    TITLE_NAME: z.string().nullish(),
+  }),
+  SPW_STATUS: z
+    .array(
+      z.object({
+        SPW_STATUS_DESC: z.string().nullable(),
+        SPW_STATUS_ID: z.number().nullable(),
+      })
+    )
+    .nullable(),
+  RAI: z
+    .array(
+      z.object({
+        RAI_RECEIVED_DATE: z.number().nullable(),
+        RAI_REQUESTED_DATE: z.number().nullable(),
+        RAI_WITHDRAWN_DATE: z.number().nullable(),
+      })
+    )
+    .nullable(),
+  ACTIONTYPES: z
+    .array(
+      z.object({
+        ACTION_ID: z.number(),
+        ACTION_NAME: z.string(),
+        PLAN_TYPE_ID: z.number(),
+      })
+    )
+    .nullable(),
+});
+export type SeaTool = z.infer<typeof seatoolSchema>;
diff --git a/src/packages/shared-types/action-types/toggle-withdraw-rai-enabled.ts b/src/packages/shared-types/action-types/toggle-withdraw-rai-enabled.ts
@@ -11,13 +11,3 @@ export const toggleWithdrawRaiEnabledSchema = z.object({
 export type ToggleWithdrawRaiEnabled = z.infer<
   typeof toggleWithdrawRaiEnabledSchema
 >;
-
-export const transformToggleWithdrawRaiEnabled = (id: string) => {
-  return toggleWithdrawRaiEnabledSchema.transform((data) => ({
-    id,
-    raiWithdrawEnabled: data.raiWithdrawEnabled,
-  }));
-};
-export type ToggleWithdrawRaiEnabledTransform = z.infer<
-  ReturnType<typeof transformToggleWithdrawRaiEnabled>
->;
diff --git a/src/packages/shared-types/action-types/withdraw-package.ts b/src/packages/shared-types/action-types/withdraw-package.ts
@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { onemacAttachmentSchema } from "../attachments";
+import { attachmentSchema } from "../attachments";
 
 // Temporary, will be refactored to an extendable schema with Brian/Mike's back-end
 // work.
@@ -11,20 +11,9 @@ export const withdrawPackageSchema = z.object({
     .string()
     .max(4000, "This field may only be up to 4000 characters.")
     .optional(),
-  attachments: z.array(onemacAttachmentSchema),
+  attachments: z.array(attachmentSchema),
   submitterName: z.string(),
   submitterEmail: z.string(),
 });
 
 export type WithdrawPackage = z.infer<typeof withdrawPackageSchema>;
-
-export const transformWithdrawPackage = (id: string) => {
-  // This does nothing.  Just putting the mechanics in place.
-  return withdrawPackageSchema.transform((data) => ({
-    id,
-    raiWithdrawEnabled: false,
-  }));
-};
-export type WithdrawPackageTransform = z.infer<
-  ReturnType<typeof transformWithdrawPackage>
->;
diff --git a/src/packages/shared-types/action-types/withdraw-rai.ts b/src/packages/shared-types/action-types/withdraw-rai.ts
@@ -1,38 +1,15 @@
 import { z } from "zod";
-import { onemacAttachmentSchema, handleAttachment } from "../attachments";
+import { attachmentSchema } from "../attachments";
 
 export const raiWithdrawSchema = z.object({
   id: z.string(),
   authority: z.string(),
   origin: z.string(),
   requestedDate: z.number(),
   withdrawnDate: z.number(),
-  attachments: z.array(onemacAttachmentSchema).nullish(),
+  attachments: z.array(attachmentSchema).nullish(),
   additionalInformation: z.string().nullable().default(null),
   submitterName: z.string(),
   submitterEmail: z.string(),
 });
 export type RaiWithdraw = z.infer<typeof raiWithdrawSchema>;
-
-export const transformRaiWithdraw = (id: string) => {
-  return raiWithdrawSchema.transform((data) => ({
-    id,
-    raiWithdrawEnabled: false,
-    rais: {
-      [data.requestedDate]: {
-        withdraw: {
-          attachments:
-            data.attachments?.map((attachment) => {
-              return handleAttachment(attachment);
-            }) ?? null,
-          additionalInformation: data.additionalInformation,
-          submitterName: data.submitterName,
-          submitterEmail: data.submitterEmail,
-        },
-      },
-    },
-  }));
-};
-export type RaiWithdrawTransform = z.infer<
-  ReturnType<typeof transformRaiWithdraw>
->;
diff --git a/src/packages/shared-types/attachments.ts b/src/packages/shared-types/attachments.ts
@@ -26,44 +26,38 @@ export const attachmentTitleMap: Record<string, string> = {
 export type AttachmentKey = keyof typeof attachmentTitleMap;
 export type AttachmentTitle = typeof attachmentTitleMap[AttachmentKey];
 
-export const onemacAttachmentSchema = z.object({
-  s3Key: z.string().nullish(),
+export const attachmentSchema = z.object({
   filename: z.string(),
   title: z.string(),
-  contentType: z.string().nullish(),
-  url: z.string().url().nullish(),
-  bucket: z.string().nullish(),
-  key: z.string().nullish(),
-  uploadDate: z.number().nullish(),
+  bucket: z.string(),
+  key: z.string(),
+  uploadDate: z.number(),
 });
-export type OnemacAttachmentSchema = z.infer<typeof onemacAttachmentSchema>;
+export type Attachment = z.infer<typeof attachmentSchema>;
 
-export function handleAttachment(attachment: OnemacAttachmentSchema) {
-  let bucket = "";
-  let key = "";
-  let uploadDate = 0;
-  if ("bucket" in attachment) {
-    bucket = attachment.bucket as string;
-  }
-  if ("key" in attachment) {
-    key = attachment.key as string;
-  }
-  if ("uploadDate" in attachment) {
-    uploadDate = attachment.uploadDate as number;
-  }
-  if (bucket == "") {
-    const parsedUrl = s3ParseUrl(attachment.url || "");
-    if (!parsedUrl) return null;
-    bucket = parsedUrl.bucket;
-    key = parsedUrl.key;
-    uploadDate = parseInt(attachment.s3Key?.split("/")[0] || "0");
-  }
+// Attachment schema for legacy records
+export const legacyAttachmentSchema = z.object({
+  s3Key: z.string(),
+  filename: z.string(),
+  title: z.string(),
+  contentType: z.string(),
+  url: z.string().url(),
+});
+export type LegacyAttachment = z.infer<typeof legacyAttachmentSchema>;
 
+export function handleLegacyAttachment(
+  attachment: LegacyAttachment
+): Attachment | null {
+  const parsedUrl = s3ParseUrl(attachment.url || "");
+  if (!parsedUrl) return null;
+  const bucket = parsedUrl.bucket;
+  const key = parsedUrl.key;
+  const uploadDate = parseInt(attachment.s3Key?.split("/")[0] || "0");
   return {
     title: attachment.title,
     filename: attachment.filename,
     uploadDate,
     bucket,
     key,
-  };
+  } as Attachment;
 }
diff --git a/src/packages/shared-types/index.ts b/src/packages/shared-types/index.ts
@@ -1,9 +1,6 @@
 export * from "./issue";
 export * from "./user";
 export * from "./errors";
-export * from "./seatool";
-export * from "./onemac";
-export * from "./onemacLegacy";
 export * as opensearch from "./opensearch";
 export * from "./uploads";
 export * from "./actions";
@@ -15,3 +12,4 @@ export * from "./inputs";
 export * from "./states";
 export * from "./statusHelper";
 export * from "./guides";
+export * from "./lambda-events";
diff --git a/src/packages/shared-types/lambda-events.ts b/src/packages/shared-types/lambda-events.ts
@@ -0,0 +1,22 @@
+export type KafkaEvent = {
+  /**
+   * @example "SelfManagedKafka"
+   */
+  eventSource: string;
+  /**
+   * @example: "b-1.master-msk.zf7e0q.c7.kafka.us-east-1.amazonaws.com:9094,b-2.master-msk.zf7e0q.c7.kafka.us-east-1.amazonaws.com:9094,b-3.master-msk.zf7e0q.c7.kafka.us-east-1.amazonaws.com:9094"
+   */
+  bootstrapServers: string; // comma separated string
+  records: Record<string, KafkaRecord[]>;
+};
+
+export type KafkaRecord = {
+  topic: string;
+  partition: number;
+  offset: number;
+  timestamp: number;
+  timestampType: string;
+  key: string;
+  headers: Record<string, string>;
+  value: string; // Kafka records typically have values as base64-encoded strings
+};