Merge pull request #871 from eisbuk/feature/db-sanity-check-3

Feature/db sanity check 3

packages/client/src/pages/debug/index.tsx

@@ -161,12 +161,12 @@ const DebugPage: React.FC = () => {
               onClick={() =>
                 createFunctionCaller(
                   functions,
-                  CloudFunction.DBSanityCheck
+                  CloudFunction.DBSlotAttendanceCheck
                 )().then(console.log)
               }
               color={ButtonColor.Primary}
             >
-              DB Sanity Check
+              DB Slot / Attendance Check
             </DebugPageButton>
           </div>
 

packages/client/vitest.config.ts

@@ -23,6 +23,7 @@ export default defineConfig({
     // but there's no toll for tests that don't need it.
     testTimeout: __isCI__ ? 15000 : 10000,
     setupFiles: ["./vitest.setup.ts"],
+    maxConcurrency: 3,
   },
   resolve: {
     alias: {

packages/functions/src/checks/https.ts

@@ -7,22 +7,26 @@ import { checkUser, throwUnauth } from "../utils";
 
 import {
   attendanceSlotMismatchAutofix,
-  findSlotAttendanceMismatches,
+  newSanityChecker,
 } from "./slotRelatedDocs";
+import { SanityCheckKind } from "@eisbuk/shared";
 
 /**
  * Goes through all 'slotsByDay' entries, checks each date to see if there are no slots in the day and deletes the day if empty.
  * If all days are empty, deletes the entry (for a month) altogether.
  */
-export const dbSanityCheck = functions
+export const dbSlotAttendanceCheck = functions
   .region(__functionsZone__)
   .https.onCall(
     async ({ organization }: { organization: string }, { auth }) => {
       if (!(await checkUser(organization, auth))) throwUnauth();
 
       const db = admin.firestore();
-
-      return findSlotAttendanceMismatches(db, organization);
+      return newSanityChecker(
+        db,
+        organization,
+        SanityCheckKind.SlotAttendance
+      ).checkAndWrite();
     }
   );
 
@@ -33,15 +37,24 @@ export const dbSlotAttendanceAutofix = functions
       if (!(await checkUser(organization, auth))) throwUnauth();
 
       const db = admin.firestore();
-
-      // TODO: This should be read from the repord document
-      const mismatches = await findSlotAttendanceMismatches(db, organization);
-
-      try {
-        attendanceSlotMismatchAutofix(db, organization, mismatches);
-        return { success: true };
-      } catch (error) {
-        return { success: false, error };
-      }
+      const checker = newSanityChecker(
+        db,
+        organization,
+        SanityCheckKind.SlotAttendance
+      );
+
+      const report = await checker
+        .getLatestReport()
+        // If report doesn't exist, or the latest report had already been fixed, get the new report
+        .then((r) => (!r || r.attendanceFixes ? checker.checkAndWrite() : r));
+
+      const attendanceFixes = await attendanceSlotMismatchAutofix(
+        db,
+        organization,
+        report
+      );
+      checker.writeReport({ ...report, attendanceFixes });
+
+      return attendanceFixes;
     }
   );

packages/functions/src/checks/slotRelatedDocs.ts

@@ -1,9 +1,13 @@
 import admin from "firebase-admin";
 
 import {
+  AttendanceAutofixReport,
   Collection,
   DateMismatchDoc,
+  FirestoreSchema,
   OrgSubCollection,
+  SanityCheckKind,
+  SlotAttendanceUpdate,
   SlotAttendnace,
   SlotInterface,
   SlotSanityCheckReport,
@@ -41,6 +45,9 @@ interface DateCheckPayload {
   entries: EntryDatePayload[];
 }
 
+/**
+ * A util used by slot related check cloud function used to find mismatch between slot and attendance entries.
+ */
 export const findSlotAttendanceMismatches = async (
   db: Firestore,
   organization: string
@@ -136,7 +143,7 @@ export const attendanceSlotMismatchAutofix = async (
   db: Firestore,
   organization: string,
   mismatches: SlotSanityCheckReport
-) => {
+): Promise<AttendanceAutofixReport> => {
   const batch = db.batch();
 
   const { unpairedEntries, dateMismatches } = mismatches;
@@ -150,6 +157,10 @@ export const attendanceSlotMismatchAutofix = async (
     attendances: {},
   });
 
+  const created = {} as Record<string, SlotAttendnace>;
+  const deleted = {} as Record<string, SlotAttendnace>;
+  const updated = {} as Record<string, SlotAttendanceUpdate>;
+
   // Create attendance entry for every slot entry without one
   await Promise.all(
     Object.entries(unpairedEntries).map(async ([id, { existing, missing }]) => {
@@ -165,11 +176,21 @@ export const attendanceSlotMismatchAutofix = async (
           const slotData = slotRef.data() as SlotInterface;
           const attendanceDoc = attendanceFromSlot(slotData);
           batch.set(attendance.doc(id), attendanceDoc, { merge: true });
+
+          // Save the created data for report
+          created[id] = attendanceDoc;
           break;
 
         case existing.includes(OrgSubCollection.Attendance) &&
           missing.includes(OrgSubCollection.Slots):
-          batch.delete(attendance.doc(id));
+          const toDelete = attendance.doc(id);
+
+          batch.delete(toDelete);
+
+          // Save the existing data before deletion (before batch.commit) and store for report
+          deleted[id] = await toDelete
+            .get()
+            .then((snap) => snap.data() as SlotAttendnace);
           break;
 
         default:
@@ -181,9 +202,91 @@ export const attendanceSlotMismatchAutofix = async (
   );
 
   // Update mismatched dates so that attendance has the same date as the corresponding slot
-  Object.entries(dateMismatches).forEach(([id, { slots: date }]) =>
-    batch.set(attendance.doc(id), { date }, { merge: true })
+  await Promise.all(
+    Object.entries(dateMismatches).map(async ([id, { slots: date }]) => {
+      const toUpdate = attendance.doc(id);
+      batch.set(attendance.doc(id), { date }, { merge: true });
+
+      // Save the update for report
+      const before = await toUpdate
+        .get()
+        .then((snap) => snap.data() as SlotAttendnace)
+        .then(({ date }) => date);
+      updated[id] = {
+        date: { before, after: date },
+      };
+    })
   );
 
-  return batch.commit();
+  await batch.commit();
+
+  return {
+    timestamp: DateTime.now().toISO(),
+    created,
+    deleted,
+    updated,
+  };
+};
+
+const getSanityChecksRef = (
+  db: Firestore,
+  organization: string,
+  kind: SanityCheckKind
+) => db.collection(Collection.SanityChecks).doc(organization).collection(kind);
+
+const getLatestSanityCheck = async <K extends SanityCheckKind>(
+  db: Firestore,
+  organization: string,
+  kind: K
+): Promise<SanityCheckReport<K> | undefined> =>
+  getSanityChecksRef(db, organization, kind)
+    .orderBy("id", "asc")
+    .limitToLast(1)
+    .get()
+    .then((snap) =>
+      !snap.docs.length
+        ? undefined
+        : (snap.docs[0].data() as SanityCheckReport<K>)
+    );
+
+const writeSanityCheckReport = async <K extends SanityCheckKind>(
+  db: Firestore,
+  organization: string,
+  kind: K,
+  report: SanityCheckReport<K>
+): Promise<SanityCheckReport<K>> => {
+  await getSanityChecksRef(db, organization, kind).doc(report.id).set(report);
+  return report;
+};
+
+interface SanityCheckerInterface<K extends SanityCheckKind> {
+  check(): Promise<SanityCheckReport<K>>;
+  writeReport: (report: SanityCheckReport<K>) => Promise<SanityCheckReport<K>>;
+  checkAndWrite: () => Promise<SanityCheckReport<K>>;
+  getLatestReport: () => Promise<SanityCheckReport<K> | undefined>;
+}
+
+export const newSanityChecker = <K extends SanityCheckKind>(
+  db: Firestore,
+  organization: string,
+  kind: K
+): SanityCheckerInterface<K> => {
+  const getLatestReport = () => getLatestSanityCheck<K>(db, organization, kind);
+  const writeReport = (report: SanityCheckReport<K>) =>
+    writeSanityCheckReport(db, organization, kind, report);
+
+  const check = (): Promise<SanityCheckReport<K>> =>
+    ({
+      [SanityCheckKind.SlotAttendance]: findSlotAttendanceMismatches(
+        db,
+        organization
+      ),
+    }[kind]);
+
+  const checkAndWrite = () => check().then(writeReport);
+
+  return { getLatestReport, writeReport, check, checkAndWrite };
 };
+
+type SanityCheckReport<K extends SanityCheckKind> =
+  FirestoreSchema["sanityChecks"][string][K];

packages/shared/src/enums/firestore.ts

@@ -17,6 +17,10 @@ export enum Collection {
    * All process delivery queues (such as email, SMS) are stored here, on per-organization basis
    */
   DeliveryQueues = "deliveryQueues",
+  /**
+   * Reports of sanity db sanity check runs.
+   */
+  SanityChecks = "sanityChecks",
 }
 
 export enum OrgSubCollection {
@@ -37,6 +41,11 @@ export enum DeliveryQueue {
   EmailQueue = "emailQueue",
   SMSQueue = "SMSQueue",
 }
+
+export enum SanityCheckKind {
+  SlotAttendance = "slotAttendance",
+}
+
 // endregion
 
 // region slots
@@ -54,5 +63,4 @@ export enum Category {
   /** @TODO Soon to be deprecated */
   // Adults = "adults",
 }
-
 // endregion

packages/shared/src/types/firestore.ts

@@ -6,6 +6,7 @@ import {
   OrgSubCollection,
   Collection,
   DeliveryQueue,
+  SanityCheckKind,
 } from "../enums/firestore";
 
 export interface PrivacyPolicyParams {
@@ -401,13 +402,28 @@ export type DateMismatchDoc = {
   [key in OrgSubCollection]: string;
 };
 
+export type SlotAttendanceUpdate = {
+  [K in keyof SlotAttendnace]?: {
+    before: SlotAttendnace[K];
+    after: SlotAttendnace[K];
+  };
+};
+
+export interface AttendanceAutofixReport {
+  timestamp: string;
+  created: Record<string, SlotAttendnace>;
+  deleted: Record<string, SlotAttendnace>;
+  updated: Record<string, SlotAttendanceUpdate>;
+}
+
 export interface SlotSanityCheckReport {
   /** ISO timestamp of the sanity check run */
   id: string;
   /** A record of "unpaired" docs - slot related docs missing an entry in one or more related collection (keyed by slot id) */
   unpairedEntries: Record<string, UnpairedDoc>;
   /** A record of docs for which the date is mismatched across collections (keyed by slot id) */
   dateMismatches: Record<string, DateMismatchDoc>;
+  attendanceFixes?: AttendanceAutofixReport;
 }
 // #endregion sanityChecks
 
@@ -447,6 +463,11 @@ export interface FirestoreSchema {
   [Collection.Secrets]: {
     [organization: string]: OrganizationSecrets;
   };
+  [Collection.SanityChecks]: {
+    [organization: string]: {
+      [SanityCheckKind.SlotAttendance]: SlotSanityCheckReport;
+    };
+  };
 }
 
 // #endregion firestoreSchema

packages/shared/src/ui/enums/misc.ts

@@ -23,6 +23,6 @@ export enum CloudFunction {
   RemoveInvalidCustomerPhones = "removeInvalidCustomerPhones",
   ClearDeletedCustomersRegistrationAndCategories = "clearDeletedCustomersRegistrationAndCategories",
 
-  DBSanityCheck = "dbSanityCheck",
+  DBSlotAttendanceCheck = "dbSlotAttendanceCheck",
   DBSlotAttendanceAutofix = "dbSlotAttendanceAutofix",
 }