Calculating and Storing Requirement Statistics

package.json

@@ -23,7 +23,8 @@
     "test": "jest",
     "cypress:open": "cypress open",
     "cypress:run": "cypress run",
-    "cypress:e2e": "npm run build:dev && concurrently \"npm run serve:build\" \"wait-on http-get://localhost:8080 && npm run cypress:run\" --kill-others --success first"
+    "cypress:e2e": "npm run build:dev && concurrently \"npm run serve:build\" \"wait-on http-get://localhost:8080 && npm run cypress:run\" --kill-others --success first",
+    "all-tests" : "npm run lint && npm run type-check && npm run test && npm run format:check"
   },
   "dependencies": {
     "@types/intro.js": "^3.0.0",

scripts/firebase-config.ts

@@ -43,6 +43,14 @@ const userCollections = {
   onboarding: 'user-onboarding-data',
 };
 
+const helperCollections = {
+  track: 'track-users',
+  courses: 'courses',
+  availableRostersForCourse: 'available-rosters-for-course',
+  crseIdToCatalogNbr: 'crseid-to-catalognbr',
+  courseFulfillmentStats: 'course-fulfillment-stats',
+};
+
 export const userCollectionNames = Object.values(userCollections);
 
 export const usernameCollection = db.collection(userCollections.name);
@@ -52,7 +60,10 @@ export const overriddenFulfillmentChoicesCollection = db.collection(userCollecti
 export const subjectColorsCollection = db.collection(userCollections.colors);
 export const uniqueIncrementerCollection = db.collection(userCollections.unique);
 export const onboardingDataCollection = db.collection(userCollections.onboarding);
-export const trackUsersCollection = db.collection('track-users');
-export const coursesCollection = db.collection('courses');
-export const availableRostersForCourseCollection = db.collection('available-rosters-for-course');
-export const crseIdToCatalogNbrCollection = db.collection('crseid-to-catalognbr');
+export const trackUsersCollection = db.collection(helperCollections.track);
+export const coursesCollection = db.collection(helperCollections.courses);
+export const availableRostersForCourseCollection = db.collection(
+  helperCollections.availableRostersForCourse
+);
+export const crseIdToCatalogNbrCollection = db.collection(helperCollections.crseIdToCatalogNbr);
+export const courseFulfillmentStats = db.collection(helperCollections.courseFulfillmentStats);

scripts/gen-req-full-stats.ts

@@ -0,0 +1,125 @@
+import {
+  onboardingDataCollection,
+  semestersCollection,
+  toggleableRequirementChoicesCollection,
+  overriddenFulfillmentChoicesCollection,
+  courseFulfillmentStats,
+} from './firebase-config';
+
+import computeGroupedRequirementFulfillmentReports from '../src/requirements/requirement-frontend-computation';
+import computeFulfillmentStats from '../src/requirements/fulfillment-stats';
+import { createAppOnboardingData } from '../src/user-data-converter';
+
+import '../src/requirements/decorated-requirements.json';
+
+// idRequirementFrequency is a hashmap where the key is the requirement ID and the value is
+// an array of maps. Each element in the array represents a slot in the requirement.
+// The map is a hashmap where the key is the course ID and the value is the frequency of the course
+// in the slot.
+let idRequirementFrequency = new Map<string, Map<number, number>[]>();
+/**
+ * Computes the requirement fulfillment statistics for all users. This is done by iterating through
+ * all the users and computing the computeGroupedRequirementFulfillmentReports for each user.
+ * This returns groupedRequirementFulfillmentReport which is then passed to computeFulfillmentStats.
+ * GroupedRequirementFulfillmentReport is a list of RequirementFulfillmentReport where each
+ * RequirementFulfillmentReport represents a requirement and a list of courses that fulfill the
+ * requirement. computeFulfillmentStats then computes the frequency of each course in each slot
+ * of the requirement and stores it in idRequirementFrequency.
+ * @param _callback
+ * @throws Error when computeGroupedRequirementFulfillmentReports fails to compute the fulfillment stats
+ */
+async function computeRequirementFullfillmentStatistics(_callback) {
+  const semQuerySnapshot = await semestersCollection.get();
+  semQuerySnapshot.forEach(async doc => {
+    // obtain the user's semesters, onboarding data, etc...
+    const semestersAndPlans = await doc.data();
+    const onboardingData = (await onboardingDataCollection.doc(doc.id).get()).data();
+    const toggleableRequirementChoices =
+      (await toggleableRequirementChoicesCollection.doc(doc.id).get()).data() ?? {};
+    const overriddenFulfillmentChoices =
+      (await overriddenFulfillmentChoicesCollection.doc(doc.id).get()).data() ?? {};
+
+    // if the user has obboarding data, semesters, etc...
+    if (
+      onboardingData !== undefined &&
+      semestersAndPlans.semesters !== undefined &&
+      toggleableRequirementChoices !== undefined &&
+      overriddenFulfillmentChoices !== undefined
+    ) {
+      // Attempt to compute the fulfillment stats for the user
+      try {
+        // use createAppOnboardingData to convert the onboarding data to the format used by the frontend
+        const newOnboardingData = await createAppOnboardingData(onboardingData);
+
+        // compute the fulfillment stats
+        const res = await computeGroupedRequirementFulfillmentReports(
+          semestersAndPlans.semesters,
+          newOnboardingData,
+          toggleableRequirementChoices,
+          overriddenFulfillmentChoices
+        );
+
+        idRequirementFrequency = await computeFulfillmentStats(
+          res.groupedRequirementFulfillmentReport,
+          idRequirementFrequency
+        );
+      } catch {
+        // There was an error computing the fulfillment stats for the user
+      }
+    }
+  });
+
+  setTimeout(_callback, 120 * 1000); // wait 2 minutes before storing the computed stats
+}
+
+/**
+ * Stores the computed requirement fulfillment statistics in firestore. This is done by iterating
+ * through all the keys in the idRequirementFrequency hashmap and storing the fulfillment stats
+ * for each requirement in firestore. We have to keep only the top fifty courses for each slot
+ * to reduce the size of the data stored in firestore. This is done by sorting the hashmap by
+ * frequency and keeping only the top fifty courses for each slot.
+ * @throws Error when courseFulfillmentStats.doc().set(data) fails to store the data in firestore
+ */
+async function storeComputedRequirementFullfillmentStatistics() {
+  // Change the hashmap to only keep the top fifty courses for each slot
+
+  for (const [reqID, slots] of idRequirementFrequency) {
+    const newSlots: Map<number, number>[] = [];
+    for (const slot of slots) {
+      const newSlot = new Map<number, number>();
+      const sorted = [...slot.entries()].sort((a, b) => b[1] - a[1]);
+
+      for (let i = 0; i < 50; i += 1) {
+        const [course, freq] = sorted[i];
+        newSlot.set(course, freq);
+      }
+
+      newSlots.push(newSlot);
+    }
+    idRequirementFrequency.set(reqID, newSlots);
+  }
+
+  // Storing fulfillment stats in firestore by iterating through the hashmap
+  for (const [reqID, slots] of idRequirementFrequency) {
+    const json = {};
+
+    let i = 0;
+    for (const slot of slots) {
+      const tempJson = {};
+      for (const [course, freq] of slot) {
+        tempJson[course] = freq;
+      }
+      json[i] = tempJson;
+      i += 1;
+    }
+
+    const data = {
+      reqID,
+      slots: json,
+    };
+    courseFulfillmentStats.doc().set(data); // store the data in firestore
+  }
+}
+
+// Run the script
+computeRequirementFullfillmentStatistics(storeComputedRequirementFullfillmentStatistics);

src/requirements/fulfillment-stats.ts

@@ -0,0 +1,65 @@
+/**
+ * @brief This function computes the frequency of courses taken to fulfill a requirement
+ *
+ * @param groups A list of requirement groups containing the courses taken to fulfill the requirements
+ * @param idRequirementFrequency A hashmap of requirement id to a list of frequency maps
+ * @returns A hashmap of requirement id to a list of frequency maps
+ *
+ * @details
+ * This function computes the frequency of courses taken to fulfill a requirement.
+ * The hashmap is of the form: requirement id -> list of frequency maps
+ * The list of frequency maps is of the form: slot number -> course id -> frequency
+ *
+ * @note
+ * The hashmap is passed in as a parameter to avoid creating a new hashmap every time this function is called.
+ * This function is called multiple times in the main algorithm.
+ *
+ * @warning
+ * This function assumes that the hashmap passed in is empty.
+ *
+ * @bug
+ * None known
+ *
+ * @todo
+ * None
+ */
+export default function computeFulfillmentStats(
+  groups: readonly GroupedRequirementFulfillmentReport[],
+  idRequirementFrequency: Map<string, Map<number, number>[]>
+) {
+  // Iterate over all groups
+  const res = idRequirementFrequency;
+  groups.forEach(currentGroup => {
+    // Iterate over all requirements in the group
+    const { reqs } = currentGroup;
+    reqs.forEach(reqFulfillment => {
+      // Obtain the requirement ID and the list of courses taken to fulfill the requirement
+      const key: string = reqFulfillment.requirement.id;
+      const { safeCourses } = reqFulfillment.fulfillment;
+
+      // Obtain the frequency list for this particular group's requirements
+      const freqList = res.get(key) ?? [];
+
+      // Iterate over all slots in the requirement group
+      // console.log(safeCourses.length);
+      for (let slotNumber = 0; slotNumber < safeCourses.length; slotNumber += 1) {
+        if (freqList.length === slotNumber) {
+          freqList.push(new Map<number, number>());
+        }
+        const currentCourseSlot = safeCourses[slotNumber];
+        const currentRequirementSlotFreq = freqList[slotNumber];
+
+        // Iterate over all courses taken to fulfill the req-slot
+        for (let j = 0; j < currentCourseSlot.length; j += 1) {
+          const currentCourseId = currentCourseSlot[j].courseId;
+          const pastFreq = currentRequirementSlotFreq.get(currentCourseId) ?? 0;
+          currentRequirementSlotFreq.set(currentCourseId, pastFreq + 1);
+        }
+        freqList[slotNumber] = currentRequirementSlotFreq; // Update the frequency list
+      }
+      res.set(key, freqList); // Update the hashmap with the new frequency list
+    });
+  });
+
+  return res; // return the hashmap
+}