diff --git a/package.json b/package.json index dd1bcdf91..7917d1059 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,8 @@ "test": "jest", "cypress:open": "cypress open", "cypress:run": "cypress run", - "cypress:e2e": "npm run build:dev && concurrently \"npm run serve:build\" \"wait-on http-get://localhost:8080 && npm run cypress:run\" --kill-others --success first" + "cypress:e2e": "npm run build:dev && concurrently \"npm run serve:build\" \"wait-on http-get://localhost:8080 && npm run cypress:run\" --kill-others --success first", + "all-tests" : "npm run lint && npm run type-check && npm run test && npm run format:check" }, "dependencies": { "@types/intro.js": "^3.0.0", diff --git a/scripts/firebase-config.ts b/scripts/firebase-config.ts index c67644d2b..46c1de8e5 100644 --- a/scripts/firebase-config.ts +++ b/scripts/firebase-config.ts @@ -43,6 +43,14 @@ const userCollections = { onboarding: 'user-onboarding-data', }; +const helperCollections = { + track: 'track-users', + courses: 'courses', + availableRostersForCourse: 'available-rosters-for-course', + crseIdToCatalogNbr: 'crseid-to-catalognbr', + courseFulfillmentStats: 'course-fulfillment-stats', +}; + export const userCollectionNames = Object.values(userCollections); export const usernameCollection = db.collection(userCollections.name); @@ -52,7 +60,10 @@ export const overriddenFulfillmentChoicesCollection = db.collection(userCollecti export const subjectColorsCollection = db.collection(userCollections.colors); export const uniqueIncrementerCollection = db.collection(userCollections.unique); export const onboardingDataCollection = db.collection(userCollections.onboarding); -export const trackUsersCollection = db.collection('track-users'); -export const coursesCollection = db.collection('courses'); -export const availableRostersForCourseCollection = db.collection('available-rosters-for-course'); -export const crseIdToCatalogNbrCollection = db.collection('crseid-to-catalognbr'); +export const trackUsersCollection = db.collection(helperCollections.track); +export const coursesCollection = db.collection(helperCollections.courses); +export const availableRostersForCourseCollection = db.collection( + helperCollections.availableRostersForCourse +); +export const crseIdToCatalogNbrCollection = db.collection(helperCollections.crseIdToCatalogNbr); +export const courseFulfillmentStats = db.collection(helperCollections.courseFulfillmentStats); diff --git a/scripts/gen-req-full-stats.ts b/scripts/gen-req-full-stats.ts new file mode 100644 index 000000000..3a839165c --- /dev/null +++ b/scripts/gen-req-full-stats.ts @@ -0,0 +1,116 @@ +import { + onboardingDataCollection, + semestersCollection, + toggleableRequirementChoicesCollection, + overriddenFulfillmentChoicesCollection, + courseFulfillmentStats, +} from './firebase-config'; + +import computeGroupedRequirementFulfillmentReports from '../src/requirements/requirement-frontend-computation'; +import computeFulfillmentStats from '../src/requirements/fulfillment-stats'; +import { createAppOnboardingData } from '../src/user-data-converter'; + +import '../src/requirements/decorated-requirements.json'; + +// idRequirementFrequency is a hashmap where the key is the requirement ID and the value is +// an array of maps. Each element in the array represents a slot in the requirement. +// The map is a hashmap where the key is the course ID and the value is the frequency of the course +// in the slot. +const idRequirementFrequency = new Map[]>(); + +/** + * Computes the requirement fulfillment statistics for all users. This is done by iterating through + * all the users and computing the computeGroupedRequirementFulfillmentReports for each user. + * This returns groupedRequirementFulfillmentReport which is then passed to computeFulfillmentStats. + * GroupedRequirementFulfillmentReport is a list of RequirementFulfillmentReport where each + * RequirementFulfillmentReport represents a requirement and a list of courses that fulfill the + * requirement. computeFulfillmentStats then computes the frequency of each course in each slot + * of the requirement and stores it in idRequirementFrequency. + * @param _callback is a function that is called after the fulfillment stats have been computed + * @throws Error when computeGroupedRequirementFulfillmentReports fails to compute the fulfillment stats + */ +async function computeRequirementFullfillmentStatistics(_callback) { + let numberOfErrors = 0; + const semQuerySnapshot = await semestersCollection.get(); + const promises = semQuerySnapshot.docs.map(async doc => { + // obtain the user's semesters, onboarding data, etc... + const semesters = (await doc.data()).semesters ?? {}; + const onboardingData = (await onboardingDataCollection.doc(doc.id).get()).data() ?? {}; + const toggleableRequirementChoices = + (await toggleableRequirementChoicesCollection.doc(doc.id).get()).data() ?? {}; + const overriddenFulfillmentChoices = + (await overriddenFulfillmentChoicesCollection.doc(doc.id).get()).data() ?? {}; + + // Attempt to compute the fulfillment stats for the user + try { + // use createAppOnboardingData to convert the onboarding data to the format used by the frontend + const newOnboardingData = await createAppOnboardingData(onboardingData); + + // compute the fulfillment stats + const res = await computeGroupedRequirementFulfillmentReports( + semesters, + newOnboardingData, + toggleableRequirementChoices, + overriddenFulfillmentChoices + ); + + await computeFulfillmentStats( + res.groupedRequirementFulfillmentReport, + idRequirementFrequency + ); + } catch { + // There was an error computing the fulfillment stats for the user + console.log(`${numberOfErrors} : Error computing fulfillment stats for ${doc.id}`); + numberOfErrors += 1; + } + }); + + await Promise.all(promises); + _callback(); +} + +/** + * Stores the computed requirement fulfillment statistics in firestore. This is done by iterating + * through all the keys in the idRequirementFrequency hashmap and storing the fulfillment stats + * for each requirement in firestore. We have to keep only the top fifty courses for each slot + * to reduce the size of the data stored in firestore. This is done by sorting the hashmap by + * frequency and keeping only the top fifty courses for each slot. + * @throws Error when courseFulfillmentStats.doc().set(data) fails to store the data in firestore + */ +async function storeComputedRequirementFullfillmentStatistics() { + // Change the hashmap to only keep the top fifty courses for each slot + for (const [reqID, slots] of idRequirementFrequency) { + const newSlots: Map[] = []; + for (const slot of slots) { + const newSlot = new Map(); + const sorted = [...slot.entries()].sort((a, b) => b[1] - a[1]); + + const numberOfCourses = sorted.length > 50 ? 50 : sorted.length; + for (let i = 0; i < numberOfCourses; i += 1) { + const [course, freq] = sorted[i]; + newSlot.set(course, freq); + } + + newSlots.push(newSlot); + } + idRequirementFrequency.set(reqID, newSlots); + } + + // Storing fulfillment stats in firestore by iterating through the hashmap + for (const [reqID, slots] of idRequirementFrequency) { + const reqFrequenciesJson = {}; + for (let i = 0; i < slots.length; i += 1) { + const slot = slots[i]; + const slotFrequenciesJson = {}; + for (const [course, freq] of slot) { + slotFrequenciesJson[course] = freq; + } + reqFrequenciesJson[i] = slotFrequenciesJson; + } + const ID = reqID.replace('/', '[FORWARD_SLASH]'); + courseFulfillmentStats.doc(ID).set(reqFrequenciesJson); // store the data in firestore + } +} + +// Run the script +computeRequirementFullfillmentStatistics(storeComputedRequirementFullfillmentStatistics); diff --git a/src/requirements/fulfillment-stats.ts b/src/requirements/fulfillment-stats.ts new file mode 100644 index 000000000..d6bec18a4 --- /dev/null +++ b/src/requirements/fulfillment-stats.ts @@ -0,0 +1,52 @@ +/** + * @brief This function computes the frequency of courses taken to fulfill a requirement + * + * @param groups A list of requirement groups containing the courses taken to fulfill the requirements + * @param idRequirementFrequency A hashmap of requirement id to a list of frequency maps + * + * @details + * This function computes the frequency of courses taken to fulfill a requirement. + * The hashmap is of the form: requirement id -> list of frequency maps + * The list of frequency maps is of the form: slot number -> course id -> frequency + * + * @note + * The hashmap is passed in as a parameter to avoid creating a new hashmap every time this function is called. + * This function is called multiple times in the main algorithm. + */ +export default function computeFulfillmentStats( + groups: readonly GroupedRequirementFulfillmentReport[], + idRequirementFrequency: Map[]> +) { + // Iterate over all groups + groups.forEach(currentGroup => { + // Iterate over all requirements in the group + const { reqs } = currentGroup; + reqs.forEach(reqFulfillment => { + // Obtain the requirement ID and the list of courses taken to fulfill the requirement + const key: string = reqFulfillment.requirement.id; + const { safeCourses } = reqFulfillment.fulfillment; + + // Obtain the frequency list for this particular group's requirements + const freqList = idRequirementFrequency.get(key) ?? []; + + // Iterate over all slots in the requirement group + // console.log(safeCourses.length); + for (let slotNumber = 0; slotNumber < safeCourses.length; slotNumber += 1) { + if (freqList.length === slotNumber) { + freqList.push(new Map()); + } + const currentCourseSlot = safeCourses[slotNumber]; + const currentRequirementSlotFreq = freqList[slotNumber]; + + // Iterate over all courses taken to fulfill the req-slot + for (let j = 0; j < currentCourseSlot.length; j += 1) { + const currentCourseId = currentCourseSlot[j].courseId; + const pastFreq = currentRequirementSlotFreq.get(currentCourseId) ?? 0; + currentRequirementSlotFreq.set(currentCourseId, pastFreq + 1); + } + freqList[slotNumber] = currentRequirementSlotFreq; // Update the frequency list + } + idRequirementFrequency.set(key, freqList); // Update the hashmap with the new frequency list + }); + }); +}