Merge pull request #202 from sandboxnu/pp/mark-past-terms-inactive-3

Pp/mark past terms inactive 3

.github/workflows/test.yaml

@@ -105,6 +105,11 @@ jobs:
           psql -U postgres -h localhost -p 5432 -d searchneu_dev -c \
             "INSERT INTO sections (class_hash, id, crn, last_update_time) VALUES ('neu.edu/202240/CS/2501', 'neu.edu/202240/CS/2501/987654321', '987654321', '1999-04-03 18:34:35.882');"
 
+        # Set all Term IDS to active for testing purposes
+      - run: |-
+          psql -U postgres -h localhost -p 5432 -d searchneu_dev -c \
+            "UPDATE term_ids SET active = true;"
+
       - name: Run the updater ONLY ONCE, so that it removes the newly-inserted section with an outdated lastUpdateTime
         run: UPDATE_ONLY_ONCE=true LOG_LEVEL=VERBOSE yarn updater | tee _updater.log
 

prisma/migrations/20231015192547_add_active_field_to_term_info/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "term_ids" ADD COLUMN     "active" BOOLEAN NOT NULL DEFAULT true;

prisma/schema.prisma

@@ -121,6 +121,7 @@ model TermInfo {
   termId     String @unique(map: "term_ids.term_id_unique") @map("term_id")
   subCollege String @map("sub_college")
   text       String
+  active     Boolean @default(true)
 
   @@map("term_ids")
 }

scrapers/classes/parsersxe/termListParser.ts

@@ -9,6 +9,15 @@ class TermListParser {
   serializeTermsList(
     termsFromBanner: { code: string; description: string }[]
   ): TermInfo[] {
+    const activeTermInfos = termsFromBanner.filter(
+      (term) => !term.description.includes("View Only")
+    );
+    const activeTermIds = activeTermInfos.map((termInfo) =>
+      Number(termInfo.code)
+    );
+    /* The smallest active termInfo code. 
+    All termInfo's with codes greater than or equal to this are considered active.*/
+    const minActiveTermInfoCode = Math.min(...activeTermIds);
     return termsFromBanner.map((term) => {
       const subCollege = this.determineSubCollegeName(term.description);
 
@@ -24,6 +33,7 @@ class TermListParser {
         termId: term.code,
         text: text,
         subCollege: subCollege,
+        active: Number(term.code) >= minActiveTermInfoCode,
       };
     });
   }

scrapers/classes/parsersxe/tests/__snapshots__/termListParser.test.js.snap

@@ -3,18 +3,21 @@
 exports[`termListParser pulls out relevant data 1`] = `
 Array [
   Object {
+    "active": true,
     "host": "neu.edu",
     "subCollege": "CPS",
     "termId": "202034",
     "text": "Spring 2020 Semester",
   },
   Object {
+    "active": true,
     "host": "neu.edu",
     "subCollege": "LAW",
     "termId": "202032",
     "text": "Spring 2020 Semester",
   },
   Object {
+    "active": true,
     "host": "neu.edu",
     "subCollege": "NEU",
     "termId": "202030",

scrapers/classes/parsersxe/tests/bannerv9Parser.test.ts

@@ -38,18 +38,21 @@ describe("getAllTermInfos", () => {
   it("serializes the term list", async () => {
     expect(await bannerv9.getAllTermInfos()).toEqual([
       {
+        active: true,
         host: "neu.edu",
         subCollege: "NEU",
         termId: "3",
         text: "Fall 2022 Semester",
       },
       {
+        active: true,
         host: "neu.edu",
         subCollege: "LAW",
         termId: "2",
         text: "Summer 2022 Semester",
       },
       {
+        active: true,
         host: "neu.edu",
         subCollege: "CPS",
         termId: "1",

services/dumpProcessor.ts

@@ -16,10 +16,7 @@ import {
   Section,
   convertSectionToPrismaType,
 } from "../types/types";
-import {
-  ParsedCourseSR,
-  convertCourseToPrismaType,
-} from "../types/scraperTypes";
+import { convertCourseToPrismaType } from "../types/scraperTypes";
 
 class DumpProcessor {
   /**
@@ -223,18 +220,20 @@ class DumpProcessor {
       termIdsWithData.includes(t.termId)
     );
 
-    // Upsert new term IDs, along with their names and sub college
-    for (const { termId, subCollege, text } of termInfosWithData) {
+    // Upsert new term IDs, along with their names, sub college, and active status
+    for (const { termId, subCollege, text, active } of termInfosWithData) {
       await prisma.termInfo.upsert({
         where: { termId },
         update: {
           text,
           subCollege,
+          active,
         },
         create: {
           termId,
           text,
           subCollege,
+          active,
         },
       });
     }

services/updater.ts

@@ -59,9 +59,10 @@ class Updater {
       return termsStr.split(",");
     }
 
-    // Get term IDs from our database
+    // Get active term IDs from our database
     const termInfos = await prisma.termInfo.findMany({
       orderBy: { termId: "desc" },
+      where: { active: true },
       take: NUMBER_OF_TERMS_TO_UPDATE,
     });
 

tests/database/dumpProcessor.test.seq.ts

@@ -25,11 +25,13 @@ afterAll(async () => {
 
 const termInfos: TermInfo[] = [
   {
+    active: true,
     termId: "123456",
     subCollege: "NEU",
     text: "This is some text",
   },
   {
+    active: true,
     termId: "654321",
     subCollege: "LAW",
     text: "This is some more text",
@@ -103,6 +105,7 @@ describe("with termInfos", () => {
   it("deletes old termInfos", async () => {
     await prisma.termInfo.create({
       data: {
+        active: true,
         termId: "1",
         subCollege: "NEU",
         text: "hello",
@@ -122,6 +125,7 @@ describe("with termInfos", () => {
   it("doesn't delete old termInfos if deleteOutdatedData is false", async () => {
     await prisma.termInfo.create({
       data: {
+        active: true,
         termId: "1",
         subCollege: "NEU",
         text: "hello",
@@ -141,6 +145,7 @@ describe("with termInfos", () => {
   it("updates existing termInfos", async () => {
     await prisma.termInfo.create({
       data: {
+        active: true,
         termId: "654321",
         subCollege: "fake college",
         text: "This is some more text",

tests/database/updater.test.seq.ts

@@ -314,6 +314,42 @@ describe("Updater", () => {
     );
   });
 
+  it("inactive terms aren't saved in the updater", async () => {
+    const INACTIVE_TERM = "000000";
+    const ACTIVE_TERM = "000001";
+    let termIdsToUpdate = await Updater.getTermIdsToUpdate();
+    // Shouldn't include these termId
+    expect(termIdsToUpdate).not.toContain(INACTIVE_TERM);
+    expect(termIdsToUpdate).not.toContain(ACTIVE_TERM);
+
+    // Create an inactive term in database
+    await prisma.termInfo.create({
+      data: {
+        termId: INACTIVE_TERM,
+        subCollege: "NEU",
+        text: "description",
+        active: false,
+      },
+    });
+
+    termIdsToUpdate = await Updater.getTermIdsToUpdate();
+
+    // Still shouldn't include this inactive term
+    expect(termIdsToUpdate).not.toContain(INACTIVE_TERM);
+    // Create an active term
+    await prisma.termInfo.create({
+      data: {
+        termId: ACTIVE_TERM,
+        subCollege: "NEU",
+        text: "description",
+        active: true,
+      },
+    });
+    termIdsToUpdate = await Updater.getTermIdsToUpdate();
+    // Should include the active term
+    expect(termIdsToUpdate).toContain(ACTIVE_TERM);
+  });
+
   describe("getNotificationInfo", () => {
     let FUNDIES_ONE_COURSE;
     let FUNDIES_TWO_COURSE;

types/types.ts

@@ -264,6 +264,7 @@ export interface TermInfo {
   termId: string;
   subCollege: string;
   text: string;
+  active: boolean;
 }
 
 export interface CourseRef {