feat(programs): undergrad program requirements

apps/api/src/graphql/resolvers/programs.ts

@@ -4,10 +4,13 @@ import {
   majorsQuerySchema,
   minorRequirementsQuerySchema,
   minorsQuerySchema,
+  type programRequirementSchema,
   specializationRequirementsQuerySchema,
   specializationsQuerySchema,
+  ugradRequirementsQuerySchema,
 } from "$schema";
 import { ProgramsService } from "$services";
+import type { z } from "@hono/zod-openapi";
 import { GraphQLError } from "graphql/error";
 
 export const programResolvers = {
@@ -72,17 +75,31 @@ export const programResolvers = {
         });
       return res;
     },
+    ugradRequirements: async (_: unknown, args: { query?: unknown }, { db }: GraphQLContext) => {
+      const parsedArgs = ugradRequirementsQuerySchema.parse(args?.query);
+      const service = new ProgramsService(db);
+      const res = await service.getUgradRequirements(parsedArgs);
+      if (!res)
+        throw new GraphQLError("Undergraduate requirements block not found", {
+          extensions: { code: "NOT_FOUND" },
+        });
+      return res;
+    },
   },
   ProgramRequirement: {
     // x outside this typehint is malformed data; meh
-    __resolveType: (x: { requirementType: "Course" | "Unit" | "Group" }) => {
+    __resolveType: (x: {
+      requirementType: z.infer<typeof programRequirementSchema>["requirementType"];
+    }) => {
       switch (x?.requirementType) {
         case "Course":
           return "ProgramCourseRequirement";
         case "Unit":
           return "ProgramUnitRequirement";
         case "Group":
           return "ProgramGroupRequirement";
+        case "Marker":
+          return "ProgramMarkerRequirement";
       }
     },
   },

apps/api/src/graphql/schema/programs.ts

@@ -54,14 +54,23 @@ type ProgramGroupRequirement implements ProgramRequirementBase @cacheControl(max
     requirements: JSON!
 }
 
-union ProgramRequirement = ProgramCourseRequirement | ProgramUnitRequirement | ProgramGroupRequirement
+type ProgramMarkerRequirement implements ProgramRequirementBase @cacheControl(maxAge: 86400) {
+    label: String!
+}
+
+union ProgramRequirement = ProgramCourseRequirement | ProgramUnitRequirement | ProgramGroupRequirement | ProgramMarkerRequirement
 
 type Program @cacheControl(maxAge: 86400) {
     id: String!
     name: String!
     requirements: [ProgramRequirement]!
 }
 
+type UgradRequirements @cacheControl(maxAge: 86400) {
+    id: String!,
+    requirements: [ProgramRequirement!]!,
+}
+
 input ProgramRequirementsQuery {
     programId: String!
 }
@@ -78,12 +87,17 @@ input SpecializationsQuery {
     majorId: String!
 }
 
+input UgradRequrementsQuery {
+    id: String!
+}
+
 extend type Query {
     majors(query: MajorsQuery): [MajorPreview!]!
     minors(query: MinorsQuery): [MinorPreview!]!
     specializations(query: SpecializationsQuery): [SpecializationPreview!]!
     major(query: ProgramRequirementsQuery!): Program!
     minor(query: ProgramRequirementsQuery!): Program!
     specialization(query: ProgramRequirementsQuery!): Program!
+    ugradRequirements(query: UgradRequrementsQuery!): UgradRequirements!
 }
 `;

apps/api/src/rest/routes/programs.ts

@@ -16,6 +16,8 @@ import {
   specializationRequirementsResponseSchema,
   specializationsQuerySchema,
   specializationsResponseSchema,
+  ugradRequirementsQuerySchema,
+  ugradRequirementsResponseSchema,
 } from "$schema";
 import { ProgramsService } from "$services";
 import { OpenAPIHono, createRoute } from "@hono/zod-openapi";
@@ -183,6 +185,36 @@ const specializationRequirements = createRoute({
   },
 });
 
+const ugradRequirements = createRoute({
+  summary: "Retrieve undergraduate requirements",
+  operationId: "ugradRequirements",
+  tags: ["Programs"],
+  method: "get",
+  path: "/ugradRequirements",
+  description: "Retrieve requirements external to, but required for, for all undergraduate degrees",
+  request: { query: ugradRequirementsQuerySchema },
+  responses: {
+    200: {
+      content: {
+        "application/json": { schema: responseSchema(ugradRequirementsResponseSchema) },
+      },
+      description: "Successful operation",
+    },
+    404: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Specialization not found",
+    },
+    422: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Parameters failed validation",
+    },
+    500: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Server error occurred",
+    },
+  },
+});
+
 programsRouter.get(
   "*",
   productionCache({ cacheName: "anteater-api", cacheControl: "max-age=86400" }),
@@ -254,4 +286,19 @@ programsRouter.openapi(specializationRequirements, async (c) => {
       );
 });
 
+programsRouter.openapi(ugradRequirements, async (c) => {
+  const query = c.req.valid("query");
+  const service = new ProgramsService(database(c.env.DB.connectionString));
+  const res = await service.getUgradRequirements(query);
+  return res
+    ? c.json({ ok: true, data: ugradRequirementsResponseSchema.parse(res) }, 200)
+    : c.json(
+        {
+          ok: false,
+          message: "Couldn't find this undergraduate requirements block; check your ID?",
+        },
+        404,
+      );
+});
+
 export { programsRouter };

apps/api/src/schema/programs.ts

@@ -44,6 +44,13 @@ export const specializationRequirementsQuerySchema = z.object({
   }),
 });
 
+export const ugradRequirementsQuerySchema = z.object({
+  id: z
+    .literal("UC")
+    .or(z.literal("GE"))
+    .openapi({ description: "The requirements block to fetch" }),
+});
+
 export const programRequirementBaseSchema = z.object({
   label: z.string().openapi({
     description: "Human description of this requirement",
@@ -141,11 +148,25 @@ export const programGroupRequirementSchema: z.ZodType<
     },
   });
 
+export const programMarkerRequirementSchema = programRequirementBaseSchema
+  .extend({
+    requirementType: z.literal("Marker"),
+  })
+  .openapi({
+    description:
+      "A rule which must be marked as complete, e.g the fulfillment of GE VIII (foreign language) via high school credit",
+    example: {
+      label: "Entry Level Writing",
+      requirementType: "Marker",
+    },
+  });
+
 // one day someone will figure out z.discriminatedUnion
 export const programRequirementSchema = z.union([
   programCourseRequirementSchema,
   programUnitRequirementSchema,
   programGroupRequirementSchema,
+  programMarkerRequirementSchema,
 ]);
 
 export const majorsResponseSchema = z.array(
@@ -246,3 +267,10 @@ export const specializationRequirementsResponseSchema = programRequirementsRespo
     example: "CS:Specialization in Bioinformatics",
   }),
 });
+
+export const ugradRequirementsResponseSchema = z.object({
+  id: z.string().openapi({ description: "ID of the requirements block fetched" }),
+  requirements: z
+    .array(programRequirementSchema)
+    .openapi({ description: "The requirements in this requirements block" }),
+});

apps/api/src/services/programs.ts

@@ -5,10 +5,11 @@ import type {
   minorsQuerySchema,
   specializationRequirementsQuerySchema,
   specializationsQuerySchema,
+  ugradRequirementsQuerySchema,
 } from "$schema";
 import type { database } from "@packages/db";
 import { eq, sql } from "@packages/db/drizzle";
-import { degree, major, minor, specialization } from "@packages/db/schema";
+import { degree, major, minor, schoolRequirement, specialization } from "@packages/db/schema";
 import type { z } from "zod";
 
 export class ProgramsService {
@@ -107,4 +108,17 @@ export class ProgramsService {
 
     return got;
   }
+
+  async getUgradRequirements(query: z.infer<typeof ugradRequirementsQuerySchema>) {
+    const [got] = await this.db
+      .select({
+        id: schoolRequirement.id,
+        requirements: schoolRequirement.requirements,
+      })
+      .from(schoolRequirement)
+      .where(eq(schoolRequirement.id, query.id))
+      .limit(1);
+
+    return got ? got : null;
+  }
 }

apps/data-pipeline/degreeworks-scraper/src/components/AuditParser.ts

@@ -1,12 +1,12 @@
 import type { Block, Rule } from "$types";
 import type { database } from "@packages/db";
 import { eq } from "@packages/db/drizzle";
-import { course } from "@packages/db/schema";
 import type {
   DegreeWorksProgram,
   DegreeWorksProgramId,
   DegreeWorksRequirement,
 } from "@packages/db/schema";
+import { course } from "@packages/db/schema";
 
 export class AuditParser {
   private static readonly specOrOtherMatcher = /"type":"(?:SPEC|OTHER)","value":"\w+"/g;
@@ -94,6 +94,16 @@ export class AuditParser {
       .limit(1);
   }
 
+  /**
+   * Certain requirements change label depending on whether they've been fulfilled.
+   * This is undesirable for archival so we will quash these.
+   * @param label The label before transformation.
+   * @private
+   */
+  private static suppressLabelPolymorphism(label: string) {
+    return label.replaceAll(/ Satisfied/g, " Required").replaceAll(/ satisfied/g, " required");
+  }
+
   async ruleArrayToRequirements(ruleArray: Rule[]) {
     const ret: DegreeWorksRequirement[] = [];
     for (const rule of ruleArray) {
@@ -129,14 +139,14 @@ export class AuditParser {
             .map(([x]) => x);
           if (rule.requirement.classesBegin) {
             ret.push({
-              label: rule.label,
+              label: AuditParser.suppressLabelPolymorphism(rule.label),
               requirementType: "Course",
               courseCount: Number.parseInt(rule.requirement.classesBegin, 10),
               courses,
             });
           } else if (rule.requirement.creditsBegin) {
             ret.push({
-              label: rule.label,
+              label: AuditParser.suppressLabelPolymorphism(rule.label),
               requirementType: "Unit",
               unitCount: Number.parseInt(rule.requirement.creditsBegin, 10),
               courses,
@@ -146,7 +156,7 @@ export class AuditParser {
         }
         case "Group": {
           ret.push({
-            label: rule.label,
+            label: AuditParser.suppressLabelPolymorphism(rule.label),
             requirementType: "Group",
             requirementCount: Number.parseInt(rule.requirement.numberOfGroups),
             requirements: await this.ruleArrayToRequirements(rule.ruleArray),
@@ -155,20 +165,31 @@ export class AuditParser {
         }
         case "IfStmt": {
           const rules = this.flattenIfStmt([rule]);
-          if (rules.length > 1 && !rules.some((x) => x.ruleType === "Block")) {
-            ret.push({
-              label: "Select 1 of the following",
-              requirementType: "Group",
-              requirementCount: 1,
-              requirements: await this.ruleArrayToRequirements(rules),
-            });
+          if (!rules.some((x) => x.ruleType === "Block")) {
+            if (rules.length > 1) {
+              ret.push({
+                label: "Select 1 of the following",
+                requirementType: "Group",
+                requirementCount: 1,
+                requirements: await this.ruleArrayToRequirements(rules),
+              });
+            } else if (rules.length === 1) {
+              ret.push(...(await this.ruleArrayToRequirements(rules)));
+            }
           }
           break;
         }
+        case "Complete":
+        case "Incomplete":
+          ret.push({
+            label: AuditParser.suppressLabelPolymorphism(rule.label),
+            requirementType: "Marker",
+          });
+          break;
         case "Subset": {
           const requirements = await this.ruleArrayToRequirements(rule.ruleArray);
           ret.push({
-            label: rule.label,
+            label: AuditParser.suppressLabelPolymorphism(rule.label),
             requirementType: "Group",
             requirementCount: Object.keys(requirements).length,
             requirements,

apps/data-pipeline/degreeworks-scraper/src/components/DegreeworksClient.ts

@@ -37,6 +37,44 @@ export class DegreeworksClient {
 
   sleep = (ms: number = this.delay) => new Promise((r) => setTimeout(r, ms));
 
+  static formatQueryParams(params: Record<string, string>) {
+    return Object.entries(params)
+      .map((kv) => kv.map(encodeURIComponent).join("="))
+      .join("&");
+  }
+
+  async getUgradRequirements(): Promise<[Block, Block] | undefined> {
+    const params = DegreeworksClient.formatQueryParams({
+      studentId: this.studentId,
+      // more schools are possible, see this.getMapping("schools"), but we want undergrad requirements
+      school: "U",
+      // there is no difference regardless of which of the four bachelor's degrees we ask for: BA, BFA, BMUS, BS
+      degree: "BS",
+    });
+    const res = await fetch(`${DegreeworksClient.AUDIT_URL}?${params}`, {
+      method: "GET",
+      headers: this.headers,
+    });
+    await this.sleep();
+
+    const json: DWAuditResponse = await res.json().catch(() => ({ error: "" }));
+    if ("error" in json) {
+      return;
+    }
+
+    // "DEGREE" block doesn't contain any material requirements, "SCHOOL" block has what we need
+    const ucRequirements = json.blockArray.find((b) => b.requirementType === "SCHOOL");
+    if (!ucRequirements) {
+      return;
+    }
+    const geRequirements = json.blockArray.find((b) => b.requirementType === "PROGRAM");
+    if (!geRequirements) {
+      return;
+    }
+
+    return [ucRequirements, geRequirements];
+  }
+
   async getMajorAudit(
     degree: string,
     school: string,